Index: /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.env
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.env	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.env	(revision 1134)
@@ -0,0 +1,9 @@
+module unload netcdf
+module unload hdf5
+
+export HDF5_INC_DIR=$HOME/lib/hdf5_intel/include
+export HDF5_LIB_DIR=$HOME/lib/hdf5_intel/lib
+
+export NETCDF_INC_DIR=$HOME/lib/netcdf_intel/include
+export NETCDF_LIB_DIR=$HOME/lib/netcdf_intel/lib
+
Index: /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.fcm
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.fcm	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.fcm	(revision 1134)
@@ -0,0 +1,24 @@
+################################################################################
+###################                Projet XIOS               ###################
+################################################################################
+
+%CCOMPILER      mpicc -openmp -D_intelmpi -D_usingEP  
+%FCOMPILER      mpif90 -openmp
+%LINKER         mpif90 -openmp -Wl,-rpath -Wl,$HOME/lib/netcdf_f_intel/lib
+
+%BASE_CFLAGS    -ansi -w -lcurl
+%PROD_CFLAGS    -O3 -DBOOST_DISABLE_ASSERTS
+%DEV_CFLAGS     -g -O2 
+%DEBUG_CFLAGS   -g -inline-debug-info -traceback 
+
+%BASE_FFLAGS    -D__NONE__ 
+%PROD_FFLAGS    -O3
+%DEV_FFLAGS     -g -O2
+%DEBUG_FFLAGS   -g 
+
+%BASE_INC       -D__NONE__
+%BASE_LD        -lstdc++
+
+%CPP            cpp
+%FPP            cpp -P
+%MAKE           gmake
Index: /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.path
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.path	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-CURIE_Intel.path	(revision 1134)
@@ -0,0 +1,15 @@
+NETCDF_INCDIR="-I $NETCDF_INC_DIR -I $HOME/lib/netcdf_f_intel/include"
+NETCDF_LIBDIR="-L $NETCDF_LIB_DIR -L $HOME/lib/netcdf_f_intel/lib"
+NETCDF_LIB="-lnetcdff -lnetcdf"
+
+MPI_INCDIR=""
+MPI_LIBDIR=""
+MPI_LIB=""
+
+HDF5_INCDIR="-I $HDF5_INC_DIR"
+HDF5_LIBDIR="-L $HDF5_LIB_DIR"
+HDF5_LIB="-lhdf5_hl -lhdf5 -lhdf5 -lz -ldl -lcurl"
+
+OASIS_INCDIR="-I$PWD/../../oasis3-mct/BLD/build/lib/psmile.MPI1"
+OASIS_LIBDIR="-L$PWD/../../oasis3-mct/BLD/lib"
+OASIS_LIB="-lpsmile.MPI1 -lscrip -lmct -lmpeu"
Index: /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.env
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.env	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.env	(revision 1134)
@@ -1,6 +1,7 @@
-export HDF5_INC_DIR=$HOME/hdf5/include
-export HDF5_LIB_DIR=$HOME/hdf5/lib
 
-export NETCDF_INC_DIR=$HOME/netcdf4/include
-export NETCDF_LIB_DIR=$HOME/netcdf4/lib
+export HDF5_INC_DIR=$HOME/lib/hdf5/include
+export HDF5_LIB_DIR=$HOME/lib/hdf5/lib
 
+export NETCDF_INC_DIR=$HOME/lib/netcdf/include
+export NETCDF_LIB_DIR=$HOME/lib/netcdf/lib
+
Index: /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.fcm
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.fcm	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.fcm	(revision 1134)
@@ -3,7 +3,7 @@
 ################################################################################
 
-%CCOMPILER      mpicc
-%FCOMPILER      mpif90
-%LINKER         mpif90  
+%CCOMPILER      mpicc -fopenmp -D_openmpi -D_usingEP 
+%FCOMPILER      mpif90 -fopenmp
+%LINKER         mpif90 -fopenmp -D_openmpi -D_usingEP 
 
 %BASE_CFLAGS    -ansi -w
Index: /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.path
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.path	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-GCC_LINUX.path	(revision 1134)
@@ -1,13 +1,13 @@
-NETCDF_INCDIR="-I $NETCDF_INC_DIR"
-NETCDF_LIBDIR="-L $NETCDF_LIB_DIR"
-NETCDF_LIB="-lnetcdff -lnetcdf"
+NETCDF_INCDIR="-I $NETCDF_INC_DIR -I $HOME/lib/netcdf_f/include"
+NETCDF_LIBDIR="-L $NETCDF_LIB_DIR -L $HOME/lib/netcdf_f/lib"
+NETCDF_LIB=" -lnetcdf"
 
-MPI_INCDIR=""
-MPI_LIBDIR=""
+MPI_INCDIR="-I /usr/local/include"
+MPI_LIBDIR="-L /usr/local/lib"
 MPI_LIB=""
 
 HDF5_INCDIR="-I $HDF5_INC_DIR"
 HDF5_LIBDIR="-L $HDF5_LIB_DIR"
-HDF5_LIB="-lhdf5_hl -lhdf5 -lhdf5 -lz"
+HDF5_LIB="-lhdf5_hl -lhdf5 -lhdf5 -lz -ldl -lcurl"
 
 OASIS_INCDIR="-I$PWD/../../oasis3-mct/BLD/build/lib/psmile.MPI1"
Index: /XIOS/dev/branch_yushan_merged/arch/arch-X64_ADA.fcm
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-X64_ADA.fcm	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-X64_ADA.fcm	(revision 1134)
@@ -3,7 +3,7 @@
 ################################################################################
 
-%CCOMPILER      mpiicc
-%FCOMPILER      mpiifort 
-%LINKER         mpiifort  -nofor-main
+%CCOMPILER      mpiicc -qopenmp -D_usingEP -D_intelmpi
+%FCOMPILER      mpiifort -qopenmp -D_usingEP -D_intelmpi
+%LINKER         mpiifort  -nofor-main -qopenmp -D_usingEP -D_intelmpi
 
 %BASE_CFLAGS    -diag-disable 1125 -diag-disable 279
Index: /XIOS/dev/branch_yushan_merged/arch/arch-X64_CURIE.fcm
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-X64_CURIE.fcm	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-X64_CURIE.fcm	(revision 1134)
@@ -3,6 +3,6 @@
 ################################################################################
 
-%CCOMPILER      mpicc
-%FCOMPILER      mpif90
+%CCOMPILER      mpicc -openmp -D_openmpi -D_usingEP
+%FCOMPILER      mpif90 -openmp -D_openmpi -D_usingEP
 %LINKER         mpif90  -nofor-main
 
Index: /XIOS/dev/branch_yushan_merged/arch/arch-local.env
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-local.env	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-local.env	(revision 1134)
@@ -0,0 +1,7 @@
+
+export HDF5_INC_DIR=$HOME/LIB/hdf5-1.8.16/hdf5/include
+export HDF5_LIB_DIR=$HOME/LIB/hdf5-1.8.16/hdf5/lib
+
+export NETCDF_INC_DIR=/usr/local/include
+export NETCDF_LIB_DIR=/usr/local/lib
+
Index: /XIOS/dev/branch_yushan_merged/arch/arch-local.fcm
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-local.fcm	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-local.fcm	(revision 1134)
@@ -0,0 +1,24 @@
+################################################################################
+###################                Projet XIOS               ###################
+################################################################################
+
+%CCOMPILER      mpicc -fopenmp -D_openmpi -D_usingEP 
+%FCOMPILER      mpif90 -fopenmp
+%LINKER         mpif90 -fopenmp -D_openmpi -D_usingEP 
+
+%BASE_CFLAGS    -ansi -w -D_GLIBCXX_USE_CXX11_ABI=0
+%PROD_CFLAGS    -O3 -DBOOST_DISABLE_ASSERTS
+%DEV_CFLAGS     -g -O2 
+%DEBUG_CFLAGS   -g 
+
+%BASE_FFLAGS    -D__NONE__ 
+%PROD_FFLAGS    -O3
+%DEV_FFLAGS     -g -O2
+%DEBUG_FFLAGS   -g 
+
+%BASE_INC       -D__NONE__
+%BASE_LD        -lstdc++
+
+%CPP            cpp
+%FPP            cpp -P
+%MAKE           make
Index: /XIOS/dev/branch_yushan_merged/arch/arch-local.path
===================================================================
--- /XIOS/dev/branch_yushan_merged/arch/arch-local.path	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/arch/arch-local.path	(revision 1134)
@@ -0,0 +1,15 @@
+NETCDF_INCDIR="-I $NETCDF_INC_DIR"
+NETCDF_LIBDIR="-L $NETCDF_LIB_DIR"
+NETCDF_LIB=" -lnetcdf"
+
+MPI_INCDIR=""
+MPI_LIBDIR=""
+MPI_LIB=""
+
+HDF5_INCDIR="-I $HDF5_INC_DIR"
+HDF5_LIBDIR="-L $HDF5_LIB_DIR"
+HDF5_LIB="-lhdf5_hl -lhdf5 -lhdf5 -lz -ldl "
+
+OASIS_INCDIR="-I$PWD/../../oasis3-mct/BLD/build/lib/psmile.MPI1"
+OASIS_LIBDIR="-L$PWD/../../oasis3-mct/BLD/lib"
+OASIS_LIB="-lpsmile.MPI1 -lscrip -lmct -lmpeu"
Index: /XIOS/dev/branch_yushan_merged/bld.cfg
===================================================================
--- /XIOS/dev/branch_yushan_merged/bld.cfg	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/bld.cfg	(revision 1134)
@@ -29,12 +29,14 @@
 src::netcdf $PWD/extern/netcdf4
 src::remap $PWD/extern/remap/src
+src::src_ep_dev $PWD/extern/src_ep_dev
 bld::lib xios
-bld::target libxios.a 
+#bld::target libxios.a 
 #bld::target generate_fortran_interface.exe 
-bld::target xios_server.exe 
+#bld::target xios_server.exe 
 #bld::target test_remap.exe
 #bld::target test_regular.exe
 #bld::target test_expand_domain.exe
 #bld::target test_new_features.exe test_unstruct_complete.exe 
+bld::target test_omp.exe test_complete_omp.exe
 bld::target test_client.exe test_complete.exe test_xios2_cmip6.exe
 #bld::target test_connectivity_expand.exe
@@ -66,2 +68,3 @@
 bld::excl_dep        use::netcdf
 bld::excl_dep        inc::mpif.h
+bld::excl_dep        use::omp_lib
Index: /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp	(revision 1134)
@@ -368,5 +368,5 @@
 			double srcArea = recvArea[rank][n1];
 			double w = (*it)->area;
-      if (quantity) w/=srcArea ;
+                        if (quantity) w/=srcArea ;
 
 			/* first order: src value times weight (weight = supermesh area), later divide by target area */
Index: /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.hpp	(revision 1134)
@@ -3,4 +3,8 @@
 #include "parallel_tree.hpp"
 #include "mpi.hpp"
+
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -18,5 +22,5 @@
 {
 public:
-       Mapper(MPI_Comm comm=MPI_COMM_WORLD) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
+       Mapper(ep_lib::MPI_Comm comm=MPI_COMM_WORLD) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
        ~Mapper();
        void setVerbosity(verbosity v) {verbose=v ;}
@@ -67,5 +71,5 @@
 
        CParallelTree sstree;
-       MPI_Comm communicator ;
+       ep_lib::MPI_Comm communicator ;
        std::vector<Elt>  sourceElements ;
        std::vector<Node> sourceMesh ;
Index: /XIOS/dev/branch_yushan_merged/extern/remap/src/mpi_routing.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/remap/src/mpi_routing.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/extern/remap/src/mpi_routing.cpp	(revision 1134)
@@ -5,4 +5,7 @@
 #include "timerRemap.hpp"
 #include <iostream>
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -122,6 +125,8 @@
 	CTimer::get("CMPIRouting::init(reduce_scatter)").print();
 
-	MPI_Alloc_mem(nbTarget *sizeof(int), MPI_INFO_NULL, &targetRank);
-	MPI_Alloc_mem(nbSource *sizeof(int), MPI_INFO_NULL, &sourceRank);
+	MPI_Info info_null;
+
+	MPI_Alloc_mem(nbTarget *sizeof(int), info_null, &targetRank);
+	MPI_Alloc_mem(nbSource *sizeof(int), info_null, &sourceRank);
 
 	targetRankToIndex = new int[mpiSize];
@@ -150,5 +155,9 @@
 	for (int i = 0; i < nbSource; i++)
 	{
+		#ifdef _usingEP
+		MPI_Irecv(&sourceRank[i], 1, MPI_INT, -1, 0, communicator, &request[indexRequest]);
+		#else
 		MPI_Irecv(&sourceRank[i], 1, MPI_INT, MPI_ANY_SOURCE, 0, communicator, &request[indexRequest]);
+		#endif
 		indexRequest++;
 	}
@@ -170,5 +179,9 @@
 	for (int i = 0; i < nbSource; i++)
 	{
+		#ifdef _usingEP
+		MPI_Irecv(&sourceRank[i], 1, MPI_INT, -1, 0, communicator, &request[indexRequest]);
+		#else
 		MPI_Irecv(&sourceRank[i], 1, MPI_INT, MPI_ANY_SOURCE, 0, communicator, &request[indexRequest]);
+		#endif
 		indexRequest++;
 	}
Index: /XIOS/dev/branch_yushan_merged/extern/remap/src/parallel_tree.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/remap/src/parallel_tree.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/extern/remap/src/parallel_tree.hpp	(revision 1134)
@@ -6,4 +6,7 @@
 #include "mpi_cascade.hpp"
 #include "mpi.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace sphereRemap {
@@ -12,5 +15,5 @@
 {
 public:
-	CParallelTree(MPI_Comm comm);
+	CParallelTree(ep_lib::MPI_Comm comm);
 	~CParallelTree();
 
@@ -34,5 +37,5 @@
 	vector<CSampleTree> treeCascade; // first for sample tree, then for routing tree
 	CMPICascade cascade;
-  MPI_Comm communicator ;
+        ep_lib::MPI_Comm communicator ;
 
 };
Index: /XIOS/dev/branch_yushan_merged/extern/remap/src/tree.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/remap/src/tree.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/extern/remap/src/tree.cpp	(revision 1134)
@@ -142,6 +142,8 @@
 	root->parent = 0;
 	root->leafCount = 0;
-// initialize root node on the sphere
-  root->centre.x=1 ; root->centre.y=0 ; root->centre.z=0 ; 
+        // initialize root node on the sphere
+        root->centre.x=1 ; 
+        root->centre.y=0 ; 
+        root->centre.z=0 ; 
 	root->radius = 0.;
 	root->reinserted = false;
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_alltoall.cpp	(revision 1134)
@@ -0,0 +1,25 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+//#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    ::MPI_Aint typesize, llb;
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(sendtype), &llb, &typesize);
+
+    for(int i=0; i<comm.ep_comm_ptr->size_rank_info[0].second; i++)
+    {
+      MPI_Gather((char*)sendbuf+i*sendcount*typesize, sendcount, sendtype, recvbuf, recvcount, recvtype, i, comm);
+    }
+
+    return 0;
+  }
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_barrier.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_barrier.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_barrier.cpp	(revision 1134)
@@ -0,0 +1,69 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+  int MPI_Barrier(MPI_Comm comm)
+  {
+    if(comm.is_intercomm) return MPI_Barrier_intercomm(comm);
+
+    if(comm.is_ep)
+    {
+      int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+
+      MPI_Barrier_local(comm);
+
+      if(ep_rank_loc == 0)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+
+        ::MPI_Barrier(mpi_comm);
+      }
+
+      MPI_Barrier_local(comm);
+
+      return 0;
+    }
+    else if(comm.mpi_comm != MPI_COMM_NULL_STD )
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      ::MPI_Barrier(mpi_comm);
+      return 0;
+    }
+
+    else return 0;
+  }
+
+
+  int MPI_Barrier_intercomm(MPI_Comm comm)
+  {
+
+    int ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+
+    MPI_Barrier_local(comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Barrier(mpi_comm);
+    }
+
+    MPI_Barrier_local(comm);
+
+    return 0;
+
+  }
+
+
+  int MPI_Barrier_local(MPI_Comm comm)
+  {
+    Message_Check(comm);
+    comm.ep_barrier->wait();
+  }
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_bcast.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_bcast.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_bcast.cpp	(revision 1134)
@@ -0,0 +1,307 @@
+/*!
+   \file ep_bcast.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Bcast
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib
+{
+  int MPI_Bcast_local(void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Bcast_local_int(buffer, count, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Bcast_local_float(buffer, count, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Bcast_local_double(buffer, count, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Bcast_local_long(buffer, count, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Bcast_local_char(buffer, count, comm);
+    }
+    else
+    {
+      printf("MPI_Bcast Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Bcast_local_int(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *tmp = static_cast<int*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_float(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *tmp = static_cast<float*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_double(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *tmp = static_cast<double*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast_local_char(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *tmp = static_cast<char*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_long(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *tmp = static_cast<long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Bcast_local_ulong(void *buf, int count, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *tmp = static_cast<unsigned long*>(buf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(tmp+j, tmp+j+min(BUFFER_SIZE, count-j), buffer);
+        }
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+
+      if(my_rank != 0)
+      {
+        #pragma omp flush
+        #pragma omp critical (read_from_buffer)
+        {
+          copy(buffer, buffer+min(BUFFER_SIZE, count-j), tmp+j);
+        }
+      }
+
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_rank_loc = comm.rank_map->at(root).first;
+
+
+    // if root is not master thread, send first to master
+    if(root_ep_rank_loc != 0 && mpi_rank == root_mpi_rank)
+    {
+      innode_memcpy(root_ep_rank_loc, buffer, 0, buffer, count, datatype, comm);
+    }
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Bcast(buffer, count, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local(buffer, count, datatype, comm);
+
+    return 0;
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_create.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_create.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_create.cpp	(revision 1134)
@@ -0,0 +1,282 @@
+/*!
+   \file ep_create.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI endpoint function: MPI_Comm_create_endpoints
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+
+namespace ep_lib {
+
+  /*!
+    Dynamic creation of endpoints for each MPI process.
+    The output of this function is an array of communicator handles of length num_ep, where each handle
+    corresponds to a nez local randk in the output communicator.
+    Once created, endpoints behave as MPI processes.
+    \param [in] mpi_comm Parent MPI communicator.
+    \param [in] num_ep Number of endpoints per process.
+    \param [out] info Information of the EP creation.
+    \param [out] out_comm_hdls Handles of EP communicators.
+  */
+  #ifdef _intelmpi
+  int MPI_Comm_create_endpoints(int base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+    int base_rank;
+    int base_size;
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+									  // parent_comm can also be endpoints communicators
+
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm_ptr;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    // printf("ep_lib::MPI_Comm_create_endpoints()       OK from int\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+  #elif _openmpi
+  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+
+    int base_rank;
+    int base_size;
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+                                                 // parent_comm can also be endpoints communicators ?
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm_ptr;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    //printf("ep_lib::MPI_Comm_create_endpoints()       OK from void*\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+  #endif
+  
+  
+  int MPI_Comm_create_endpoints(MPI_Comm base_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls)
+  {
+
+    int base_rank;
+    int base_size;
+    
+    assert(!base_comm.is_ep);
+
+    ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm.mpi_comm);
+
+    ::MPI_Comm_size(mpi_base_comm, &base_size);  // ep_lib::mpi_comm_size
+    ::MPI_Comm_rank(mpi_base_comm, &base_rank);  // ep_lib::mpi_comm_rank
+                    // parent_comm can also be endpoints communicators
+
+    std::vector<int> recv_num_ep(base_size);
+
+    out_comm_hdls = new MPI_Comm[num_ep];
+
+    for (int idx = 0; idx < num_ep; ++idx)
+    {
+      out_comm_hdls[idx].is_ep = true;
+      out_comm_hdls[idx].is_intercomm = false;
+      out_comm_hdls[idx].ep_comm_ptr = new ep_communicator;
+      out_comm_hdls[idx].mpi_comm = base_comm.mpi_comm;
+      out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls;
+      out_comm_hdls[idx].ep_comm_ptr->comm_label = 0;
+    }
+
+    ::MPI_Allgather(&num_ep, 1, MPI_INT_STD, &recv_num_ep[0], 1, MPI_INT_STD, mpi_base_comm);
+
+
+    int sum = 0;  // representing total ep number of process with smaller rank
+    for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; }
+
+    int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0);
+
+    out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep);
+
+    out_comm_hdls[0].my_buffer = new BUFFER;
+    out_comm_hdls[0].my_buffer->buf_double = new double[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_float  = new float[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_int    = new int[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_long   = new long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_ulong  = new unsigned long[BUFFER_SIZE];
+    out_comm_hdls[0].my_buffer->buf_char   = new char[BUFFER_SIZE];
+
+    out_comm_hdls[0].rank_map = new RANK_MAP;
+    out_comm_hdls[0].rank_map->resize(ep_size);
+
+
+    for (int i = 1; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier;
+      out_comm_hdls[i].my_buffer  = out_comm_hdls[0].my_buffer;
+      out_comm_hdls[i].rank_map   = out_comm_hdls[0].rank_map;
+    }
+
+
+    for (int i = 0; i < num_ep; i++)
+    {
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep);
+      out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size);
+
+      out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list;
+    }
+
+
+    int ind = 0;
+
+    for(int i=0; i<base_size; i++)
+    {
+      for(int j=0; j<recv_num_ep[i]; j++)
+      {
+        out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i);
+        ind++;
+      }
+    }
+
+    // printf("ep_lib::MPI_Comm_create_endpoints()       OK from MPI_Comm\n");
+
+    return 0;
+
+  } //MPI_Comm_create_endpoints
+
+
+} //namespace ep_lib
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.cpp	(revision 1134)
@@ -0,0 +1,69 @@
+
+
+#include "ep_lib.hpp"
+
+#include <mpi.h>
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_CHAR
+#undef MPI_LONG
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+#undef MPI_COMM_WORLD
+#undef MPI_COMM_NULL
+
+#undef MPI_STATUS_IGNORE
+#undef MPI_REQUEST_NULL
+#undef MPI_INFO_NULL
+
+
+
+// _STD defined in ep_type.cpp
+
+extern ::MPI_Datatype MPI_INT_STD;
+extern ::MPI_Datatype MPI_FLOAT_STD;
+extern ::MPI_Datatype MPI_DOUBLE_STD;
+extern ::MPI_Datatype MPI_LONG_STD;
+extern ::MPI_Datatype MPI_CHAR_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_LONG_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_CHAR_STD;
+
+extern ::MPI_Op MPI_SUM_STD;
+extern ::MPI_Op MPI_MAX_STD;
+extern ::MPI_Op MPI_MIN_STD;
+
+extern ::MPI_Comm MPI_COMM_WORLD_STD;
+extern ::MPI_Comm MPI_COMM_NULL_STD;
+
+extern ::MPI_Status MPI_STATUS_IGNORE_STD;
+extern ::MPI_Request MPI_REQUEST_NULL_STD;
+extern ::MPI_Info MPI_INFO_NULL_STD;
+
+ep_lib::MPI_Datatype MPI_INT = MPI_INT_STD;
+ep_lib::MPI_Datatype MPI_FLOAT = MPI_FLOAT_STD;
+ep_lib::MPI_Datatype MPI_DOUBLE = MPI_DOUBLE_STD;
+ep_lib::MPI_Datatype MPI_CHAR = MPI_CHAR_STD;
+ep_lib::MPI_Datatype MPI_LONG = MPI_LONG_STD;
+ep_lib::MPI_Datatype MPI_UNSIGNED_LONG = MPI_UNSIGNED_LONG_STD;
+ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR = MPI_UNSIGNED_CHAR_STD;
+
+ep_lib::MPI_Op MPI_SUM = MPI_SUM_STD;
+ep_lib::MPI_Op MPI_MAX = MPI_MAX_STD;
+ep_lib::MPI_Op MPI_MIN = MPI_MIN_STD;
+
+ep_lib::MPI_Comm MPI_COMM_WORLD(MPI_COMM_WORLD_STD);
+ep_lib::MPI_Comm MPI_COMM_NULL(MPI_COMM_NULL_STD);
+
+ep_lib::MPI_Request MPI_REQUEST_NULL(MPI_REQUEST_NULL_STD);
+ep_lib::MPI_Info MPI_INFO_NULL(MPI_INFO_NULL_STD);
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_declaration.hpp	(revision 1134)
@@ -0,0 +1,64 @@
+#ifndef EP_DECLARATION_HPP_INCLUDED
+#define EP_DECLARATION_HPP_INCLUDED
+
+extern ::MPI_Datatype MPI_INT_STD;
+extern ::MPI_Datatype MPI_FLOAT_STD;
+extern ::MPI_Datatype MPI_DOUBLE_STD;
+extern ::MPI_Datatype MPI_LONG_STD;
+extern ::MPI_Datatype MPI_CHAR_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_LONG_STD;
+extern ::MPI_Datatype MPI_UNSIGNED_CHAR_STD;
+
+extern ::MPI_Op MPI_SUM_STD;
+extern ::MPI_Op MPI_MAX_STD;
+extern ::MPI_Op MPI_MIN_STD;
+
+extern ::MPI_Comm MPI_COMM_WORLD_STD;
+extern ::MPI_Comm MPI_COMM_NULL_STD;
+
+extern ::MPI_Status MPI_STATUS_IGNORE_STD;
+extern ::MPI_Request MPI_REQUEST_NULL_STD;
+extern ::MPI_Info MPI_INFO_NULL_STD;
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_CHAR
+#undef MPI_LONG
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+#undef MPI_COMM_WORLD
+#undef MPI_COMM_NULL
+
+#undef MPI_REQUEST_NULL
+
+
+
+#undef MPI_STATUS_IGNORE
+
+extern ep_lib::MPI_Datatype MPI_INT;
+extern ep_lib::MPI_Datatype MPI_FLOAT;
+extern ep_lib::MPI_Datatype MPI_DOUBLE;
+extern ep_lib::MPI_Datatype MPI_CHAR;
+extern ep_lib::MPI_Datatype MPI_LONG;
+extern ep_lib::MPI_Datatype MPI_UNSIGNED_LONG;
+extern ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR;
+
+extern ep_lib::MPI_Op MPI_SUM;
+extern ep_lib::MPI_Op MPI_MAX;
+extern ep_lib::MPI_Op MPI_MIN;
+
+extern ep_lib::MPI_Comm MPI_COMM_WORLD;
+extern ep_lib::MPI_Comm MPI_COMM_NULL;
+
+extern ep_lib::MPI_Status MPI_STATUS_IGNORE;
+extern ep_lib::MPI_Request MPI_REQUEST_NULL;
+//extern ep_lib::MPI_Info MPI_INFO_NULL;
+
+#endif // EP_DECLARATION_HPP_INCLUDED
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_dup.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_dup.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_dup.cpp	(revision 1134)
@@ -0,0 +1,124 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
+  {
+    if(!comm.is_ep)
+    {
+      Debug("Comm_dup MPI\n");
+      newcomm = new MPI_Comm;
+      newcomm->is_ep = comm.is_ep;
+
+      ::MPI_Comm input = static_cast< ::MPI_Comm>(comm.mpi_comm);
+      ::MPI_Comm output;
+
+      
+      ::MPI_Comm_dup(input, &output);
+
+      newcomm->mpi_comm = output;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm);
+
+    // for intracomm
+    if(comm.mpi_comm == MPI_COMM_NULL_STD) return 0;
+
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    if(0 == my_rank)
+    {
+      MPI_Info info;
+      MPI_Comm *out_comm;
+      ::MPI_Comm mpi_dup;
+
+      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+
+      ::MPI_Comm_dup(in_comm, &mpi_dup);
+
+      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm);
+      comm.ep_comm_ptr->comm_list->mem_bridge = out_comm;
+    }
+
+    MPI_Barrier(comm);
+
+    *newcomm = (comm.ep_comm_ptr->comm_list->mem_bridge[my_rank]);
+
+    return MPI_SUCCESS;
+  }
+
+  int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm)
+  {
+    
+    if(comm.mpi_comm == MPI_COMM_NULL_STD) return 0;
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    if(0 == my_rank)
+    {
+      MPI_Info info;
+      MPI_Comm *out_comm;
+      ::MPI_Comm mpi_dup;
+
+      ::MPI_Comm in_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+
+      ::MPI_Comm_dup(in_comm, &mpi_dup);
+
+      MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm);
+
+      ::MPI_Comm mpi_inter;
+
+      ::MPI_Comm_dup(static_cast< ::MPI_Comm>(comm.ep_comm_ptr->intercomm->mpi_inter_comm), &mpi_inter);
+      
+      for(int i=0; i<num_ep; i++)
+      {
+        out_comm[i].ep_comm_ptr->comm_label = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->comm_label;
+        out_comm[i].ep_comm_ptr->intercomm = new ep_intercomm;
+
+        out_comm[i].ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter;
+        out_comm[i].is_intercomm = true;
+
+        out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+        out_comm[i].ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+        out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+
+        out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map;
+        out_comm[i].ep_comm_ptr->intercomm->local_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map;
+        out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map;
+
+        out_comm[i].ep_comm_ptr->intercomm->local_comm = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_comm;        
+        out_comm[i].ep_comm_ptr->intercomm->intercomm_tag = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_tag;
+
+        for(int j =0; j<3; j++)
+        {
+          out_comm[i].ep_comm_ptr->intercomm->size_rank_info[j] = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->size_rank_info[j];
+        }
+
+      }
+
+      comm.ep_comm_ptr->comm_list->mem_bridge = out_comm;
+    }
+
+    MPI_Barrier(comm);
+
+    *newcomm = comm.ep_comm_ptr->comm_list->mem_bridge[my_rank];
+    
+
+    return MPI_SUCCESS;
+  }
+
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_exscan.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_exscan.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_exscan.cpp	(revision 1134)
@@ -0,0 +1,876 @@
+/*!
+   \file ep_scan.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Exscan
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+  int MPI_Exscan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Exscan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Exscan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Exscan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Exscan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Exscan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Exscan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Exscan Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+
+
+  int MPI_Exscan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Exscan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Exscan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.ep_comm_ptr->comm_list->my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          fill(recv_buf+j, recv_buf+j+min(BUFFER_SIZE, count-j), MPI_UNDEFINED);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+            }
+            else if(op == MPI_MAX)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+            }
+            else if(op == MPI_MIN)
+            {
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Exscan(const_cast<void*>(sendbuf), recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                   static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    ::MPI_Aint datasize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Exscan_local(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
+
+    if(ep_rank_loc == 0)
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] += sum_buf[i];
+            }
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          if(ep_rank_loc == 0)
+          {
+            copy(sum_buf, sum_buf+count, recv_buf);
+          }
+          else
+          {
+            for(int i=0; i<count; i++)
+            {
+              recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+            }
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+
+  }
+
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_finalize.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_finalize.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_finalize.cpp	(revision 1134)
@@ -0,0 +1,36 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib
+{
+
+  int MPI_Finalize()
+  {
+    printf("calling EP Finalize\n");
+
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Finalize();
+    }
+    return 0;
+  }
+
+  int MPI_Abort(MPI_Comm comm, int errorcode)
+  {
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+      ::MPI_Abort(mpi_comm, errorcode);
+    }
+    return 0;
+  }
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp	(revision 1134)
@@ -0,0 +1,89 @@
+#include "ep_lib.hpp"
+#include "ep_lib_fortran.hpp"
+#include <mpi.h>
+#include <map>
+#include <utility>
+#include "ep_declaration.hpp"
+
+#ifdef _openmpi
+//#undef MPI_Fint
+#endif
+
+namespace ep_lib
+{
+
+  int EP_Comm_c2f(MPI_Comm comm)
+  {
+    Debug("MPI_Comm_c2f");
+    int fint;
+    #ifdef _intelmpi
+    fint = (::MPI_Fint)(comm.mpi_comm);
+    #elif _openmpi
+    fint = ::MPI_Comm_c2f(static_cast< ::MPI_Comm>(comm.mpi_comm));
+    #endif
+    
+    std::map<std::pair<int, int>, MPI_Comm > ::iterator it;
+    
+    #pragma omp critical (fc_comm_map)
+    {
+      it = fc_comm_map.find(std::make_pair(fint, omp_get_thread_num()));
+      if(it == fc_comm_map.end())
+      {
+        fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
+        //printf("EP_Comm_c2f : MAP insert: %d, %d, %p\n", fint, omp_get_thread_num(), comm.ep_comm_ptr);
+      }
+    }
+    
+    
+    return fint;
+    
+  }
+
+  MPI_Comm EP_Comm_f2c(int comm)
+  {
+    Debug("MPI_Comm_f2c");
+    
+    
+    std::map<std::pair<int, int>, MPI_Comm > ::iterator it;
+    
+    #pragma omp critical (fc_comm_map)
+    it = fc_comm_map.find(std::make_pair(comm, omp_get_thread_num()));
+    
+    if(it != fc_comm_map.end())
+    {
+      MPI_Comm comm_ptr;
+      comm_ptr = it->second;
+      //printf("EP_Comm_f2c : MAP find: %d, %d, %p\n", it->first.first, it->first.second, comm_ptr.ep_comm_ptr);
+      return  comm_ptr;
+    }
+       
+      
+    #ifdef _openmpi
+    ::MPI_Comm base_comm = ::MPI_Comm_f2c(comm);
+    #elif _intelmpi
+    ::MPI_Comm base_comm = (::MPI_Comm)(comm);
+    #endif
+
+    if(base_comm != MPI_COMM_NULL_STD)
+    {
+      if(omp_get_thread_num() == 0)
+      {
+        int num_ep = omp_get_num_threads();
+        MPI_Comm *new_comm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(base_comm, num_ep, info, new_comm);
+        passage = new_comm;
+      }
+      #pragma omp barrier
+
+      MPI_Comm return_comm = passage[omp_get_thread_num()];
+      return return_comm;
+        
+    }
+    return MPI_COMM_NULL;
+
+  }
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_free.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_free.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_free.cpp	(revision 1134)
@@ -0,0 +1,178 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Comm_free(MPI_Comm *comm)
+  {
+
+    if(! comm->is_ep)
+    {
+      if(comm->mpi_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
+
+        ::MPI_Comm_free(&mpi_comm);
+        Debug("comm is MPI, freed\n");
+      }
+      return 0;
+    }
+
+    else if(comm->is_intercomm)
+    {
+      return MPI_Comm_free_intercomm(comm);
+    }
+
+    else
+    {
+      int ep_rank_loc, num_ep;
+
+      ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+      num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+
+      MPI_Barrier(*comm);
+
+      if(ep_rank_loc == 0)
+      {
+        Debug("comm is EP, mpi_comm_ptr != NULL\n");
+
+        if(comm->my_buffer != NULL)
+        {
+          if(comm->my_buffer->buf_int != NULL) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+          if(comm->my_buffer->buf_float != NULL) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+          if(comm->my_buffer->buf_double != NULL) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+          if(comm->my_buffer->buf_long != NULL) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+          if(comm->my_buffer->buf_ulong != NULL) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+          if(comm->my_buffer->buf_char != NULL) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+        }
+
+        if(comm->ep_barrier != NULL)
+        {
+          comm->ep_barrier->~OMPbarrier();
+          Debug("ep_barrier freed\n");
+        }
+
+
+        if( ! comm->rank_map->empty() )
+        {
+          comm->rank_map->clear();
+          Debug("rank_map emptied\n");
+        }
+
+        for(int i=0; i<num_ep; i++)
+        {
+          comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear();
+          Debug("message queue freed\n");
+          
+
+          if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr != NULL)
+          {
+            delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr;
+            Debug("ep_comm_ptr freed\n");
+          }
+        }
+
+        if(comm->mpi_comm != MPI_COMM_NULL_STD)
+        {
+          ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
+          ::MPI_Comm_free(&mpi_comm);
+          Debug("mpi_comm freed\n");
+        }
+
+       if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");}
+
+      }
+      return 0;
+    }
+
+
+
+  }
+
+  int MPI_Comm_free_intercomm(MPI_Comm *comm)
+  {
+    int ep_rank_loc, num_ep;
+
+    ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+    num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+
+    //MPI_Barrier(*comm);
+
+    if(ep_rank_loc == 0)
+    {
+      Debug("comm is EP, mpi_comm_ptr != NULL\n");
+
+      if(comm->my_buffer != NULL)
+      {
+        if(comm->my_buffer->buf_int != NULL) delete[] comm->my_buffer->buf_int; Debug("buf_int freed\n");
+        if(comm->my_buffer->buf_float != NULL) delete[] comm->my_buffer->buf_float; Debug("buf_float freed\n");
+        if(comm->my_buffer->buf_double != NULL) delete[] comm->my_buffer->buf_double; Debug("buf_double freed\n");
+        if(comm->my_buffer->buf_long != NULL) delete[] comm->my_buffer->buf_long; Debug("buf_long freed\n");
+        if(comm->my_buffer->buf_ulong != NULL) delete[] comm->my_buffer->buf_ulong; Debug("buf_ulong freed\n");
+        if(comm->my_buffer->buf_char != NULL) delete[] comm->my_buffer->buf_char; Debug("buf_char freed\n");
+      }
+
+      if(comm->ep_barrier != NULL)
+      {
+        comm->ep_barrier->~OMPbarrier();
+        Debug("ep_barrier freed\n");
+      }
+
+
+      if( ! comm->rank_map->empty() )
+      {
+        comm->rank_map->clear();
+        Debug("rank_map emptied\n");
+      }
+
+      for(int i=0; i<num_ep; i++)
+      {
+        comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear();
+        Debug("message queue freed\n");
+
+        #pragma omp critical (memory_free)
+        if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm != NULL)
+        {
+          comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->clear();
+          comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->clear();
+          comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->clear();
+          Debug("intercomm local/remote/intercomm_rank_map emptied\n");
+        }  
+
+        if(comm->ep_comm_ptr->comm_list[i].ep_comm_ptr != NULL)
+        {
+          delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr;
+          Debug("ep_comm_ptr freed\n");
+        }
+      }
+
+      if(comm->mpi_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->mpi_comm);
+        ::MPI_Comm_free(&mpi_comm);
+        Debug("mpi_comm freed\n");
+      }
+
+      // if(comm->ep_comm_ptr->intercomm->mpi_inter_comm != MPI_COMM_NULL_STD)
+      // {
+         // ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm->ep_comm_ptr->comm_list->ep_comm_ptr->intercomm->mpi_inter_comm);
+         // ::MPI_Comm_free(&mpi_comm);
+      //   Debug("mpi_intercomm freed\n");
+      // }
+
+     if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");}
+
+    }
+    
+    return 0;
+  }
+
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gather.cpp	(revision 1134)
@@ -0,0 +1,531 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Gather, MPI_Allgather
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Gather_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gather_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gather_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gather_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gather_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gather_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gather_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Gather Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gather_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gather_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gather_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      copy(send_buf, send_buf+count, recv_buf);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j+k*count);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                   root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+
+      ::MPI_Gatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count*ep_size, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+    }
+
+
+  }
+
+
+  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      local_gather_recvbuf = new void*[datasize*num_ep*count];
+    }
+
+    // local gather to master
+    MPI_Gather_local(sendbuf, count, datatype, local_gather_recvbuf, comm);
+
+    //MPI_Gather
+
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = count*num_ep;
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      ::MPI_Allgatherv(local_gather_recvbuf, count*num_ep, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+    MPI_Bcast_local(recvbuf, count*ep_size, datatype, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else// if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_gatherv.cpp	(revision 1134)
@@ -0,0 +1,567 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Gatherv, MPI_Allgatherv
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+using namespace std;
+
+namespace ep_lib
+{
+  int MPI_Gatherv_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Gatherv_local_int(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Gatherv_local_float(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Gatherv_local_double(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Gatherv_local_long(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Gatherv_local_ulong(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Gatherv_local_char(sendbuf, count, recvbuf, recvcounts, displs, comm);
+    }
+    else
+    {
+      printf("MPI_Gatherv Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Gatherv_local_int(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_float(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_long(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_ulong(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Gatherv_local_char(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    if(my_rank == 0)
+    {
+      assert(count == recvcounts[0]);
+      copy(send_buf, send_buf+count, recv_buf + displs[0]);
+    }
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      for(int k=1; k<num_ep; k++)
+      {
+        if(my_rank == k)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf + min(BUFFER_SIZE, count-j) , buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == 0)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, recvcounts[k]-j), recv_buf+j+displs[k]);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+  
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, const_cast<int*>(recvcounts), const_cast<int*>(displs),
+                    static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+    if(ep_rank != root)
+    {
+      recvcounts = new int[ep_size];
+      displs = new int[ep_size];
+    }
+    
+    MPI_Bcast(const_cast< int* >(recvcounts), ep_size, MPI_INT, root, comm);
+    MPI_Bcast(const_cast< int* >(displs), ep_size, MPI_INT, root, comm);
+
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      int buffer_size = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+    MPI_Gatherv_local(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, local_displs, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+
+      int gatherv_recvcnt[mpi_size];
+      int gatherv_displs[mpi_size];
+      int gatherv_cnt = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+
+      //gatherv_recvcnt = new int[mpi_size];
+      //gatherv_displs = new int[mpi_size];
+
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      gatherv_displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+
+      ::MPI_Gatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      //delete[] gatherv_recvcnt;
+      //delete[] gatherv_displs;
+    }
+
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, accumulate(recvcounts, recvcounts+ep_size, 0), datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+    else
+    {
+      delete[] recvcounts;
+      delete[] displs;
+    }
+    return 0;
+  }
+
+
+
+  int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allgatherv(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcounts, displs,
+                       static_cast< ::MPI_Datatype>(recvtype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+    
+
+    assert(accumulate(recvcounts, recvcounts+ep_size-1, 0) == displs[ep_size-1]); // Only for continuous gather.
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void *local_gather_recvbuf;
+
+    if(ep_rank_loc==0)
+    {
+      int buffer_size = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      local_gather_recvbuf = new void*[datasize*buffer_size];
+    }
+
+    // local gather to master
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+    MPI_Gatherv_local(sendbuf, count, datatype, local_gather_recvbuf, recvcounts+ep_rank-ep_rank_loc, local_displs, comm);
+
+    //MPI_Gather
+    if(ep_rank_loc == 0)
+    {
+      int *gatherv_recvcnt;
+      int *gatherv_displs;
+      int gatherv_cnt = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+
+      gatherv_recvcnt = new int[mpi_size];
+      gatherv_displs = new int[mpi_size];
+
+      ::MPI_Allgather(&gatherv_cnt, 1, MPI_INT_STD, gatherv_recvcnt, 1, MPI_INT_STD, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      gatherv_displs[0] = displs[0];
+      for(int i=1; i<mpi_size; i++)
+      {
+        gatherv_displs[i] = gatherv_recvcnt[i-1] + gatherv_displs[i-1];
+      }
+
+      ::MPI_Allgatherv(local_gather_recvbuf, gatherv_cnt, static_cast< ::MPI_Datatype>(datatype), recvbuf, gatherv_recvcnt,
+                    gatherv_displs, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      delete[] gatherv_recvcnt;
+      delete[] gatherv_displs;
+    }
+
+    MPI_Bcast_local(recvbuf, accumulate(recvcounts, recvcounts+ep_size, 0), datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_gather_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_gather_recvbuf);
+      }
+      else // if(datatype == MPI_CHAR)
+      {
+        delete[] static_cast<char*>(local_gather_recvbuf);
+      }
+    }
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_init.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_init.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_init.cpp	(revision 1134)
@@ -0,0 +1,49 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Init_thread(int *argc, char*** argv, int required, int*provided)
+  {
+    //printf("MPI_Init_thread\n");
+
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Init_thread(argc, argv, required, provided);
+    }
+    return 0;
+  }
+
+  int MPI_Init(int *argc, char ***argv)
+  {
+    //printf("MPI_init called\n");
+    int id = omp_get_thread_num();
+
+    if(id == 0)
+    {
+      ::MPI_Init(argc, argv);
+    }
+  	return 0;
+  }
+
+  int MPI_Initialized(int *flag)
+  {
+    //printf("MPI_initialized called\n");
+
+    ::MPI_Initialized(flag);
+    
+    return 0;
+  }
+
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp	(revision 1134)
@@ -0,0 +1,206 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    assert(local_comm.is_ep);
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Barrier(local_comm);
+
+
+
+    int leader_ranks[6]; //! 0: rank in world, 1: mpi_size, 2: rank_in_peer.
+                         //! 3, 4, 5 : remote
+
+    bool is_decider = false;
+
+
+
+    if(ep_rank == local_leader)
+    {
+      ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &leader_ranks[0]);
+
+      leader_ranks[1] = mpi_size;
+      MPI_Comm_rank(peer_comm, &leader_ranks[2]);
+
+      MPI_Request req_s, req_r;
+      MPI_Isend(&leader_ranks[0], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Status status;
+      MPI_Irecv(&leader_ranks[3], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+    MPI_Bcast(leader_ranks, 6, MPI_INT_STD, local_leader, local_comm);
+
+    MPI_Barrier(local_comm);
+
+    if(leader_ranks[0] == leader_ranks[3])
+    {
+      if( leader_ranks[1] * leader_ranks[4] == 1)
+      {
+        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_unique_leader\n");
+        local_comm.ep_comm_ptr->comm_label = -99;
+
+        return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+      }
+      else // leader_ranks[1] * leader_ranks[4] != 1
+      {
+        // change leader
+        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_from_world\n");
+
+        int new_local_leader;
+
+        if(leader_ranks[2] < leader_ranks[5])
+        {
+          if(leader_ranks[1] > 1) //! change leader
+          {
+            // change leader
+            is_decider = true;
+            int target = local_comm.rank_map->at(local_leader).second;
+            {
+              for(int i=0; i<ep_size; i++)
+              {
+                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                {
+                  new_local_leader = i;
+                  break;
+                }
+              }
+            }
+          }
+          else
+          {
+            new_local_leader = local_leader;
+          }
+        }
+        else
+        {
+          if(leader_ranks[4] == 1)
+          {
+            // change leader
+            is_decider = true;
+            int target = local_comm.rank_map->at(local_leader).second;
+            {
+              for(int i=0; i<ep_size; i++)
+              {
+                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                {
+                  new_local_leader = i;
+                  break;
+                }
+              }
+            }
+          }
+          else
+          {
+            new_local_leader = local_leader;
+          }
+        }
+
+
+        int new_tag_in_world;
+
+        int leader_in_world[2];
+
+
+        if(is_decider)
+        {
+          if(ep_rank == new_local_leader)
+          {
+            new_tag_in_world = TAG++;
+          }
+          MPI_Bcast(&new_tag_in_world, 1, MPI_INT_STD, new_local_leader, local_comm);
+          if(ep_rank == local_leader) MPI_Send(&new_tag_in_world, 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+        }
+        else
+        {
+          if(ep_rank == local_leader)
+          {
+            MPI_Status status;
+            MPI_Recv(&new_tag_in_world, 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+          }
+          MPI_Bcast(&new_tag_in_world, 1, MPI_INT_STD, new_local_leader, local_comm);
+        }
+
+
+        if(ep_rank == new_local_leader)
+        {
+          ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &leader_in_world[0]);
+        }
+
+        MPI_Bcast(&leader_in_world[0], 1, MPI_INT_STD, new_local_leader, local_comm);
+
+
+        if(ep_rank == local_leader)
+        {
+          MPI_Request req_s, req_r;
+
+          MPI_Isend(&leader_in_world[0], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+          MPI_Irecv(&leader_in_world[1], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+          MPI_Status status;
+          MPI_Wait(&req_s, &status);
+          MPI_Wait(&req_r, &status);
+
+      /*
+          MPI_Send(&leader_in_world[0], 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+          MPI_Status status;
+          MPI_Recv(&leader_in_world[1], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+   */
+        }
+
+
+
+        MPI_Bcast(&leader_in_world[1], 1, MPI_INT, local_leader, local_comm);
+
+
+
+        local_comm.ep_comm_ptr->comm_label = tag;
+
+        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD_STD, leader_in_world[1], new_tag_in_world, newintercomm);
+      }
+    }
+
+    if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_kernel\n");
+
+    return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+
+  }
+
+  int MPI_Comm_test_inter(MPI_Comm comm, int *flag)
+  {
+    *flag = false;
+    if(comm.is_ep)
+    {
+      *flag = comm.is_intercomm;
+      return 0;
+    } 
+    else if(comm.mpi_comm != MPI_COMM_NULL_STD)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      
+      ::MPI_Comm_test_inter(mpi_comm, flag);
+      return 0;  
+    }
+    return 0;
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_kernel.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_kernel.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_kernel.cpp	(revision 1134)
@@ -0,0 +1,763 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib
+{
+  int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int rank_in_peer_mpi[2];
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    
+    ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+    
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = remote_leader;
+
+        ::MPI_Comm_rank(static_cast< ::MPI_Comm>(peer_comm.mpi_comm), &rank_in_peer_mpi[0]);
+
+        MPI_Status status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+        send_buf[2] = rank_in_peer_mpi[0];
+
+        MPI_Send(send_buf.data(), 3, MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+        size_info[1] = recv_buf[0];
+        remote_ep_size = recv_buf[1];
+        rank_in_peer_mpi[1] = recv_buf[2];
+
+      }
+
+      send_buf[0] = size_info[1];
+      send_buf[1] = leader_info[0];
+      send_buf[2] = leader_info[1];
+      send_buf[3] = rank_in_peer_mpi[0];
+      send_buf[4] = rank_in_peer_mpi[1];
+
+      ::MPI_Bcast(send_buf.data(), 5, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = send_buf[0];
+      leader_info[0] = send_buf[1];
+      leader_info[1] = send_buf[2];
+      rank_in_peer_mpi[0] = send_buf[3];
+      rank_in_peer_mpi[1] = send_buf[4];
+
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        MPI_Status status;
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      //! size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        MPI_Status status;
+        MPI_Send(&size_info[2], 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(&size_info[3], 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+      }
+
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        MPI_Status status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, remote_leader, tag, peer_comm);
+        MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+      }
+
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      //! leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+
+        ::MPI_Barrier(new_comm);
+
+        ::MPI_Intercomm_create(new_comm, leader_info[2], static_cast< ::MPI_Comm>(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, &intercomm);
+
+        int id;
+
+        ::MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+        //printf("tag_list size = %lu\n", tag_list.size());
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+    if(is_proc_master)
+    {
+      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+      {
+        if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+        {
+          tag_list.erase(iter);
+          break;
+        }
+      }
+    }
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+    int new_bcast_root_0 = 0;
+    int new_bcast_root = 0;
+
+
+    if(is_local_leader)
+    {
+      MPI_Status status;
+      MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+      MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+
+      new_bcast_root_0 = intercomm_ep_rank;
+    }
+
+    MPI_Allreduce(&new_bcast_root_0, &new_bcast_root, 1, MPI_INT_STD, MPI_SUM_STD, *newintercomm);
+
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, new_bcast_root, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+
+
+
+    if(is_local_leader)
+    {
+      MPI_Status status;
+      MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, remote_leader, tag, peer_comm);
+      MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, new_bcast_root, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
+
+/*
+    for(int i=0; i<local_ep_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, local_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
+
+    for(int i=0; i<remote_ep_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).second);
+
+    for(int i=0; i<remote_intercomm_size; i++)
+    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+          (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
+*/
+
+//    for(int i=0; i<(*newintercomm).rank_map->size(); i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second);
+
+//    MPI_Comm *test_comm = newintercomm->ep_comm_ptr->intercomm->local_comm;
+//    int test_rank;
+//    MPI_Comm_rank(*test_comm, &test_rank);
+//    printf("=================test_rank = %d\n", test_rank);
+
+    return MPI_SUCCESS;
+
+  }
+
+
+
+
+  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    //! mpi_size of local comm = 1
+    //! same world rank of leaders
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+
+    int rank_in_peer_mpi[2];
+
+    ::MPI_Comm_rank(MPI_COMM_WORLD_STD, &rank_in_world);
+
+
+    int local_num_ep = num_ep;
+    int remote_num_ep;
+    int total_num_ep;
+
+    int leader_rank_in_peer[2];
+
+    int my_position;
+    int tag_label[2];
+
+    vector<int> send_buf(4);
+    vector<int> recv_buf(4);
+
+
+    if(ep_rank == local_leader)
+    {
+      MPI_Status status;
+
+
+
+      MPI_Comm_rank(peer_comm, &leader_rank_in_peer[0]);
+
+      send_buf[0] = local_num_ep;
+      send_buf[1] = leader_rank_in_peer[0];
+
+      MPI_Request req_s, req_r;
+
+      MPI_Isend(send_buf.data(), 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Irecv(recv_buf.data(), 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+
+      recv_buf[2] = leader_rank_in_peer[0];
+
+    }
+
+    MPI_Bcast(recv_buf.data(), 3, MPI_INT_STD, local_leader, local_comm);
+
+    remote_num_ep = recv_buf[0];
+    leader_rank_in_peer[1] = recv_buf[1];
+    leader_rank_in_peer[0] = recv_buf[2];
+
+    total_num_ep = local_num_ep + remote_num_ep;
+
+
+    if(leader_rank_in_peer[0] < leader_rank_in_peer[1])
+    {
+      my_position = ep_rank_loc;
+      //! LEADER create EP
+      if(ep_rank == local_leader)
+      {
+        ::MPI_Comm mpi_dup;
+        
+        ::MPI_Comm_dup(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &mpi_dup);
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(mpi_dup, total_num_ep, info, ep_intercomm);
+
+
+        for(int i=0; i<total_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = 0;
+
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_rank_in_peer[0];
+        }
+
+        tag_label[0] = TAG++;
+        tag_label[1] = rank_in_world;
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag_label[0], tag_label[1]) , ep_intercomm));
+
+        MPI_Request req_s;
+        MPI_Status sta_s;
+        MPI_Isend(tag_label, 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+
+        MPI_Wait(&req_s, &sta_s);
+
+      }
+    }
+    else
+    {
+      //! Wait for EP creation
+      my_position = remote_num_ep + ep_rank_loc;
+      if(ep_rank == local_leader)
+      {
+        MPI_Status status;
+        MPI_Request req_r;
+        MPI_Irecv(tag_label, 2, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+        MPI_Wait(&req_r, &status);
+      }
+    }
+
+    MPI_Bcast(tag_label, 2, MPI_INT_STD, local_leader, local_comm);
+
+
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag_label[0], tag_label[1]))
+          {
+            *newintercomm =  iter->second[my_position];
+            found = true;
+            // tag_list.erase(iter);
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    if(leader_rank_in_peer[0] < leader_rank_in_peer[1])
+    {
+      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag_label[0], tag_label[1]))
+          {
+            tag_list.erase(iter);
+            break;
+          }
+        }
+    }
+
+
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map  = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_num_ep);
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_num_ep);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+
+    int local_rank_map_ele[2];
+    local_rank_map_ele[0] = intercomm_ep_rank;
+    local_rank_map_ele[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    MPI_Allgather(local_rank_map_ele, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    if(ep_rank == local_leader)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+
+      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
+      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
+
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT_STD, local_leader, local_comm);
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
+
+
+    return MPI_SUCCESS;
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_world.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_world.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_world.cpp	(revision 1134)
@@ -0,0 +1,1057 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  #ifdef _openmpi
+
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+    ::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    ::MPI_Comm_rank(peer_comm, &rank_in_world);
+
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = mpi_remote_leader;
+
+        ::MPI_Status mpi_status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+
+        ::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+        recv_buf[2] = rank_in_world;
+        recv_buf[3] = mpi_remote_leader;
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = recv_buf[0];
+      remote_ep_size = recv_buf[1];
+      leader_info[0] = recv_buf[2];
+      leader_info[1] = recv_buf[3];
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        ::MPI_Status mpi_status;
+
+        ::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+        ::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      // leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Barrier(new_comm);
+
+        ::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
+
+        int id;
+        MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+
+            found = true;
+            //tag_list.erase(iter);
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    if(is_proc_master)
+    {
+      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+      {
+        if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+        {
+          tag_list.erase(iter);
+          break;
+        }
+      }
+    }
+
+    
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+
+      ::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
+
+
+    return MPI_SUCCESS;
+
+  }
+
+
+
+  #elif _intelmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
+                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+
+    int rank_in_world;
+    int rank_in_local_parent;
+
+    int local_ep_size = ep_size;
+    int remote_ep_size;
+
+    ::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
+    ::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
+
+    ::MPI_Comm_rank(peer_comm, &rank_in_world);
+
+    ::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
+
+    bool is_proc_master = false;
+    bool is_local_leader = false;
+    bool is_final_master = false;
+
+
+    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
+    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
+
+
+    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
+
+    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
+
+
+    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
+
+    std::vector<int> new_rank_info[4];
+    std::vector<int> new_ep_info[2];
+
+    std::vector<int> offset;
+
+    if(is_proc_master)
+    {
+
+      size_info[0] = mpi_size;
+
+      rank_info[0].resize(size_info[0]);
+      rank_info[1].resize(size_info[0]);
+
+      ep_info[0].resize(size_info[0]);
+
+      vector<int> send_buf(6);
+      vector<int> recv_buf(3*size_info[0]);
+
+      send_buf[0] = rank_in_world;
+      send_buf[1] = rank_in_local_parent;
+      send_buf[2] = num_ep;
+
+      ::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
+
+      for(int i=0; i<size_info[0]; i++)
+      {
+        rank_info[0][i] = recv_buf[3*i];
+        rank_info[1][i] = recv_buf[3*i+1];
+        ep_info[0][i]   = recv_buf[3*i+2];
+      }
+
+
+      if(is_local_leader)
+      {
+        leader_info[0] = rank_in_world;
+        leader_info[1] = mpi_remote_leader;
+
+        ::MPI_Status mpi_status;
+
+        send_buf[0] = size_info[0];
+        send_buf[1] = local_ep_size;
+
+        ::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+        recv_buf[2] = rank_in_world;
+        recv_buf[3] = mpi_remote_leader;
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      size_info[1] = recv_buf[0];
+      remote_ep_size = recv_buf[1];
+      leader_info[0] = recv_buf[2];
+      leader_info[1] = recv_buf[3];
+
+      rank_info[2].resize(size_info[1]);
+      rank_info[3].resize(size_info[1]);
+
+      ep_info[1].resize(size_info[1]);
+
+      send_buf.resize(3*size_info[0]);
+      recv_buf.resize(3*size_info[1]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+
+        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
+        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
+        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+
+      }
+
+      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
+
+      offset.resize(size_info[0]);
+
+      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+
+            if(target == leader_info[0]) // the leader is doubled in remote
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else
+            {
+              offset[i] = ep_local;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+
+            offset[i] = 0;
+          }
+
+        }
+      }
+
+      else // erase rank doubled with remote leader
+      {
+
+        bool found = false;
+        int ep_tmp;
+        int ep_local;
+        int ep_remote;
+        for(int i=0; i<size_info[0]; i++)
+        {
+          int target = rank_info[0][i];
+          found = false;
+          for(int j=0; j<size_info[1]; j++)
+          {
+
+            if(target == rank_info[2][j])
+            {
+              found = true;
+              ep_tmp = ep_info[1][j];
+              ep_local = ep_info[0][j];
+              ep_remote = ep_info[1][j];
+              break;
+            }
+          }
+          if(found)
+          {
+            if(target != leader_info[1])
+            {
+              new_rank_info[0].push_back(target);
+              new_rank_info[1].push_back(rank_info[1][i]);
+
+              new_ep_info[0].push_back(ep_local + ep_remote);
+              offset[i] = 0;
+            }
+            else // found remote leader
+            {
+              offset[i] = ep_remote;
+            }
+          }
+          else
+          {
+            new_rank_info[0].push_back(target);
+            new_rank_info[1].push_back(rank_info[1][i]);
+
+            new_ep_info[0].push_back(ep_info[0][i]);
+            offset[i] = 0;
+          }
+        }
+      }
+
+      if(offset[mpi_rank] == 0)
+      {
+        is_final_master = true;
+      }
+
+
+      // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
+
+      if(is_local_leader)
+      {
+        size_info[2] = new_ep_info[0].size();
+        ::MPI_Status mpi_status;
+
+        ::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      new_rank_info[2].resize(size_info[3]);
+      new_rank_info[3].resize(size_info[3]);
+      new_ep_info[1].resize(size_info[3]);
+
+      send_buf.resize(size_info[2]);
+      recv_buf.resize(size_info[3]);
+
+      if(is_local_leader)
+      {
+        ::MPI_Status mpi_status;
+
+        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
+        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
+        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
+
+        ::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+        ::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
+      }
+
+      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
+      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
+      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
+
+    }
+
+
+
+    if(is_proc_master)
+    {
+      // leader_info[4]: 2-> rank of local leader in new_group generated comm;
+                      // 3-> rank of remote leader in new_group generated comm;
+      ::MPI_Group local_group;
+      ::MPI_Group new_group;
+      ::MPI_Comm new_comm;
+      ::MPI_Comm intercomm;
+
+      ::MPI_Comm_group(local_mpi_comm, &local_group);
+
+      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
+
+      ::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
+
+
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(new_comm, &leader_info[2]);
+      }
+
+      ::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
+
+      if(new_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Barrier(new_comm);
+
+        ::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
+
+        int id;
+        ::MPI_Comm_rank(new_comm, &id);
+        int my_num_ep = new_ep_info[0][id];
+
+        MPI_Comm *ep_intercomm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
+
+        for(int i= 0; i<my_num_ep; i++)
+        {
+          ep_intercomm[i].is_intercomm = true;
+
+          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
+          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
+        }
+
+
+        #pragma omp critical (write_to_tag_list)
+        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
+
+      }
+
+
+    }
+
+
+    MPI_Barrier_local(local_comm);
+
+
+    vector<int> bcast_buf(8);
+    if(is_local_leader)
+    {
+      std::copy(size_info, size_info+4, bcast_buf.begin());
+      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
+    }
+
+
+
+    MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
+
+
+    if(!is_local_leader)
+    {
+      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
+      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
+    }
+
+
+
+    if(!is_local_leader)
+    {
+      new_rank_info[1].resize(size_info[2]);
+      ep_info[1].resize(size_info[1]);
+      offset.resize(size_info[0]);
+    }
+
+    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
+
+    if(is_local_leader)
+    {
+      bcast_buf[0] = remote_ep_size;
+      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
+      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
+      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
+    }
+
+    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
+
+
+    if(!is_local_leader)
+    {
+      remote_ep_size = bcast_buf[0];
+      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
+      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
+    }
+
+
+    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
+
+
+    MPI_Barrier_local(local_comm);
+    #pragma omp flush
+
+
+    #pragma omp critical (read_from_tag_list)
+    {
+      bool found = false;
+      while(!found)
+      {
+        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+        {
+          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+          {
+            *newintercomm =  iter->second[my_position];
+
+            found = true;
+            //tag_list.erase(iter);
+            break;
+          }
+        }
+      }
+    }
+
+    MPI_Barrier_local(local_comm);
+
+    if(is_proc_master)
+    {
+      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
+      {
+        if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
+        {
+          tag_list.erase(iter);
+          break;
+        }
+      }
+    }
+
+    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
+    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
+
+    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
+    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
+    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
+    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
+    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
+    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
+
+    int my_rank_map_elem[2];
+
+
+    my_rank_map_elem[0] = intercomm_ep_rank;
+
+    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
+
+    vector<pair<int, int> > local_rank_map_array;
+    vector<pair<int, int> > remote_rank_map_array;
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
+
+    MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
+
+    int local_intercomm_size = intercomm_ep_size;
+    int remote_intercomm_size;
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+
+      ::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT_STD, local_leader, local_comm);
+    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
+
+
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
+    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
+
+
+    if(is_local_leader)
+    {
+      ::MPI_Status status;
+
+      ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
+
+      ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
+    }
+
+    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
+
+    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
+    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
+
+
+//    for(int i=0; i<local_ep_size; i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
+
+//    for(int i=0; i<remote_intercomm_size; i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
+
+//    for(int i=0; i<(*newintercomm).rank_map->size(); i++)
+//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
+//          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second);
+
+
+
+
+
+    return MPI_SUCCESS;
+
+  }
+
+  #endif
+
+}
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp	(revision 1134)
@@ -0,0 +1,376 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+
+namespace ep_lib
+{
+
+
+  int tag_combine(int real_tag, int src, int dest)
+  {
+    int a = real_tag << 16;
+    int b = src << 8;
+    int c = dest;
+
+    return a+b+c;
+  }
+
+  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
+  {
+    for(int i=0; i<comm.rank_map->size(); i++)
+    {
+      if(   ( comm.rank_map->at(i).first  == ep_rank_loc )
+         && ( comm.rank_map->at(i).second == mpi_rank ) )
+      {
+        return i;
+      }
+    }
+    printf("rank not find\n");
+  }
+  
+  int get_ep_rank_intercomm(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
+  {
+    // intercomm
+    int inter_rank;
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->intercomm_rank_map->size(); i++)
+    {
+      if(   ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first  == ep_rank_loc )
+         && ( comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second == mpi_rank ) )
+      {
+        inter_rank =  i;
+        break;
+      }
+    }
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->remote_rank_map->size(); i++)
+    {
+      if(  comm.ep_comm_ptr->intercomm->remote_rank_map->at(i).first  == inter_rank  )
+      {
+        //printf("get_ep_rank for intercomm, ep_rank_loc = %d, mpi_rank = %d => ep_src = %d\n", ep_rank_loc, mpi_rank, i);
+        return i;
+      }
+    }
+
+    printf("rank not find\n");
+    
+  }
+
+
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    if(datatype == MPI_INT)
+    {
+
+      int* send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+      int* recv_buf = static_cast<int*>(recvbuf);
+      int* buffer = comm.my_buffer->buf_int;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+
+      float* send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+      float* recv_buf = static_cast<float*>(recvbuf);
+      float* buffer = comm.my_buffer->buf_float;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+
+
+      double* send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+      double* recv_buf = static_cast<double*>(recvbuf);
+      double* buffer = comm.my_buffer->buf_double;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_LONG)
+    {
+      long* send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+      long* recv_buf = static_cast<long*>(recvbuf);
+      long* buffer = comm.my_buffer->buf_long;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+      unsigned long* buffer = comm.my_buffer->buf_ulong;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      char* send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+      char* recv_buf = static_cast<char*>(recvbuf);
+      char* buffer = comm.my_buffer->buf_char;
+
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(ep_rank_loc == sender)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          }
+          #pragma omp flush
+        }
+
+        MPI_Barrier_local(comm);
+
+
+        if(ep_rank_loc == receiver)
+        {
+          #pragma omp flush
+          #pragma omp critical (read_from_buffer)
+          {
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+    else
+    {
+      printf("datatype not supported!!\n");
+      exit(1);
+    }
+    return 0;
+  }
+
+
+  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count)
+  {
+/*
+    ::MPI_Aint datasize, char_size, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR, &lb, &char_size);
+
+    *count = status->char_count / ( datasize/ char_size);
+
+    //printf("MPI_Get_count, status_count  = %d\n", *count);
+    return 0;
+*/
+    ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
+    ::MPI_Datatype mpi_datatype = static_cast< ::MPI_Datatype >(datatype);
+
+    ::MPI_Get_count(mpi_status, mpi_datatype, count);
+  }
+
+  double MPI_Wtime()
+  {
+    return ::MPI_Wtime();
+
+  }
+
+  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type)
+  {
+    int src_rank;
+    int int_count;
+    ::MPI_Aint datasize, intsize, charsize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR_STD, &lb, &intsize);
+
+    int_count = count * datasize / intsize ;
+
+    char *buffer = static_cast< char* >(const_cast< void*> (buf));
+    
+    unsigned long sum = 0;
+    for(int i = 0; i<int_count; i++)
+      sum += *(buffer+i); 
+
+
+    MPI_Comm_rank(comm, &src_rank);
+    
+    ofstream myfile;
+    myfile.open ("send_log.txt", ios::app);
+    if (myfile.is_open())
+    {
+      myfile << "type = " << type << " src = "<< src_rank<< " dest = "<< dest <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
+      myfile.close();  
+    }
+    else printf("Unable to open file\n");
+
+  }
+
+
+  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type)
+  {
+    int dest_rank;
+    int int_count;
+    ::MPI_Aint datasize, intsize, charsize, lb;
+    
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+    ::MPI_Type_get_extent(MPI_CHAR_STD, &lb, &intsize);
+
+    int_count = count * datasize / intsize ;
+
+    char *buffer = static_cast< char* >(buf);
+    
+    unsigned long sum = 0;
+    for(int i = 0; i<int_count; i++)
+      sum += *(buffer+i); 
+
+
+    MPI_Comm_rank(comm, &dest_rank);
+    
+    ofstream myfile;
+    myfile.open ("recv_log.txt", ios::app);
+    if (myfile.is_open())
+    {
+      myfile << "type = " << type << " src = "<< src << " dest = "<< dest_rank <<" tag = "<< tag << "  count = "<< count << " sum = "<< sum << "\n";
+      myfile.close();  
+    }
+    else printf("Unable to open file\n");
+
+  }
+}
+
+
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp	(revision 1134)
@@ -0,0 +1,92 @@
+#ifndef EP_LIB_HPP_INCLUDED
+#define EP_LIB_HPP_INCLUDED
+
+#include "ep_type.hpp"
+#include "ep_lib_intercomm.hpp"
+#include "ep_lib_local.hpp"
+#include "ep_lib_collective.hpp"
+#include "ep_tag.hpp"
+#include "ep_lib_fortran.hpp"
+
+
+namespace ep_lib
+{
+#ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #define EP_ANY_SOURCE -2 
+  #define EP_ANY_TAG -1 
+#elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #define EP_ANY_SOURCE -1 
+  #define EP_ANY_TAG -1 
+#endif
+
+  int MPI_Init_thread(int* argc, char*** argv, int required, int*provided);
+
+  int MPI_Init(int *argc, char ***argv);
+
+  int MPI_Initialized(int *flag);
+
+  int MPI_Comm_rank(MPI_Comm comm, int* rank);
+
+  int MPI_Comm_size(MPI_Comm comm, int* rank);
+
+  int MPI_Comm_free(MPI_Comm* comm);
+
+  int MPI_Finalize();
+  
+  double MPI_Wtime();
+
+  int MPI_Abort(MPI_Comm comm, int errorcode);
+
+  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count);
+
+  #ifdef _openmpi
+  int MPI_Comm_create_endpoints(void* mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints
+  #elif _intelmpi
+  int MPI_Comm_create_endpoints(int mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints
+  #endif
+  
+  int MPI_Comm_create_endpoints(MPI_Comm base_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints
+
+  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);
+  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);
+  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+
+  int tag_combine(int real_tag, int src, int dest);
+  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank);
+  int get_ep_rank_intercomm(MPI_Comm comm, int ep_rank_loc, int mpi_rank);
+
+  int Message_Check(MPI_Comm comm);
+
+  int MPI_Recv  (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status);
+  int MPI_Irecv (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Mrecv (void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status);
+  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request);
+
+
+  int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status);
+  int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses);
+
+  int MPI_Wait(MPI_Request *request, MPI_Status *status);
+  int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses);
+
+
+  int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Improbe(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+  
+  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr);
+  
+  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr);
+
+
+  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type);
+  void check_sum_recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, int type);
+}
+
+
+
+#endif // EP_LIB_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_collective.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_collective.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_collective.hpp	(revision 1134)
@@ -0,0 +1,44 @@
+#ifndef EP_LIB_COLLECTIVE_HPP_INCLUDED
+#define EP_LIB_COLLECTIVE_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+
+  int MPI_Barrier(MPI_Comm comm);
+
+  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
+
+  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm);
+  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm);
+
+  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+
+  int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[],
+                  MPI_Datatype recvtype, MPI_Comm comm);
+
+  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm);
+  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm);
+
+  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm);
+
+}
+
+#endif // EP_LIB_COLLECTIVE_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_fortran.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_fortran.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_fortran.hpp	(revision 1134)
@@ -0,0 +1,26 @@
+#ifndef EP_LIB_FORTRAN_HPP_INCLUDED
+#define EP_LIB_FORTRAN_HPP_INCLUDED
+
+#include "ep_type.hpp"
+
+namespace ep_lib
+{
+  
+  // #ifdef _intelmpi
+  
+  // MPI_Fint MPI_Comm_c2f(MPI_Comm comm);
+  // MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  // #elif _openmpi
+  
+  // int MPI_Comm_c2f(MPI_Comm comm);
+  // ep_lib::MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
+  
+  // #endif
+
+  int EP_Comm_c2f(MPI_Comm comm);
+  MPI_Comm EP_Comm_f2c(int comm);
+}
+
+
+#endif // EP_LIB_FORTRAN_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_intercomm.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_intercomm.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_intercomm.hpp	(revision 1134)
@@ -0,0 +1,55 @@
+#ifndef EP_LIB_INTERCOMM_HPP_INCLUDED
+#define EP_LIB_INTERCOMM_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+  
+  int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm);
+
+  int MPI_Comm_free_intercomm(MPI_Comm* comm);
+
+  int MPI_Barrier_intercomm(MPI_Comm comm);
+
+  int Message_Check_intercomm(MPI_Comm comm);
+
+  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);
+
+
+  int MPI_Iprobe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Iprobe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  int MPI_Improbe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+  int MPI_Improbe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+
+  int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm);
+
+  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+
+  int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+  
+  #ifdef _intelmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm);
+  #elif _openmpi
+  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm);
+  #endif
+  
+  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+
+  int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm);
+
+  int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm);
+
+  int MPI_Comm_remote_size(MPI_Comm comm, int *size);
+
+  int MPI_Comm_test_inter(MPI_Comm comm, int *flag);
+}
+
+
+#endif // EP_LIB_INTERCOMM_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_local.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_local.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib_local.hpp	(revision 1134)
@@ -0,0 +1,88 @@
+#ifndef EP_LIB_LOCAL_HPP_INCLUDED
+#define EP_LIB_LOCAL_HPP_INCLUDED
+
+namespace ep_lib
+{
+  #ifdef _intelmpi
+  typedef int MPI_Datatype;
+  typedef int MPI_Op;
+  #elif _openmpi
+  typedef void* MPI_Datatype;
+  typedef void* MPI_Op;
+  #endif
+
+
+  int MPI_Reduce_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Reduce_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+
+  int MPI_Scan_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Scan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Exscan_local       (const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_int   (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_float (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_long  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_ulong (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+  int MPI_Exscan_local_char  (const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm);
+
+  int MPI_Bcast_local       (void *buffer, int count, MPI_Datatype datatype, MPI_Comm comm);
+  int MPI_Bcast_local_int   (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_float (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_double(void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_long  (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_ulong (void *buffer, int count, MPI_Comm comm);
+  int MPI_Bcast_local_char  (void *buffer, int count, MPI_Comm comm);
+
+  int MPI_Gather_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Gather_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+
+  int MPI_Gatherv_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf,
+                               const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_int   (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_float (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_double(const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_long  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_ulong (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+  int MPI_Gatherv_local_char  (const void *sendbuf, int count, void *recvbuf, const int recvcounts[], const int displs[], MPI_Comm comm);
+
+  int MPI_Scatter_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_int   (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_float (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_long  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_ulong (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatter_local_char  (const void *sendbuf, int count, void *recvbuf, MPI_Comm comm);
+
+  int MPI_Scatterv_local       (const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_int   (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_float (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_double(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_long  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_ulong (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+  int MPI_Scatterv_local_char  (const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm);
+
+  int innode_memcpy(int sender, const void* sendbuf, int receiver, void* recvbuf, int count, MPI_Datatype datatype, MPI_Comm comm);
+
+  int MPI_Barrier_local(MPI_Comm comm);
+
+}
+
+#endif // EP_LIB_LOCAL_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_memory.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_memory.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_memory.cpp	(revision 1134)
@@ -0,0 +1,26 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr)
+  {
+    ::MPI_Alloc_mem(size.mpi_aint, static_cast< ::MPI_Info>(info.mpi_info), baseptr);
+    return 0;
+   }
+   
+  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr)
+  {
+    ::MPI_Alloc_mem(size, static_cast< ::MPI_Info>(info.mpi_info), baseptr);
+    return 0;
+  }
+
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_merge.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_merge.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_merge.cpp	(revision 1134)
@@ -0,0 +1,286 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+  int MPI_Intercomm_merge_unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
+  {
+    Debug("intercomm_merge with unique leader\n");
+
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+    int local_high = high;
+    int remote_high;
+
+    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+//    printf("%d, %d, %d, %d\n", local_ep_size, remote_ep_size, local_high, remote_high);
+
+
+    MPI_Comm_dup(inter_comm, newintracomm);
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
+
+    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+    }
+
+    return MPI_SUCCESS;
+  }
+
+
+
+
+
+  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
+  {
+
+    assert(inter_comm.is_intercomm);
+
+    if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99)
+    {
+        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm);
+    }
+
+
+    Debug("intercomm_merge kernel\n");
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    int local_ep_rank, local_ep_rank_loc, local_mpi_rank;
+    int local_ep_size, local_num_ep, local_mpi_size;
+
+
+    local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first;
+    local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
+    local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first;
+    local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second;
+    local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
+    local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+    int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+
+    int local_high = high;
+    int remote_high;
+
+    MPI_Barrier(inter_comm);
+
+//    if(local_ep_rank == 0 && high == false)
+//    {
+//      MPI_Status status;
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//    }
+//
+//    if(local_ep_rank == 0 && high == true)
+//    {
+//      MPI_Status status;
+//      MPI_Recv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &status);
+//      MPI_Send(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm);
+//    }
+
+    if(local_ep_rank == 0)
+    {
+      MPI_Status status;
+      MPI_Request req_s, req_r;
+      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_s);
+      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &req_r);
+
+      MPI_Wait(&req_s, &status);
+      MPI_Wait(&req_r, &status);
+    }
+
+    MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm));
+
+    int intercomm_high;
+    if(ep_rank == 0) intercomm_high = local_high;
+    MPI_Bcast(&intercomm_high, 1, MPI_INT, 0, inter_comm);
+
+    //printf("remote_ep_size = %d, local_high = %d, remote_high = %d, intercomm_high = %d\n", remote_ep_size, local_high, remote_high, intercomm_high);
+
+
+    ::MPI_Comm mpi_intracomm;
+    MPI_Comm *ep_intracomm;
+
+    if(ep_rank_loc == 0)
+    {
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+
+      ::MPI_Intercomm_merge(mpi_comm, intercomm_high, &mpi_intracomm);
+      MPI_Info info;
+      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
+
+      inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm;
+
+    }
+
+
+
+    MPI_Barrier_local(inter_comm);
+
+    *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[ep_rank_loc];
+
+    int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size;
+
+    int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank;
+    int intra_ep_size, intra_num_ep, intra_mpi_size;
+
+    intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first;
+    intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first;
+    intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first;
+    intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second;
+    intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second;
+    intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second;
+
+
+
+    MPI_Barrier_local(*newintracomm);
+
+
+    int *reorder;
+    if(intra_ep_rank_loc == 0)
+    {
+      reorder = new int[intra_ep_size];
+    }
+
+
+
+    MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm);
+    if(intra_ep_rank_loc == 0)
+    {
+
+      ::MPI_Bcast(reorder, intra_ep_size, MPI_INT_STD, 0, static_cast< ::MPI_Comm>(newintracomm->mpi_comm));
+
+      vector< pair<int, int> > tmp_rank_map(intra_ep_size);
+
+
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ;
+      }
+
+      newintracomm->rank_map->swap(tmp_rank_map);
+
+      tmp_rank_map.clear();
+    }
+
+    MPI_Barrier_local(*newintracomm);
+
+    (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
+
+
+    if(intra_ep_rank_loc == 0)
+    {
+      delete[] reorder;
+
+    }
+
+    /*
+    if(intra_ep_rank == 0)
+    {
+      for(int i=0; i<intra_ep_size; i++)
+      {
+        printf("intra rank_map[%d] = (%d, %d)\n", i, newintracomm->rank_map->at(i).first, newintracomm->rank_map->at(i).second);
+      }
+    }
+*/
+    return MPI_SUCCESS;
+
+  }
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp	(revision 1134)
@@ -0,0 +1,225 @@
+/*!
+   \file ep_message.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI endpoint function: Message_Check
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int Message_Check(MPI_Comm comm)
+  {
+    int myRank;
+    MPI_Comm_rank(comm, &myRank);
+
+    if(!comm.is_ep) return 0;
+
+    if(comm.is_intercomm)
+    {
+      return  Message_Check_intercomm(comm);
+    }
+
+    int flag = true;
+    ::MPI_Message message;
+    ::MPI_Status status;
+    int mpi_source;
+
+    while(flag) // loop until the end of global queue
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status); 
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message; //printf("myRank = %d, new ok\n", myRank);
+        msg_block->mpi_message = message;  //printf("myRank = %d, msg_block->mpi_message = %d\n", myRank, msg_block->mpi_message);
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();  //printf("myRank = %d, msg_block->ep_tag = %d\n", myRank, msg_block->ep_tag);
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  //printf("myRank = %d, src_loc = %d\n",           myRank, src_loc);
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();  //printf("myRank = %d, dest_loc = %d\n",          myRank, dest_loc);
+        int src_mpi       = status.MPI_SOURCE;                            //printf("myRank = %d, src_mpi = %d\n",           myRank, src_mpi);
+             
+        msg_block->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);        //printf("myRank = %d, msg_block->ep_src = %d\n",  myRank, msg_block->ep_src);
+
+        //printf("myRank = %d, probed one message, ep_src = %d, ep_dest = %d, tag = %d, message = %d\n", myRank, msg_block->ep_src, msg_block->ep_dest, msg_block->ep_tag, msg_block->mpi_message);
+        msg_block->mpi_status = new ::MPI_Status(status);
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);  
+          // printf("myRank = %d, push_back OK, ep_src = %d, ep_tag = %d, mpi_status = %p (%p)\n", myRank, 
+          //                                                                                   ptr_comm_target->ep_comm_ptr->message_queue->back().ep_src,
+          //                                                                                   ptr_comm_target->ep_comm_ptr->message_queue->back().ep_tag,
+          //                                                                                   msg_block->mpi_status, &status);
+    
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        //printf("myRank = %d, delete msg_block, queue size = %lu\n", myRank, ptr_comm_target->ep_comm_ptr->message_queue->size());
+        
+      }
+
+    }
+
+    return MPI_SUCCESS;
+  }
+
+
+
+  int Message_Check_intercomm(MPI_Comm comm)
+  {
+    if(!comm.ep_comm_ptr->intercomm->mpi_inter_comm) return 0;
+
+    Debug("Message probing for intercomm\n");
+
+    int flag = true;
+    ::MPI_Message message;
+    ::MPI_Status status;
+    int mpi_source;
+    int current_ep_rank;
+    MPI_Comm_rank(comm, &current_ep_rank);
+
+    while(flag) // loop until the end of global queue "comm.ep_comm_ptr->intercomm->mpi_inter_comm"
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.ep_comm_ptr->intercomm->mpi_inter_comm);  // => mpi_intercomm
+      
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status);       
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message;
+
+        msg_block->mpi_message = message;
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();
+        int src_mpi       = status.MPI_SOURCE;
+        int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first;
+             
+        msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc,  src_mpi);
+        msg_block->mpi_status = new ::MPI_Status(status);
+
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        
+      }
+
+    }
+
+    flag = true;
+    while(flag) // loop until the end of global queue "comm.mpi_comm"
+    {
+      Debug("Message probing for intracomm\n");
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &status);
+        if(flag)
+        {
+          Debug("find message in mpi comm \n");
+          mpi_source = status.MPI_SOURCE;
+          int tag = status.MPI_TAG;
+          ::MPI_Mprobe(mpi_source, tag, mpi_comm, &message, &status);
+
+        }
+      }
+      #elif _intelmpi
+      ::MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, mpi_comm, &flag, &message, &status);       
+      #endif
+
+      if(flag)
+      {
+
+        MPI_Message *msg_block = new MPI_Message;
+       
+        msg_block->mpi_message = message;
+        msg_block->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();
+        int src_loc       = bitset<8> (status.MPI_TAG >> 8) .to_ulong();
+        int dest_loc      = bitset<8> (status.MPI_TAG)	    .to_ulong();
+        int src_mpi       = status.MPI_SOURCE;
+        int current_inter = comm.ep_comm_ptr->intercomm->local_rank_map->at(current_ep_rank).first;
+        
+        msg_block->ep_src  = get_ep_rank_intercomm(comm, src_loc, src_mpi);
+        msg_block->mpi_status = new ::MPI_Status(status);
+        
+
+        MPI_Comm* ptr_comm_list = comm.ep_comm_ptr->comm_list;
+        MPI_Comm* ptr_comm_target = &ptr_comm_list[dest_loc];
+
+
+        #pragma omp critical (_query)
+        {
+          #pragma omp flush
+          ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
+          printf("probed one message, ep_src = %d, tag = %d, mpi_status = %p (%p), message = %d\n", msg_block->ep_src, msg_block->ep_tag, msg_block->mpi_status, &status, msg_block->mpi_message);
+          #pragma omp flush
+        }
+        
+        delete msg_block;
+        
+      }
+
+    }
+
+    return MPI_SUCCESS;
+  }
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_probe.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_probe.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_probe.cpp	(revision 1134)
@@ -0,0 +1,147 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Iprobe(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status)
+  {
+    *flag = false;
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+      ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status);
+      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, flag, mpi_status);
+
+      status->mpi_status = mpi_status;
+      status->ep_src = src;
+      status->ep_tag = tag;
+      return 0;
+    }
+
+    Debug("calling MPI_Iprobe EP\n");
+
+    Message_Check(comm);
+
+    #pragma omp flush
+
+    
+    //printf("iprobe, message queue size = %lu, queue = %p\n", comm.ep_comm_ptr->message_queue->size(), comm.ep_comm_ptr->message_queue);
+
+    #pragma omp flush
+
+    #pragma omp critical (_query)
+    if(!comm.ep_comm_ptr->message_queue->empty())
+    {
+      for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); ++it)
+      {
+        bool src_matched = src<0? true: it->ep_src == src;
+        bool tag_matched = tag<0? true: it->ep_tag == tag;
+        
+        if(src_matched && tag_matched)        
+        {
+          Debug("find message\n");
+          // printf("iprobe, find in local message queue %p, src = %d, tag = %d\n", comm.ep_comm_ptr->message_queue, it->ep_src, it->ep_tag);
+
+          *flag = true;
+
+          ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status));
+
+          status->mpi_status = new ::MPI_Status(mpi_status);
+          status->ep_src = it->ep_src;
+          status->ep_tag = it->ep_tag;
+
+          break;
+        }
+
+      }
+    }
+
+    return 0;
+  }
+
+
+
+  int MPI_Improbe(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status)
+  {
+    *flag = false;
+    if(!comm.is_ep)
+    {
+      Debug("calling MPI_Improbe MPI\n");
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm>(comm.mpi_comm);
+      ::MPI_Status mpi_status;
+      ::MPI_Message mpi_message;
+
+      #ifdef _openmpi
+      #pragma omp critical (_mpi_call)
+      {
+        ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, flag, &mpi_status);
+        if(*flag)
+        {
+          ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, &mpi_message, &mpi_status);
+        }
+      }
+      #elif _intelmpi
+    	::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, flag, &mpi_message, &mpi_status);
+      #endif
+    	
+      status->mpi_status = new ::MPI_Status(mpi_status);
+      status->ep_src = src;
+      status->ep_tag = tag;
+
+      message->mpi_message = mpi_message;
+      message->ep_src = src;
+      message->ep_tag = tag;
+      return 0;
+    }
+
+    
+    Message_Check(comm);
+
+    #pragma omp flush
+
+    #pragma omp critical (_query)
+    if(! comm.ep_comm_ptr->message_queue->empty())
+    {
+      for(Message_list::iterator it = comm.ep_comm_ptr->message_queue->begin(); it!= comm.ep_comm_ptr->message_queue->end(); ++it)
+      {
+        bool src_matched = src<0? true: it->ep_src == src;
+        bool tag_matched = tag<0? true: it->ep_tag == tag;
+        
+        if(src_matched && tag_matched)
+        {
+          *flag = true;
+
+          ::MPI_Status mpi_status;
+          mpi_status = *(static_cast< ::MPI_Status *>(it->mpi_status));
+
+          status->mpi_status = new ::MPI_Status(mpi_status);
+          status->ep_src = it->ep_src;
+          status->ep_tag = it->ep_tag;
+
+          message->mpi_message = it->mpi_message;
+          message->ep_tag = it->ep_tag;
+          message->ep_src = it->ep_src;
+
+          #pragma omp critical (_query2)
+          {              
+            //printf("local message erased. src = %d, dest = %d, tag = %d\n", it->ep_src, it->ep_dest, it->ep_tag);     
+            delete it->mpi_status;
+            comm.ep_comm_ptr->message_queue->erase(it);
+            #pragma omp flush
+          }
+
+          break;
+        }
+
+      }
+    }
+  }
+
+}
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_rank.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_rank.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_rank.cpp	(revision 1134)
@@ -0,0 +1,50 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+  int MPI_Comm_rank(MPI_Comm comm, int* rank)
+  {
+
+    if(comm.is_ep)
+    {
+      Debug("Calling EP_Comm_rank\n");
+
+      if(comm.is_intercomm)
+      {
+        *rank = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+        return 0;
+      }
+      else
+      {
+        *rank = comm.ep_comm_ptr->size_rank_info[0].first;
+        return 0;
+      }
+    }
+
+
+    if(comm != MPI_COMM_NULL)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Comm_rank(mpi_comm, rank);
+      return 0;
+    }
+    else
+    {
+      printf("rank is undefined\n");
+
+      *rank = MPI_UNDEFINED;
+      return 0;
+    }
+
+  }
+
+
+
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1134)
@@ -0,0 +1,141 @@
+/*!
+   \file ep_recv.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI receive functions: MPI_Recv, MPI_Mrecv, MPI_Irecv, MPI_Imrecv
+ */
+
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+	int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status)
+	{
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.mpi_comm);
+      ::MPI_Status mpi_status;
+      ::MPI_Recv(buf, count, static_cast< ::MPI_Datatype >(datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, &mpi_status);
+
+      status->ep_src = src;
+      status->ep_tag = tag;
+      status->ep_datatype = datatype;
+
+      return 0;  
+    }
+
+    Message_Check(comm);
+
+    MPI_Request request;
+    MPI_Irecv(buf, count, datatype, src, tag, comm, &request);
+    MPI_Wait(&request, status);
+
+    return 0;
+	}
+
+
+
+
+	int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request)
+	{
+
+		Debug("MPI_Irecv with EP");
+    int dest_rank;
+    MPI_Comm_rank(comm, &dest_rank);
+
+ 		if(!comm.is_ep)
+		{
+		  ::MPI_Request mpi_request;
+		  ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+		  ::MPI_Irecv(buf, count, static_cast< ::MPI_Datatype> (datatype), src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, mpi_comm, &mpi_request);
+
+		  request->mpi_request = mpi_request;
+      request->ep_src = src;
+      request->ep_datatype = datatype;
+      request->ep_tag = tag;
+		}
+
+		Message_Check(comm);
+		
+
+		request->mpi_request = MPI_REQUEST_NULL_STD;
+		request->buf = buf;
+    request->comm = comm;
+		request->type = 2;
+
+		request->ep_src = src;
+    request->ep_tag = tag;
+    request->ep_datatype = datatype;
+
+
+    
+		/* With Improbe*/
+		Message_Check(comm);
+
+    int flag = false;
+    MPI_Message message;
+    MPI_Status status;
+    
+    MPI_Improbe(src, tag, comm, &flag, &message, &status);
+            
+    if(flag)
+    {
+      MPI_Imrecv(buf, count, datatype, &message, request);
+    }
+
+    return 0;
+	}
+
+	int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
+	{
+		Debug("MPI_Imrecv");
+
+		request->type = 3;
+
+    ::MPI_Request mpi_request;
+    ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
+		
+		::MPI_Imrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->ep_datatype = datatype;
+    request->ep_tag = message->ep_tag;
+    request->ep_src = message->ep_src;
+    request->buf = buf;
+
+    return 0;
+	}
+
+
+  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Status *status)
+	{
+    Debug("EP Mrecv called\n");
+
+    ::MPI_Status mpi_status;
+    ::MPI_Message mpi_message = static_cast< ::MPI_Message >(message->mpi_message);
+		
+		::MPI_Mrecv(buf, count, static_cast< ::MPI_Datatype>(datatype), &mpi_message, &mpi_status);
+
+    status->mpi_status = new ::MPI_Status(mpi_status);
+    status->ep_src = message->ep_src;
+    status->ep_datatype = datatype;
+    status->ep_tag = message->ep_tag;
+
+    //check_sum_recv(buf, count, datatype, message->ep_src, message->ep_tag);
+
+    return 0;
+	}
+
+
+  
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_reduce.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_reduce.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_reduce.cpp	(revision 1134)
@@ -0,0 +1,670 @@
+/*!
+   \file ep_reduce.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Reduce, MPI_Allreduce
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+
+  int MPI_Reduce_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Reduce_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Reduce_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Reduce_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Reduce Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+
+  int MPI_Reduce_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Reduce_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+          }
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+  int MPI_Reduce_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(const_cast<void*>(recvbuf));
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if( 0 == my_rank )
+      {
+        #pragma omp critical (write_to_buffer)
+        copy(send_buf+j, send_buf+j + min(BUFFER_SIZE, count-j), buffer);
+        #pragma omp flush
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank !=0 )
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          #pragma omp flush
+
+
+          if(op == MPI_SUM)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+          }
+
+          else if (op == MPI_MAX)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+          }
+
+
+          else if (op == MPI_MIN)
+          {
+            transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+          }
+
+          else
+          {
+            printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+            exit(1);
+          }
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      if(my_rank == 0)
+      {
+        #pragma omp flush
+        copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+      }
+      MPI_Barrier_local(comm);
+    }
+  }
+
+
+  int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Reduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root,
+                   static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+
+    if(!comm.mpi_comm) return 0;
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Reduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    if(root_ep_loc != 0 && mpi_rank == root_mpi_rank) // root is not master, master send to root and root receive from master
+    {
+      innode_memcpy(0, recvbuf, root_ep_loc, recvbuf, count, datatype, comm);
+    }
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+
+  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Allreduce(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                      static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint recvsize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &recvsize);
+
+    void *local_recvbuf;
+    if(ep_rank_loc==0)
+    {
+      local_recvbuf = new void*[recvsize*count];
+    }
+
+    MPI_Reduce_local(sendbuf, local_recvbuf, count, datatype, op, comm);
+
+
+    if(ep_rank_loc==0)
+    {
+      ::MPI_Allreduce(local_recvbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+    MPI_Bcast_local(recvbuf, count, datatype, comm);
+
+    if(ep_rank_loc==0)
+    {
+      if(datatype == MPI_INT) delete[] static_cast<int*>(local_recvbuf);
+      else if(datatype == MPI_FLOAT) delete[] static_cast<float*>(local_recvbuf);
+      else if(datatype == MPI_DOUBLE) delete[] static_cast<double*>(local_recvbuf);
+      else if(datatype == MPI_LONG) delete[] static_cast<long*>(local_recvbuf);
+      else if(datatype == MPI_UNSIGNED_LONG) delete[] static_cast<unsigned long*>(local_recvbuf);
+      else delete[] static_cast<char*>(local_recvbuf);
+    }
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+
+    if(!comm.is_ep && comm.mpi_comm)
+    {
+      ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op),
+                           static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+    void* local_buf;
+    void* local_buf2;
+    int local_buf_size = accumulate(recvcounts, recvcounts+ep_size, 0);
+    int local_buf2_size = accumulate(recvcounts+ep_rank-ep_rank_loc, recvcounts+ep_rank-ep_rank_loc+num_ep, 0);
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    if(ep_rank_loc == 0)
+    {
+      local_buf = new void*[local_buf_size*datasize];
+      local_buf2 = new void*[local_buf2_size*datasize];
+    }
+    MPI_Reduce_local(sendbuf, local_buf, local_buf_size, MPI_INT, op, comm);
+
+
+    if(ep_rank_loc == 0)
+    {
+      int local_recvcnt[mpi_size];
+      for(int i=0; i<mpi_size; i++)
+      {
+        local_recvcnt[i] = accumulate(recvcounts+ep_rank, recvcounts+ep_rank+num_ep, 0);
+      }
+
+      ::MPI_Reduce_scatter(local_buf, local_buf2, local_recvcnt, static_cast< ::MPI_Datatype>(datatype),
+                         static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    int displs[num_ep];
+    displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      displs[i] = displs[i-1] + recvcounts[ep_rank-ep_rank_loc+i-1];
+    }
+
+    MPI_Scatterv_local(local_buf2, recvcounts+ep_rank-ep_rank_loc, displs, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        delete[] static_cast<int*>(local_buf);
+        delete[] static_cast<int*>(local_buf2);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        delete[] static_cast<float*>(local_buf);
+        delete[] static_cast<float*>(local_buf2);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<double*>(local_buf);
+        delete[] static_cast<double*>(local_buf2);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        delete[] static_cast<long*>(local_buf);
+        delete[] static_cast<long*>(local_buf2);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        delete[] static_cast<unsigned long*>(local_buf);
+        delete[] static_cast<unsigned long*>(local_buf2);
+      }
+      else // if(datatype == MPI_DOUBLE)
+      {
+        delete[] static_cast<char*>(local_buf);
+        delete[] static_cast<char*>(local_buf2);
+      }
+    }
+
+    Message_Check(comm);
+    return 0;
+  }
+}
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scan.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scan.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scan.cpp	(revision 1134)
@@ -0,0 +1,747 @@
+/*!
+   \file ep_scan.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scan
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+  template<typename T>
+  T max_op(T a, T b)
+  {
+    return max(a,b);
+  }
+
+  template<typename T>
+  T min_op(T a, T b)
+  {
+    return min(a,b);
+  }
+
+
+  int MPI_Scan_local(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      return MPI_Scan_local_int(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      return MPI_Scan_local_float(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      return MPI_Scan_local_double(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      return MPI_Scan_local_long(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      return MPI_Scan_local_ulong(sendbuf, recvbuf, count, op, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      return MPI_Scan_local_char(sendbuf, recvbuf, count, op, comm);
+    }
+    else
+    {
+      printf("MPI_Scan Datatype not supported!\n");
+      exit(0);
+    }
+
+  }
+
+
+
+
+  int MPI_Scan_local_int(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<int>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<int>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+
+  }
+
+  int MPI_Scan_local_float(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<float>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<float>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_double(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<double>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<double>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_long(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_ulong(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<unsigned long>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<unsigned long>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scan_local_char(const void *sendbuf, void *recvbuf, int count, MPI_Op op, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int j=0; j<count; j+=BUFFER_SIZE)
+    {
+      if(my_rank == 0)
+      {
+
+        #pragma omp critical (write_to_buffer)
+        {
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), buffer);
+          copy(send_buf+j, send_buf+j+min(BUFFER_SIZE, count-j), recv_buf+j);
+          #pragma omp flush
+        }
+      }
+
+      MPI_Barrier_local(comm);
+
+      for(int k=1; k<num_ep; k++)
+      {
+        #pragma omp critical (write_to_buffer)
+        {
+          if(my_rank == k)
+          {
+            #pragma omp flush
+            if(op == MPI_SUM)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, std::plus<char>());
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MAX)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, max_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else if(op == MPI_MIN)
+            {
+              transform(buffer, buffer+min(BUFFER_SIZE, count-j), send_buf+j, buffer, min_op<char>);
+              copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+            }
+            else
+            {
+              printf("Supported operation: MPI_SUM, MPI_MAX, MPI_MIN\n");
+              exit(1);
+            }
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+
+      ::MPI_Scan(sendbuf, recvbuf, count, static_cast< ::MPI_Datatype>(datatype),
+                 static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    void* local_scan_recvbuf;
+    local_scan_recvbuf = new void*[datasize * count];
+
+
+    // local scan
+    MPI_Scan_local(sendbuf, recvbuf, count, datatype, op, comm);
+
+//     MPI_scan
+    void* local_sum;
+    void* mpi_scan_recvbuf;
+
+
+    mpi_scan_recvbuf = new void*[datasize*count];
+
+    if(ep_rank_loc == 0)
+    {
+      local_sum = new void*[datasize*count];
+    }
+
+
+    MPI_Reduce_local(sendbuf, local_sum, count, datatype, op, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Exscan(local_sum, mpi_scan_recvbuf, count, static_cast< ::MPI_Datatype>(datatype), static_cast< ::MPI_Op>(op), static_cast< ::MPI_Comm>(comm.mpi_comm));
+    }
+
+
+    if(mpi_rank > 0)
+    {
+      MPI_Bcast_local(mpi_scan_recvbuf, count, datatype, comm);
+    }
+
+
+    if(datatype == MPI_DOUBLE)
+    {
+      double* sum_buf = static_cast<double*>(mpi_scan_recvbuf);
+      double* recv_buf = static_cast<double*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<double*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<double*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_FLOAT)
+    {
+      float* sum_buf = static_cast<float*>(mpi_scan_recvbuf);
+      float* recv_buf = static_cast<float*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<float*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<float*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_INT)
+    {
+      int* sum_buf = static_cast<int*>(mpi_scan_recvbuf);
+      int* recv_buf = static_cast<int*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<int*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<int*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_LONG)
+    {
+      long* sum_buf = static_cast<long*>(mpi_scan_recvbuf);
+      long* recv_buf = static_cast<long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      unsigned long* sum_buf = static_cast<unsigned long*>(mpi_scan_recvbuf);
+      unsigned long* recv_buf = static_cast<unsigned long*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<unsigned long*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<unsigned long*>(local_sum);
+      }
+    }
+
+    else if(datatype == MPI_CHAR)
+    {
+      char* sum_buf = static_cast<char*>(mpi_scan_recvbuf);
+      char* recv_buf = static_cast<char*>(recvbuf);
+
+      if(mpi_rank != 0)
+      {
+        if(op == MPI_SUM)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] += sum_buf[i];
+          }
+        }
+        else if (op == MPI_MAX)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = max(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else if(op == MPI_MIN)
+        {
+          for(int i=0; i<count; i++)
+          {
+            recv_buf[i] = min(recv_buf[i], sum_buf[i]);
+          }
+        }
+        else
+        {
+          printf("Support operator for MPI_Scan is MPI_SUM, MPI_MAX, and MPI_MIN\n");
+          exit(1);
+        }
+      }
+
+      delete[] static_cast<char*>(mpi_scan_recvbuf);
+      if(ep_rank_loc == 0)
+      {
+        delete[] static_cast<char*>(local_sum);
+      }
+    }
+
+
+  }
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatter.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatter.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatter.cpp	(revision 1134)
@@ -0,0 +1,396 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scatter
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Scatter_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Scatter_local_int(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Scatter_local_float(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Scatter_local_double(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Scatter_local_long(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Scatter_local_ulong(sendbuf, count, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Scatter_local_char(sendbuf, count, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_Scatter Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Scatter_local_int(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_float(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_double(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatter_local_long(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_ulong(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatter_local_char(const void *sendbuf, int count, void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+k*count+j, send_buf+k*count+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+
+
+
+  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Scatter(sendbuf, sendcount, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount, static_cast< ::MPI_Datatype>(recvtype),
+                    root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype) && sendcount == recvcount);
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+    MPI_Datatype datatype = sendtype;
+    int count = sendcount;
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+
+    void *master_sendbuf;
+    void *local_recvbuf;
+
+    if(root_ep_loc!=0 && mpi_rank == root_mpi_rank)
+    {
+      if(ep_rank_loc == 0) master_sendbuf = new void*[datasize*count*ep_size];
+
+      innode_memcpy(root_ep_loc, sendbuf, 0, master_sendbuf, count*ep_size, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc == 0)
+    {
+      int mpi_sendcnt = count*num_ep;
+      int mpi_scatterv_sendcnt[mpi_size];
+      int displs[mpi_size];
+
+      local_recvbuf = new void*[datasize*mpi_sendcnt];
+
+      ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      displs[0] = 0;
+      for(int i=1; i<mpi_size; i++)
+        displs[i] = displs[i-1] + mpi_scatterv_sendcnt[i-1];
+
+
+      if(root_ep_loc!=0)
+      {
+        ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+    }
+
+    MPI_Scatter_local(local_recvbuf, count, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<int*>(master_sendbuf);
+        delete[] static_cast<int*>(local_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<float*>(master_sendbuf);
+        delete[] static_cast<float*>(local_recvbuf);
+      }
+      else if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<double*>(master_sendbuf);
+        delete[] static_cast<double*>(local_recvbuf);
+      }
+      else if(datatype == MPI_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<long*>(master_sendbuf);
+        delete[] static_cast<long*>(local_recvbuf);
+      }
+      else if(datatype == MPI_UNSIGNED_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<unsigned long*>(master_sendbuf);
+        delete[] static_cast<unsigned long*>(local_recvbuf);
+      }
+      else //if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<char*>(master_sendbuf);
+        delete[] static_cast<char*>(local_recvbuf);
+      }
+    }
+
+  }
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatterv.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatterv.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_scatterv.cpp	(revision 1134)
@@ -0,0 +1,429 @@
+/*!
+   \file ep_gather.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI collective function: MPI_Scatterv
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  int MPI_Scatterv_local(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    if(datatype == MPI_INT)
+    {
+      Debug("datatype is INT\n");
+      return MPI_Scatterv_local_int(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_FLOAT)
+    {
+      Debug("datatype is FLOAT\n");
+      return MPI_Scatterv_local_float(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_DOUBLE)
+    {
+      Debug("datatype is DOUBLE\n");
+      return MPI_Scatterv_local_double(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_LONG)
+    {
+      Debug("datatype is LONG\n");
+      return MPI_Scatterv_local_long(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_UNSIGNED_LONG)
+    {
+      Debug("datatype is uLONG\n");
+      return MPI_Scatterv_local_ulong(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else if(datatype == MPI_CHAR)
+    {
+      Debug("datatype is CHAR\n");
+      return MPI_Scatterv_local_char(sendbuf, sendcounts, displs, recvbuf, comm);
+    }
+    else
+    {
+      printf("MPI_scatterv Datatype not supported!\n");
+      exit(0);
+    }
+  }
+
+  int MPI_Scatterv_local_int(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    int *buffer = comm.my_buffer->buf_int;
+    int *send_buf = static_cast<int*>(const_cast<void*>(sendbuf));
+    int *recv_buf = static_cast<int*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_float(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    float *buffer = comm.my_buffer->buf_float;
+    float *send_buf = static_cast<float*>(const_cast<void*>(sendbuf));
+    float *recv_buf = static_cast<float*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_double(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    double *buffer = comm.my_buffer->buf_double;
+    double *send_buf = static_cast<double*>(const_cast<void*>(sendbuf));
+    double *recv_buf = static_cast<double*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+  int MPI_Scatterv_local_long(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    long *buffer = comm.my_buffer->buf_long;
+    long *send_buf = static_cast<long*>(const_cast<void*>(sendbuf));
+    long *recv_buf = static_cast<long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv_local_ulong(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    unsigned long *buffer = comm.my_buffer->buf_ulong;
+    unsigned long *send_buf = static_cast<unsigned long*>(const_cast<void*>(sendbuf));
+    unsigned long *recv_buf = static_cast<unsigned long*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv_local_char(const void *sendbuf, const int sendcounts[], const int displs[], void *recvbuf, MPI_Comm comm)
+  {
+    int my_rank = comm.ep_comm_ptr->size_rank_info[1].first;
+    int num_ep  = comm.ep_comm_ptr->size_rank_info[1].second;
+
+
+    char *buffer = comm.my_buffer->buf_char;
+    char *send_buf = static_cast<char*>(const_cast<void*>(sendbuf));
+    char *recv_buf = static_cast<char*>(recvbuf);
+
+    for(int k=0; k<num_ep; k++)
+    {
+      int count = sendcounts[k];
+      for(int j=0; j<count; j+=BUFFER_SIZE)
+      {
+        if(my_rank == 0)
+        {
+          #pragma omp critical (write_to_buffer)
+          {
+            copy(send_buf+displs[k]+j, send_buf+displs[k]+j+min(BUFFER_SIZE, count-j), buffer);
+            #pragma omp flush
+          }
+        }
+
+        MPI_Barrier_local(comm);
+
+        if(my_rank == k)
+        {
+          #pragma omp critical (read_from_buffer)
+          {
+            #pragma omp flush
+            copy(buffer, buffer+min(BUFFER_SIZE, count-j), recv_buf+j);
+          }
+        }
+        MPI_Barrier_local(comm);
+      }
+    }
+  }
+
+
+  int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Scatterv(sendbuf, sendcounts, displs, static_cast< ::MPI_Datatype>(sendtype), recvbuf, recvcount,
+                     static_cast< ::MPI_Datatype>(recvtype), root, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      return 0;
+    }
+    if(!comm.mpi_comm) return 0;
+
+    assert(static_cast< ::MPI_Datatype>(sendtype) == static_cast< ::MPI_Datatype>(recvtype));
+
+    MPI_Datatype datatype = sendtype;
+
+    int root_mpi_rank = comm.rank_map->at(root).second;
+    int root_ep_loc = comm.rank_map->at(root).first;
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+    if(ep_rank != root)
+    {
+      sendcounts = new int[ep_size];
+      displs = new int[ep_size];
+    }
+    
+    MPI_Bcast(const_cast<int*>(sendcounts), ep_size, MPI_INT, root, comm);
+    MPI_Bcast(const_cast<int*>(displs), ep_size, MPI_INT, root, comm);
+
+
+    int count = recvcount;
+
+    ::MPI_Aint datasize, lb;
+
+    ::MPI_Type_get_extent(static_cast< ::MPI_Datatype>(datatype), &lb, &datasize);
+
+    assert(accumulate(sendcounts, sendcounts+ep_size-1, 0) == displs[ep_size-1]); // Only for contunuous gather.
+
+
+    void *master_sendbuf;
+    void *local_recvbuf;
+
+    if(root_ep_loc!=0 && mpi_rank == root_mpi_rank)
+    {
+      int count_sum = accumulate(sendcounts, sendcounts+ep_size, 0);
+      if(ep_rank_loc == 0) master_sendbuf = new void*[datasize*count_sum];
+
+      innode_memcpy(root_ep_loc, sendbuf, 0, master_sendbuf, count_sum, datatype, comm);
+    }
+
+
+
+    if(ep_rank_loc == 0)
+    {
+      int mpi_sendcnt = accumulate(sendcounts+ep_rank, sendcounts+ep_rank+num_ep, 0);
+      int mpi_scatterv_sendcnt[mpi_size];
+      int mpi_displs[mpi_size];
+
+      local_recvbuf = new void*[datasize*mpi_sendcnt];
+
+      ::MPI_Gather(&mpi_sendcnt, 1, MPI_INT_STD, mpi_scatterv_sendcnt, 1, MPI_INT_STD, root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+
+      mpi_displs[0] = displs[0];
+      for(int i=1; i<mpi_size; i++)
+        mpi_displs[i] = mpi_displs[i-1] + mpi_scatterv_sendcnt[i-1];
+
+
+      if(root_ep_loc!=0)
+      {
+        ::MPI_Scatterv(master_sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+      else
+      {
+        ::MPI_Scatterv(sendbuf, mpi_scatterv_sendcnt, mpi_displs, static_cast< ::MPI_Datatype>(datatype),
+                     local_recvbuf, mpi_sendcnt, static_cast< ::MPI_Datatype>(datatype), root_mpi_rank, static_cast< ::MPI_Comm>(comm.mpi_comm));
+      }
+    }
+
+    int local_displs[num_ep];
+    local_displs[0] = 0;
+    for(int i=1; i<num_ep; i++)
+    {
+      local_displs[i] = displs[ep_rank-ep_rank_loc+i]-displs[ep_rank-ep_rank_loc];
+    }
+
+    MPI_Scatterv_local(local_recvbuf, sendcounts+ep_rank-ep_rank_loc, local_displs, datatype, recvbuf, comm);
+
+    if(ep_rank_loc == 0)
+    {
+      if(datatype == MPI_INT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<int*>(master_sendbuf);
+        delete[] static_cast<int*>(local_recvbuf);
+      }
+      else if(datatype == MPI_FLOAT)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<float*>(master_sendbuf);
+        delete[] static_cast<float*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<double*>(master_sendbuf);
+        delete[] static_cast<double*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<long*>(master_sendbuf);
+        delete[] static_cast<long*>(local_recvbuf);
+      }
+      else  if(datatype == MPI_UNSIGNED_LONG)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<unsigned long*>(master_sendbuf);
+        delete[] static_cast<unsigned long*>(local_recvbuf);
+      }
+      else // if(datatype == MPI_DOUBLE)
+      {
+        if(root_ep_loc!=0 && mpi_rank == root_mpi_rank) delete[] static_cast<char*>(master_sendbuf);
+        delete[] static_cast<char*>(local_recvbuf);
+      }
+    }
+    else
+    {
+      delete[] sendcounts;
+      delete[] displs;
+    }
+
+  }
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1134)
@@ -0,0 +1,318 @@
+/*!
+   \file ep_send.hpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI send functions: MPI_Send, MPI_Ssend, MPI_Isend, MPI_Issend
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+
+namespace ep_lib {
+
+
+  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Send(buf, count, static_cast< ::MPI_Datatype>(datatype), dest, tag, mpi_comm);
+      return 0;
+    }
+
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Isend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm);
+
+    return 0;
+  }
+
+
+  int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Ssend(buf, count, static_cast< ::MPI_Datatype>(datatype), dest, tag, mpi_comm);
+      return 0;
+    }
+
+    MPI_Request request;
+    MPI_Status status;
+    MPI_Issend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+    //check_sum_send(buf, count, datatype, dest, tag, comm);
+    return 0;
+  }
+
+
+
+  int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Isend with EP\n");
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+    
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype> (datatype), dest, tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+
+      request->ep_src = src_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+      request->type = 1;
+      request->comm = comm;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    // EP intracomm
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+    int mpi_tag 		= tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest 	= comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+
+    request->ep_src  = src_rank;
+    request->ep_tag  = tag;
+    request->ep_datatype = datatype;
+
+    ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+    ::MPI_Request mpi_request;
+
+    ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype>(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->type = 1; 		// used in wait
+    request->comm = comm;
+
+    Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+
+  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Issend with EP\n");
+
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+    
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype> (datatype), dest, tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->ep_src = src_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+      request->type = 1;
+      request->comm = comm;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+    // EP intracomm
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+    int mpi_tag 		= tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest 	= comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+    
+    request->ep_src = src_rank;
+    request->ep_tag = tag;
+    request->ep_datatype = datatype;
+
+    ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+    ::MPI_Request mpi_request;
+
+    ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype>(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->type = 1; 		// used in wait
+    request->comm = comm;
+    
+
+    //Message_Check(comm);
+
+    return 0;
+  }
+
+
+
+  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Isend with intercomm\n");
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
+    int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second;
+
+    int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int src_comm_label;
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++)
+    {
+      if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
+      {
+        src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second;
+        break;
+      }
+    }
+
+    Message_Check(comm);
+
+
+    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs
+    {
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Request mpi_request;
+  
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    else   // dest_remote_comm_label != src_comm_label
+    { 
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Request mpi_request;
+
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+   
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    return 0;
+
+  }
+
+
+  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("MPI_Issend with intercomm\n");
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
+    int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second;
+
+    int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int src_comm_label;
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++)
+    {
+      if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
+      {
+        src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second;
+        break;
+      }
+    }
+
+    Message_Check(comm);
+
+
+    if(dest_remote_comm_label == src_comm_label)       // dest rank (loc, mpi) differs
+    {
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Request mpi_request;
+  
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    else   // dest_remote_comm_label != src_comm_label
+    { 
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Request mpi_request;
+
+      ::MPI_Issend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+   
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    return 0;
+
+  }
+}
+
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_size.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_size.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_size.cpp	(revision 1134)
@@ -0,0 +1,80 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+namespace ep_lib
+{
+
+
+
+
+  int MPI_Comm_size(MPI_Comm comm, int* size)
+  {
+    
+    if(comm.is_ep)
+    {
+      if(!comm.is_intercomm)
+      {
+        *size = comm.ep_comm_ptr->size_rank_info[0].second;
+
+        return 0;
+      }
+      else
+      {
+        *size = comm.ep_comm_ptr->intercomm->size_rank_info[0].second;
+
+        return 0;
+      }
+
+    }
+
+    Debug("Calling EP_Comm_size\n");
+
+    if(comm.mpi_comm != MPI_COMM_NULL_STD)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
+      int mpi_size;
+
+      ::MPI_Comm_size(mpi_comm, &mpi_size);
+      printf("============ ep comm size called for non ep comm, %d\n", mpi_size);
+      *size = mpi_size;
+      return 0;
+    }
+
+    else
+    {
+      *size = MPI_UNDEFINED;
+      printf("============ ep comm size called for non ep comm not defined\n");
+      return 0;
+    }
+  }
+
+  int MPI_Comm_remote_size(MPI_Comm comm, int *size)
+  {
+    if(!comm.is_ep)
+    {
+      if(comm.mpi_comm != MPI_COMM_NULL_STD)
+      {
+        ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm); 
+        ::MPI_Comm_remote_size(mpi_comm, size);
+        return 0;
+      }
+
+      else
+      {
+        *size = MPI_UNDEFINED;
+        return 0;
+      }
+    }
+    
+    if(comm.is_intercomm)
+    {
+      *size = comm.ep_comm_ptr->intercomm->remote_rank_map->size();
+      return 0;
+    }
+    *size = MPI_UNDEFINED;
+    return 0; 
+  }
+}
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_split.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_split.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_split.cpp	(revision 1134)
@@ -0,0 +1,179 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+namespace ep_lib
+{
+
+  void vec_simplify(std::vector<int> *inout_vector)
+  {
+    std::vector<int> out_vec;
+    int found=false;
+    for(std::vector<int>::iterator it_in = inout_vector->begin() ; it_in != inout_vector->end(); ++it_in)
+    {
+      for(std::vector<int>::iterator it = out_vec.begin() ; it != out_vec.end(); ++it)
+      {
+        if(*it_in == *it)
+        {
+          found=true;
+          break;
+        }
+        else found=false;
+      }
+      if(found == false)
+      {
+        out_vec.push_back(*it_in);
+      }
+    }
+    inout_vector->swap(out_vec);
+  }
+
+
+
+  int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = comm.ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = comm.ep_comm_ptr->size_rank_info[2].first;
+    ep_size = comm.ep_comm_ptr->size_rank_info[0].second;
+    num_ep = comm.ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = comm.ep_comm_ptr->size_rank_info[2].second;
+
+
+
+
+    int num_color = 0;
+
+    int color_index;
+
+    vector<int> matched_number;
+    vector<int> matched_number_loc;
+
+
+    vector<int> all_color(ep_size);
+    vector<int> all_color_loc(num_ep);
+
+    MPI_Gather_local(&color, 1, MPI_INT, all_color_loc.data(), comm);
+    MPI_Bcast_local(all_color_loc.data(), num_ep, MPI_INT, comm);
+
+
+    MPI_Allgather(&color, 1, MPI_INT, all_color.data(), 1, MPI_INT, comm);
+
+    list<int> color_list(all_color.begin(), all_color.end());
+    list<int> color_list_loc(all_color_loc.begin(), all_color_loc.end());
+
+    vec_simplify(&all_color);
+
+    matched_number.resize(all_color.size(), 0);
+    matched_number_loc.resize(all_color.size(), 0);
+
+
+    while(!color_list.empty())
+    {
+      int target_color = color_list.front();
+      for(list<int>::iterator it = color_list.begin(); it != color_list.end(); ++it)
+      {
+        if(*it == target_color)
+        {
+          matched_number[num_color]++;
+        }
+      }
+      for(list<int>::iterator it = color_list_loc.begin(); it != color_list_loc.end(); ++it)
+      {
+        if(*it == target_color)
+        {
+          matched_number_loc[num_color]++;
+        }
+      }
+      color_list.remove(target_color);
+      color_list_loc.remove(target_color);
+      num_color++;
+    }
+
+
+    vector<int> colored_key_loc[num_color];
+    vector<int> key_loc(num_ep);
+
+    MPI_Gather_local(&key, 1, MPI_INT, key_loc.data(), comm);
+    MPI_Bcast_local(key_loc.data(), num_ep, MPI_INT, comm);
+
+    for(int i=0; i<num_ep; i++)
+    {
+      for(int j = 0; j<num_color; j++)
+      {
+        if(all_color_loc[i] == all_color[j])
+        {
+          colored_key_loc[j].push_back(key_loc[i]);
+        }
+      }
+    }
+
+    int new_ep_rank_loc;
+
+    for(int i=0; i<num_color; i++)
+    {
+      if(colored_key_loc[i].size()>1)
+      {
+        std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end());
+      }
+      if(color == all_color[i])
+      {
+        //color_index = i;
+        for(int j=0; j<colored_key_loc[i].size(); j++)
+        {
+          if(key == colored_key_loc[i][j])
+          {
+            new_ep_rank_loc = j;
+            break;
+          }
+        }
+      }
+    }
+
+    ::MPI_Comm split_mpi_comm[num_color];
+
+    for(int j=0; j<num_color; j++)
+    {
+      if(ep_rank_loc == 0)
+      {
+        int master_color = 1;
+        if(matched_number_loc[j] == 0) master_color = MPI_UNDEFINED;
+
+        ::MPI_Comm_split(static_cast< ::MPI_Comm>(comm.mpi_comm), master_color, mpi_rank, &split_mpi_comm[j]);
+        
+        comm.ep_comm_ptr->comm_list->mpi_bridge = split_mpi_comm[j];
+      }
+      MPI_Barrier_local(comm);
+      int num_new_ep = 0;
+
+      if(new_ep_rank_loc == 0 && color == all_color[j])
+      {
+        num_new_ep = matched_number_loc[j];
+        MPI_Info info;
+        MPI_Comm *ep_comm;
+
+        MPI_Comm_create_endpoints(comm.ep_comm_ptr->comm_list->mpi_bridge, num_new_ep, info, ep_comm);
+
+
+        comm.ep_comm_ptr->comm_list->mem_bridge = ep_comm;
+      }
+      MPI_Barrier_local(comm);
+      if(color == all_color[j])
+      {
+        *newcomm = comm.ep_comm_ptr->comm_list->mem_bridge[new_ep_rank_loc];
+ //       newcomm = &(comm.ep_comm_ptr->comm_list->mem_bridge[new_ep_rank_loc]);
+        (*newcomm).ep_comm_ptr->comm_label = color;
+      }
+    }
+
+
+
+    return 0;
+  }
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_tag.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_tag.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_tag.hpp	(revision 1134)
@@ -0,0 +1,25 @@
+#ifndef EP_TAG_HPP_INCLUDED
+#define EP_TAG_HPP_INCLUDED
+
+
+
+namespace ep_lib
+{
+  int print_comm_info(MPI_Comm comm);
+
+
+
+//  int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request);
+//  int MPI_Mrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message);
+
+
+//  int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses);
+//  int MPI_Testany(int count, MPI_Request *array_of_requests, int *index, int *flag, MPI_Status *status);
+//  int MPI_Testsome(int incount, MPI_Request *array_of_requests, int *outcount, int *array_of_indices, MPI_Status *array_of_statuses);
+
+
+//  int MPI_Waitany(int count, MPI_Request *array_of_requests,  int *index, MPI_Status *array_of_statuses);
+//  int MPI_Waitsome(int incount, MPI_Request *array_of_requests, int *outcount, int *array_of_indices, MPI_Status *array_of_statuses);
+}
+
+#endif // EP_TAG_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_test.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_test.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_test.cpp	(revision 1134)
@@ -0,0 +1,97 @@
+/*!
+   \file ep_test.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI test function: MPI_Test, MPI_Testsome, MPI_Testany, MPI_Testall
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+namespace ep_lib {
+
+	int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
+	{
+		Debug("MPI_Test with EP");
+
+    *flag = false;
+
+    if(request->type == 1)      // isend
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Test(&mpi_request, flag, &mpi_status);
+      
+      if(*flag) 
+      {
+        status->mpi_status = new ::MPI_Status(mpi_status);
+        status->ep_src = request->ep_src;
+        status->ep_tag = request->ep_tag;
+        status->ep_datatype = request->ep_datatype;
+      }
+
+      return 0;
+    }
+
+    if(request->type == 2)   // irecv message not probed
+    {
+      Message_Check(request->comm);
+      #pragma omp flush
+      MPI_Message message;
+      MPI_Improbe(request->ep_src, request->ep_tag, request->comm, flag, &message, status);
+      if(*flag)
+      {
+        int count;
+        MPI_Get_count(status, request->ep_datatype, &count);
+        MPI_Imrecv(request->buf, count, request->ep_datatype, &message, request);
+        MPI_Test(request, flag, status);
+      }
+      return 0;
+    }
+
+    if(request->type == 3)  // imrecv
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Test(&mpi_request, flag, &mpi_status);
+      if(*flag)
+      {
+        status->mpi_status = new ::MPI_Status(mpi_status);
+        status->ep_src = request->ep_src;
+        status->ep_tag = request->ep_tag;
+        status->ep_datatype = request->ep_datatype;
+        //int count;
+        //MPI_Get_count(status, request->ep_datatype, &count);
+        //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+      }
+
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      
+
+      return 0;
+    }
+		
+	}
+
+
+	int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses)
+	{
+	  Debug("MPI_Testall with EP");
+    *flag = true;
+    int i=0;
+    while(*flag && i<count )
+    {
+      MPI_Test(&array_of_requests[i], flag, &array_of_statuses[i]);
+      i++;
+    }
+	}
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.cpp	(revision 1134)
@@ -0,0 +1,56 @@
+#include "ep_lib.hpp"
+#include <mpi.h>
+
+
+using namespace std;
+
+::MPI_Comm MPI_COMM_WORLD_STD = MPI_COMM_WORLD;
+#undef MPI_COMM_WORLD
+
+
+::MPI_Comm MPI_COMM_NULL_STD = MPI_COMM_NULL;
+#undef MPI_COMM_NULL
+
+
+::MPI_Request MPI_REQUEST_NULL_STD = MPI_REQUEST_NULL;
+#undef MPI_REQUEST_NULL
+
+::MPI_Info MPI_INFO_NULL_STD = MPI_INFO_NULL;
+#undef MPI_INFO_NULL
+
+::MPI_Datatype MPI_INT_STD = MPI_INT;
+::MPI_Datatype MPI_FLOAT_STD = MPI_FLOAT;
+::MPI_Datatype MPI_DOUBLE_STD = MPI_DOUBLE;
+::MPI_Datatype MPI_LONG_STD = MPI_LONG;
+::MPI_Datatype MPI_CHAR_STD = MPI_CHAR;
+::MPI_Datatype MPI_UNSIGNED_LONG_STD = MPI_UNSIGNED_LONG;
+::MPI_Datatype MPI_UNSIGNED_CHAR_STD = MPI_UNSIGNED_CHAR;
+
+#undef MPI_INT
+#undef MPI_FLOAT
+#undef MPI_DOUBLE
+#undef MPI_LONG
+#undef MPI_CHAR
+#undef MPI_UNSIGNED_LONG
+#undef MPI_UNSIGNED_CHAR
+
+
+::MPI_Op MPI_SUM_STD = MPI_SUM;
+::MPI_Op MPI_MAX_STD = MPI_MAX;
+::MPI_Op MPI_MIN_STD = MPI_MIN;
+
+#undef MPI_SUM
+#undef MPI_MAX
+#undef MPI_MIN
+
+
+
+
+
+
+
+
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp	(revision 1134)
@@ -0,0 +1,496 @@
+#ifndef EP_TYPE_HPP_INCLUDED
+#define EP_TYPE_HPP_INCLUDED
+
+
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <list>
+#include <map>
+#include <omp.h>
+#include <vector>
+#include <numeric>
+#include <bitset>
+//#include <memory.h>
+#include <algorithm>
+#include <assert.h>
+#include <math.h>
+
+#ifdef _Debug
+#define Debug(x) std::cout << x << std::endl
+#else
+#define Debug(x)
+#endif
+
+#define BUFFER_SIZE 10000
+
+typedef std::pair< int, int > SIZE_RANK_INFO; // < rank, size>
+
+typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
+
+typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+
+
+typedef struct
+{
+  int first;
+  int second;
+  int third;
+} Triple_int;
+
+namespace ep_lib
+{
+  //#define EP_UNDEFINED -32766
+
+  class ep_communicator;
+  class ep_intercomm;
+  class OMPbarrier;
+  typedef ep_communicator* EP_Comm;
+  class MPI_Comm;
+
+
+  class MPI_Status
+  {
+    public:
+
+      #ifdef _intelmpi
+      int ep_datatype;
+      #elif _openmpi
+      void * ep_datatype;
+      #endif
+      
+      int ep_src;
+      int ep_tag;
+
+      
+      void* mpi_status;
+  };
+
+  class MPI_Message
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_message;
+      #elif _openmpi
+      void * mpi_message;
+      #endif
+
+      int ep_src;
+      int ep_tag;
+
+      void* mpi_status;
+
+      MPI_Message() {}
+
+      #ifdef _intelmpi
+      MPI_Message(int message): mpi_message(message) {}
+      #elif _openmpi
+      MPI_Message(void* message): mpi_message(message) {}
+      #endif
+  };
+
+  typedef std::list<MPI_Message > Message_list;
+
+
+  class OMPbarrier
+  {
+    private:
+      int nbThreads;          //<The number of threads for this barrier
+      int currentNbThread;    //<The current number of threads waiting
+      bool sense;             //<Direct barrier feedback protection
+      omp_lock_t mutex;       //<To have an atomic int
+
+      OMPbarrier(OMPbarrier&){}
+      OMPbarrier& operator=(OMPbarrier&){return *this;}
+
+    public:
+      /** Constructor with the number of threads */
+      explicit OMPbarrier(const int inNbThreads)
+          : nbThreads(inNbThreads), currentNbThread(0), sense(false) {
+          omp_init_lock( &mutex );
+      }
+
+      /** Destructor, release the omp lock */
+      ~OMPbarrier(){
+          omp_destroy_lock( &mutex );
+      }
+
+      /** Perform a barrier */
+      void wait(){
+          const bool mySense = sense;
+          omp_set_lock( &mutex );
+          const int nbThreadsArrived = (++currentNbThread);
+          omp_unset_lock( &mutex );
+
+          if(nbThreadsArrived == nbThreads) {
+              currentNbThread = 0;
+              sense = !sense;
+              #pragma omp flush
+          }
+          else {
+              volatile const bool* const ptSense = &sense;
+              while( (*ptSense) == mySense){
+              }
+          }
+      }
+
+
+      /** Change the number of threads */
+      void setNbThreads(const int inNbThread){
+          omp_set_lock( &mutex );
+          nbThreads = inNbThread;
+          omp_unset_lock( &mutex );
+      }
+  };
+
+  class ep_intercomm
+  {
+    public:
+
+
+    #ifdef _intelmpi
+    int mpi_inter_comm;
+    #elif _openmpi
+    void * mpi_inter_comm;
+    #endif
+
+    RANK_MAP *intercomm_rank_map;
+    RANK_MAP *local_rank_map;
+    RANK_MAP *remote_rank_map;
+
+
+    SIZE_RANK_INFO size_rank_info[3];
+
+
+    MPI_Comm *local_comm;
+    int intercomm_tag;
+
+    ep_intercomm()
+    {
+      intercomm_rank_map = NULL;
+      local_rank_map = NULL;
+      remote_rank_map = NULL;
+    }
+
+    // ep_intercomm(ep_intercomm &ref)
+    // {
+    //   printf("calling copy Constructor of ep_intercomm\n");
+    //   ep_intercomm return_intercomm;
+
+    //   return_intercomm.mpi_inter_comm = ref.mpi_inter_comm;
+    //   return_intercomm.intercomm_rank_map = ref.intercomm_rank_map;
+    //   return_intercomm.local_rank_map = ref.local_rank_map;
+    //   return_intercomm.remote_rank_map = ref.remote_rank_map;
+    //   return_intercomm.size_rank_info[0] = ref.size_rank_info[0];
+    //   return_intercomm.size_rank_info[1] = ref.size_rank_info[1];
+    //   return_intercomm.size_rank_info[2] = ref.size_rank_info[2];
+    //   return_intercomm.local_comm = ref.local_comm;
+    //   return_intercomm.intercomm_tag = ref.intercomm_tag;
+    // }
+
+    bool operator == (ep_intercomm right)
+    {
+      bool a = intercomm_rank_map == right.intercomm_rank_map;
+      bool b = local_rank_map == right.local_rank_map;
+      bool c = remote_rank_map == right.remote_rank_map;
+      bool d = mpi_inter_comm == right.mpi_inter_comm;
+      bool e = size_rank_info == right.size_rank_info;
+      bool f = intercomm_tag == right.intercomm_tag;
+      return a&&b&&c&&d&&e&&f;
+    }
+
+    bool operator != (ep_intercomm right)
+    {
+      bool a = intercomm_rank_map != right.intercomm_rank_map;
+      bool b = local_rank_map != right.local_rank_map;
+      bool c = remote_rank_map != right.remote_rank_map;
+      bool d = mpi_inter_comm != right.mpi_inter_comm;
+      bool e = size_rank_info != right.size_rank_info;
+      bool f = intercomm_tag != right.intercomm_tag;
+      return a||b||c||d||e||f;
+    }
+
+    // ep_intercomm operator = (ep_intercomm ref)
+    // {
+    //   printf("calling = operator of ep_intercomm\n");
+    //   ep_intercomm return_intercomm;
+
+    //   return_intercomm.mpi_inter_comm = ref.mpi_inter_comm;
+    //   return_intercomm.intercomm_rank_map = ref.intercomm_rank_map;
+    //   return_intercomm.local_rank_map = ref.local_rank_map;
+    //   return_intercomm.remote_rank_map = ref.remote_rank_map;
+    //   return_intercomm.size_rank_info[0] = ref.size_rank_info[0];
+    //   return_intercomm.size_rank_info[1] = ref.size_rank_info[1];
+    //   return_intercomm.size_rank_info[2] = ref.size_rank_info[2];
+    //   return_intercomm.local_comm = ref.local_comm;
+    //   return_intercomm.intercomm_tag = ref.intercomm_tag;
+    // }
+  };
+
+
+  class ep_communicator
+  {
+    public:
+
+    SIZE_RANK_INFO size_rank_info[3]; // 0: ep_rank,     ep_size
+                                      // 1: ep_rank_loc, num_ep
+                                      // 2: mpi_rank,    mpi_size
+
+
+    MPI_Comm *comm_list;
+
+    Message_list *message_queue;
+
+
+    int comm_label;
+
+    ep_intercomm *intercomm;
+
+    ep_communicator()
+    {
+      comm_list = NULL;
+      message_queue = NULL;
+      intercomm = NULL;
+    }
+
+    bool operator == (ep_communicator right)
+    {
+      bool a = size_rank_info == right.size_rank_info;
+      bool b = comm_label == right.comm_label;
+      bool c = intercomm == right.intercomm;
+      return a&&b&&c;
+    }
+
+    bool operator != (ep_communicator right)
+    {
+      bool a = size_rank_info != right.size_rank_info;
+      bool b = comm_label != right.comm_label;
+      bool c = intercomm != right.intercomm;
+      return a||b||c;
+    }
+
+    // ep_communicator operator = (ep_communicator ref)
+    // {
+    //   printf("calling = operator of ep_communicator\n");
+    //   ep_communicator return_ep;
+
+    //   return_ep.intercomm = ref.intercomm;
+    //   return_ep.comm_label = ref.comm_label;
+    //   return_ep.message_queue = ref.message_queue;
+    //   return_ep.comm_list = ref.comm_list;
+    //   return_ep.size_rank_info[0] = ref.size_rank_info[0];
+    //   return_ep.size_rank_info[1] = ref.size_rank_info[1];
+    //   return_ep.size_rank_info[2] = ref.size_rank_info[2];
+    // }
+  };
+
+
+  struct BUFFER
+  {
+    double *buf_double;
+    float  *buf_float;
+    int    *buf_int;
+    long    *buf_long;
+    unsigned long    *buf_ulong;
+    char    *buf_char;
+  };
+
+
+  class MPI_Comm
+  {
+    public:
+
+    #ifdef _intelmpi
+    int mpi_comm;
+    #elif _openmpi
+    void * mpi_comm;
+    #endif
+
+    bool is_ep;
+    bool is_intercomm;
+
+    BUFFER     *my_buffer;
+    OMPbarrier *ep_barrier;
+    RANK_MAP   *rank_map;
+
+    EP_Comm ep_comm_ptr;
+
+    MPI_Comm *mem_bridge;
+
+    #ifdef _intelmpi
+    int mpi_bridge;
+    #elif _openmpi
+    void * mpi_bridge;
+    #endif
+
+    MPI_Comm()
+    {
+      is_ep = true;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+    }
+
+    #ifdef _intelmpi
+    MPI_Comm(int comm)
+    {
+      is_ep = false;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+      mpi_comm = comm;
+    }
+
+    #elif _openmpi
+
+    MPI_Comm(void* comm)
+    {
+      is_ep = false;
+      is_intercomm = false;
+      my_buffer = NULL;
+      ep_barrier = NULL;
+      rank_map = NULL;
+      ep_comm_ptr = NULL;
+      mem_bridge = NULL;
+      mpi_bridge = NULL;
+      mpi_comm = comm;
+    }
+    #endif
+
+
+    bool operator == (MPI_Comm right)
+    {
+      bool a = is_ep == right.is_ep;
+      bool b = is_intercomm == right.is_intercomm;
+      bool c = mpi_comm == right.mpi_comm;
+      bool d = is_ep ? ep_comm_ptr == right.ep_comm_ptr : true;
+      return a&&b&&c&&d;
+    }
+
+    bool operator != (MPI_Comm right)
+    {
+      bool a = is_ep != right.is_ep;
+      bool b = is_intercomm != right.is_intercomm;
+      bool c = mpi_comm != right.mpi_comm;
+      bool d = is_ep ? ep_comm_ptr != right.ep_comm_ptr : true;
+
+      return a||b||c||d;
+    }
+
+    // MPI_Comm operator = (MPI_Comm ref)
+    // {
+    //   printf("calling = operator of MPI_Comm\n");
+    //   MPI_Comm return_comm;
+
+    //   return_comm.mpi_comm = ref.mpi_comm;
+    //   return_comm.is_ep = ref.is_ep;
+    //   return_comm.is_intercomm = ref.is_intercomm;
+    //   return_comm.my_buffer = ref.my_buffer;
+    //   return_comm.ep_barrier = ref.ep_barrier;
+    //   return_comm.rank_map = ref.rank_map;
+    //   return_comm.ep_comm_ptr = ref.ep_comm_ptr;
+    // }
+  };
+
+
+  class MPI_Info
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_info;
+      #elif _openmpi
+      void * mpi_info;
+      #endif
+
+      MPI_Info(){ }
+      
+      #ifdef _intelmpi
+      MPI_Info(int info): mpi_info(info) {}
+      #elif _openmpi
+      MPI_Info(void* info): mpi_info(info) {}
+      #endif
+  };
+
+
+  class MPI_Request
+  {
+    public:
+
+      #ifdef _intelmpi
+      int mpi_request;
+      #elif _openmpi
+      void * mpi_request;
+      #endif
+
+      int type;	//! type of the non-blocking communication. 1: Isend; 2:Irecv; 3:Imrecv; 4:Issend
+      void* buf;
+
+      int ep_src;
+      int ep_tag;
+      #ifdef _intelmpi
+      int ep_datatype;
+      #elif _openmpi
+      void * ep_datatype;
+      #endif
+
+      MPI_Comm comm;	//! EP communicator related to the communication
+
+      MPI_Request() {}
+
+      #ifdef _intelmpi
+      MPI_Request(int request): mpi_request(request) {}
+      #elif _openmpi
+      MPI_Request(void* request): mpi_request(request) {}
+      #endif
+  };
+
+  
+  class MPI_Aint
+  {
+    public:
+
+    unsigned long mpi_aint;
+
+    MPI_Aint() {}
+    MPI_Aint(int a): mpi_aint(a) {}
+  };
+
+  class MPI_Fint
+  {
+    public:
+
+    int mpi_fint;
+
+    MPI_Fint() {}
+    MPI_Fint(int f): mpi_fint(f) {}
+    
+  };
+
+
+  static MPI_Comm *passage;
+
+  static int TAG = 40000;
+
+  static std::list<std::pair<std::pair<int, int>, MPI_Comm * > > tag_list;
+
+  static std::map<std::pair<int, int>, MPI_Comm >  fc_comm_map;
+
+  static std::map<std::pair<int, int>, MPI_Comm >  *fc_comm_map_ptr;
+  #pragma omp threadprivate(fc_comm_map_ptr)
+            //    <MPI_Fint,thread_num>   EP_Comm
+
+}
+
+
+
+#endif // EP_TYPE_HPP_INCLUDED
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1134)
@@ -0,0 +1,142 @@
+/*!
+   \file ep_wait.cpp
+   \since 2 may 2016
+
+   \brief Definitions of MPI wait function: MPI_Wait, MPI_Waitall, MPI_Waitsome, MPI_Waitany
+ */
+
+#include "ep_lib.hpp"
+#include <mpi.h>
+#include "ep_declaration.hpp"
+
+using namespace std;
+
+
+
+namespace ep_lib {	
+
+	int MPI_Wait(MPI_Request *request, MPI_Status *status)
+	{
+
+    if(request->type == 1)
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
+
+
+      
+      status->mpi_status = &mpi_status;
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      return 0;
+    }
+
+    if(request->type == 2)
+    {
+      int flag = false;
+      MPI_Message message;
+
+      while(!flag)
+      {
+        Message_Check(request->comm);
+        #pragma omp flush
+        MPI_Improbe(request->ep_src, request->ep_tag, request->comm, &flag, &message, status);
+      }
+
+      int count;
+      MPI_Get_count(status, request->ep_datatype, &count);
+      MPI_Mrecv(request->buf, count, request->ep_datatype, &message, status);
+      status->ep_datatype = request->ep_datatype;
+
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+
+      return 0;
+    }
+
+    if(request->type == 3)
+    {
+      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Status mpi_status;
+      ::MPI_Wait(&mpi_request, &mpi_status);
+      
+      status->mpi_status = new ::MPI_Status(mpi_status);
+      status->ep_src = request->ep_src;
+      status->ep_tag = request->ep_tag;
+      status->ep_datatype = request->ep_datatype;
+
+      //int count;
+      //MPI_Get_count(status, request->ep_datatype, &count);
+      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
+    }
+	  return MPI_SUCCESS;
+	}
+
+
+
+	
+
+
+	int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses)
+	{
+    int dest_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &dest_rank);
+    
+    int finished = 0;
+    bool finished_index[count];
+
+	  for(int i=0; i<count; i++)
+    {
+      finished_index[i] = false;
+    }
+
+    while(finished < count)
+    {
+      for(int i=0; i<count; i++)
+      {
+        if(finished_index[i] == false) // this request has not been tested.
+        {
+          if(array_of_requests[i].type != 2) // isend or imrecv
+          {      
+            MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
+            if(array_of_requests[i].type == 3)
+            {
+              //int check_count;
+              //MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &check_count);
+              //check_sum_recv(array_of_requests[i].buf, count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+            }
+            finished++;
+            finished_index[i] = true;
+          }
+          else // irecv
+          {
+            int flag = false;
+            MPI_Message message;
+            
+            MPI_Improbe(array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, &flag, &message, &array_of_statuses[i]);
+            
+            if(flag)
+            {
+              //printf("dest_rank = %d, Waiting one message with src = %d, tag = %d, buf = %p\n", dest_rank, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].buf);
+              int recv_count;
+              MPI_Get_count(&array_of_statuses[i], array_of_requests[i].ep_datatype, &recv_count);
+              MPI_Mrecv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, &message, &array_of_statuses[i]);
+              //check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
+
+              finished++;
+              finished_index[i] = true;
+            }
+          }
+        }
+      }    
+    }
+	  return MPI_SUCCESS;
+	}
+
+
+
+
+
+}
Index: /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_atmosphere.xml
===================================================================
--- /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_atmosphere.xml	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_atmosphere.xml	(revision 1134)
@@ -8,5 +8,5 @@
   </field_definition>
 
-  <file_definition type="multiple_file" par_access="collective" output_freq="6h" sync_freq="6h" output_level="10" enabled=".TRUE.">
+  <file_definition type="one_file" par_access="collective" output_freq="6h" sync_freq="6h" output_level="10" enabled=".TRUE.">
     <file id="output_atmosphere" name="output_atmosphere">
       <field field_ref="field_A_atm" />
Index: /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_surface.xml
===================================================================
--- /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_surface.xml	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/inputs/COMPLETE/context_surface.xml	(revision 1134)
@@ -12,5 +12,5 @@
   </field_definition>
 
-  <file_definition type="multiple_file" par_access="collective" output_level="10" enabled=".TRUE.">
+  <file_definition type="one_file" par_access="collective" output_level="10" enabled=".TRUE.">
     <file id="output_surface" name="output_surface_6h" output_freq="6h">
       <field field_ref="field_A_srf">
Index: /XIOS/dev/branch_yushan_merged/inputs/iodef.xml
===================================================================
--- /XIOS/dev/branch_yushan_merged/inputs/iodef.xml	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/inputs/iodef.xml	(revision 1134)
@@ -12,5 +12,5 @@
 
 
-   <file_definition type="multiple_file" par_access="collective" output_freq="6h" output_level="10" enabled=".TRUE.">
+   <file_definition type="one_file" par_access="collective" output_freq="6h" output_level="10" enabled=".TRUE.">
      <file id="output" name="output">
         <field field_ref="field_A_zoom" name="field_A" />
Index: /XIOS/dev/branch_yushan_merged/src/array_new.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/array_new.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/array_new.hpp	(revision 1134)
@@ -554,5 +554,5 @@
         TinyVector<int,N_rank> vect;
         size_t ne;
-        
+
         ret =  buffer.get(numDim);
         ret &= buffer.get(vect.data(), N_rank);
Index: /XIOS/dev/branch_yushan_merged/src/attribute.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/attribute.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/attribute.cpp	(revision 1134)
@@ -54,4 +54,5 @@
       ///--------------------------------------------------------------
 
+
       CMessage& operator<<(CMessage& msg,CAttribute& type)
       {
Index: /XIOS/dev/branch_yushan_merged/src/attribute_enum.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/attribute_enum.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/attribute_enum.hpp	(revision 1134)
@@ -14,75 +14,76 @@
 namespace xios
 {
-      /// ////////////////////// Déclarations ////////////////////// ///
-        /*!
-        \class CAttributeEnum
-        This class implements the attribute representing enumeration
+    /// ////////////////////// DÃ©clarations ////////////////////// ///
+    /*!
+      \class CAttributeEnum
+      This class implements the attribute representing enumeration
       */
-      template <class T>
-         class CAttributeEnum : public CAttribute, public CEnum<T>
-      {
+    template <class T>
+        class CAttributeEnum : public CAttribute, public CEnum<T>
+    {
         typedef typename T::t_enum T_enum ;
         public :
 
-            /// Constructeurs ///
-            explicit CAttributeEnum(const StdString & id);
-            CAttributeEnum(const StdString & id,
-                               xios_map<StdString, CAttribute*> & umap);
-            CAttributeEnum(const StdString & id, const T_enum & value);
-            CAttributeEnum(const StdString & id, const T_enum & value,
-                               xios_map<StdString, CAttribute*> & umap);
+        /// Constructeurs ///
+        explicit CAttributeEnum(const StdString & id);
+        CAttributeEnum(const StdString & id,
+                xios_map<StdString, CAttribute*> & umap);
+        CAttributeEnum(const StdString & id, const T_enum & value);
+        CAttributeEnum(const StdString & id, const T_enum & value,
+                xios_map<StdString, CAttribute*> & umap);
 
-            /// Accesseur ///
-            T_enum getValue(void) const;
-            string getStringValue(void) const;
+        /// Accesseur ///
+        T_enum getValue(void) const;
+        string getStringValue(void) const;
 
 
-            /// Mutateurs ///
-            void setValue(const T_enum & value);
-            
-            void set(const CAttribute& attr) ;
-            void set(const CAttributeEnum& attr) ;
-            void reset(void);
-            
-            void setInheritedValue(const CAttributeEnum& attr );
-            void setInheritedValue(const CAttribute& attr );
-            T_enum getInheritedValue(void)  const;
-            string getInheritedStringValue(void) const;
-            bool hasInheritedValue(void) const;          
-          
-            bool isEqual(const CAttributeEnum& attr );
-            bool isEqual(const CAttribute& attr );
+        /// Mutateurs ///
+        void setValue(const T_enum & value);
 
-            /// Destructeur ///
-            virtual ~CAttributeEnum(void) { }
+        void set(const CAttribute& attr) ;
+        void set(const CAttributeEnum& attr) ;
+        void reset(void);
 
-            /// Operateur ///
-            CAttributeEnum& operator=(const T_enum & value);
+        void setInheritedValue(const CAttributeEnum& attr );
+        void setInheritedValue(const CAttribute& attr );
+        T_enum getInheritedValue(void)  const;
+        string getInheritedStringValue(void) const;
+        bool hasInheritedValue(void) const;          
 
-            /// Autre ///
-            virtual StdString toString(void) const { return _toString();}
-            virtual void fromString(const StdString & str) { if (str==resetInheritanceStr) { reset(); _canInherite=false ;}  else _fromString(str);}
+        bool isEqual(const CAttributeEnum& attr );
+        bool isEqual(const CAttribute& attr );
 
-            virtual bool toBuffer  (CBufferOut& buffer) const { return _toBuffer(buffer);} 
-            virtual bool fromBuffer(CBufferIn& buffer) { return _fromBuffer(buffer); } 
-            
-            virtual void generateCInterface(ostream& oss,const string& className) ;
-            virtual void generateFortran2003Interface(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceDeclaration_(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceBody_(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceDeclaration(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceGetBody_(ostream& oss,const string& className) ;
-            virtual void generateFortranInterfaceGetDeclaration(ostream& oss,const string& className) ;      
+        /// Destructeur ///
+        virtual ~CAttributeEnum(void) { }
 
-         private :
-          StdString _toString(void) const;
-          void _fromString(const StdString & str);
-          bool _toBuffer  (CBufferOut& buffer) const;
-          bool _fromBuffer(CBufferIn& buffer) ;
-          CEnum<T> inheritedValue ;
-      }; // class CAttributeEnum    
-   
+        /// Operateur ///
+        CAttributeEnum& operator=(const T_enum & value);
+
+        /// Autre ///
+        virtual StdString toString(void) const { return _toString();}
+        virtual void fromString(const StdString & str) { if (str==resetInheritanceStr) { reset(); _canInherite=false ;}  else _fromString(str);}
+
+        virtual bool toBuffer  (CBufferOut& buffer) const { return _toBuffer(buffer);} 
+        virtual bool fromBuffer(CBufferIn& buffer) { return _fromBuffer(buffer); } 
+
+        virtual void generateCInterface(ostream& oss,const string& className) ;
+        virtual void generateFortran2003Interface(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceDeclaration_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceBody_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceDeclaration(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetBody_(ostream& oss,const string& className) ;
+        virtual void generateFortranInterfaceGetDeclaration(ostream& oss,const string& className) ;      
+
+        private :
+        StdString _toString(void) const;
+        void _fromString(const StdString & str);
+        bool _toBuffer  (CBufferOut& buffer) const;
+        bool _fromBuffer(CBufferIn& buffer) ;
+        CEnum<T> inheritedValue ;
+    }; // class CAttributeEnum    
+
 } // namespace xios
 
 #endif // __XIOS_ATTRIBUTE_ENUM__
+
Index: /XIOS/dev/branch_yushan_merged/src/attribute_enum_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/attribute_enum_impl.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/attribute_enum_impl.hpp	(revision 1134)
@@ -10,5 +10,5 @@
 namespace xios
 {
-  /// ////////////////////// Définitions ////////////////////// ///
+  /// ////////////////////// DÃ©finitions ////////////////////// ///
   template <class T>
   CAttributeEnum<T>::CAttributeEnum(const StdString & id)
@@ -30,5 +30,5 @@
      umap.insert(umap.end(), std::make_pair(id, this));
   }
-
+ 
   template <class T>
   CAttributeEnum<T>::CAttributeEnum
@@ -40,5 +40,5 @@
      umap.insert(umap.end(), std::make_pair(id, this));
   }
-
+ 
   ///--------------------------------------------------------------
   template <class T>
@@ -54,10 +54,11 @@
      return CEnum<T>::get();
   }
-
+ 
   template <class T>
   string CAttributeEnum<T>::getStringValue(void) const
   {
-     return CEnum<T>::toString();
-  }
+    return CEnum<T>::toString();
+  }
+
 
   template <class T>
@@ -70,143 +71,144 @@
   void CAttributeEnum<T>::set(const CAttribute& attr)
   {
-    this->set(dynamic_cast<const CAttributeEnum<T>& >(attr));
-  }
-
- template <class T>
+     this->set(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
+   
+  template <class T>
   void CAttributeEnum<T>::set(const CAttributeEnum& attr)
   {
-    CEnum<T>::set(attr);
-  }
-
+     CEnum<T>::set(attr);
+  }
+ 
   template <class T>
   void CAttributeEnum<T>::setInheritedValue(const CAttribute& attr)
   {
-    this->setInheritedValue(dynamic_cast<const CAttributeEnum<T>& >(attr));
-  }
-
+     this->setInheritedValue(dynamic_cast<const CAttributeEnum<T>& >(attr));
+  }
+  
   template <class T>
   void CAttributeEnum<T>::setInheritedValue(const CAttributeEnum& attr)
   {
-    if (this->isEmpty() && _canInherite && attr.hasInheritedValue()) inheritedValue.set(attr.getInheritedValue());
-  }
-
+     if (this->isEmpty() && _canInherite && attr.hasInheritedValue()) inheritedValue.set(attr.getInheritedValue());
+  }
+  
   template <class T>
   typename T::t_enum CAttributeEnum<T>::getInheritedValue(void) const
   {
-    if (this->isEmpty()) return inheritedValue.get();
-    else return getValue();
-  }
-
-  template <class T>
-  string CAttributeEnum<T>::getInheritedStringValue(void) const
-  {
-     if (this->isEmpty()) return inheritedValue.toString();
-     else return CEnum<T>::toString();;
-  }
-
-  template <class T>
-  bool CAttributeEnum<T>::hasInheritedValue(void) const
-  {
-    return !this->isEmpty() || !inheritedValue.isEmpty();
-  }
-
-  template <class T>
-  bool CAttributeEnum<T>::isEqual(const CAttribute& attr)
-  {
-    return (this->isEqual(dynamic_cast<const CAttributeEnum<T>& >(attr)));
-  }
-
-  template <class T>
-  bool CAttributeEnum<T>::isEqual(const CAttributeEnum& attr)
-  {
-    return ((dynamic_cast<const CEnum<T>& >(*this)) == (dynamic_cast<const CEnum<T>& >(attr)));
-  }
+     if (this->isEmpty()) return inheritedValue.get();
+     else return getValue();
+  }
+
+  template <class T>
+      string CAttributeEnum<T>::getInheritedStringValue(void) const
+      {
+          if (this->isEmpty()) return inheritedValue.toString();
+          else return CEnum<T>::toString();;
+      }
+
+  template <class T>
+      bool CAttributeEnum<T>::hasInheritedValue(void) const
+      {
+          return !this->isEmpty() || !inheritedValue.isEmpty();
+      }
+
+  template <class T>
+      bool CAttributeEnum<T>::isEqual(const CAttribute& attr)
+      {
+          return (this->isEqual(dynamic_cast<const CAttributeEnum<T>& >(attr)));
+      }
+
+  template <class T>
+      bool CAttributeEnum<T>::isEqual(const CAttributeEnum& attr)
+      {
+          return ((dynamic_cast<const CEnum<T>& >(*this)) == (dynamic_cast<const CEnum<T>& >(attr)));
+      }
 
   //---------------------------------------------------------------
 
   template <class T>
-  CAttributeEnum<T>& CAttributeEnum<T>::operator=(const T_enum & value)
-  {
-     this->setValue(value);
-     return *this;
-  }
+      CAttributeEnum<T>& CAttributeEnum<T>::operator=(const T_enum & value)
+      {
+          this->setValue(value);
+          return *this;
+      }
 
   //---------------------------------------------------------------
 
   template <class T>
-  StdString CAttributeEnum<T>::_toString(void) const
-  {
-     StdOStringStream oss;
-     if (!CEnum<T>::isEmpty() && this->hasId())
-        oss << this->getName() << "=\"" << CEnum<T>::toString() << "\"";
-     return (oss.str());
-  }
-
-  template <class T>
-  void CAttributeEnum<T>::_fromString(const StdString & str)
-  {
-    CEnum<T>::fromString(str);
-  }
-
-  template <class T>
-  bool CAttributeEnum<T>::_toBuffer (CBufferOut& buffer) const
-  {
-     return CEnum<T>::toBuffer(buffer);
-  }
-
-  template <class T>
-  bool CAttributeEnum<T>::_fromBuffer(CBufferIn& buffer)
-  {
-    return CEnum<T>::fromBuffer(buffer);
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateCInterface(ostream& oss,const string& className)
-  {
-    CInterface::AttributeCInterface<CEnumBase>(oss, className, this->getName());
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortran2003Interface(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortran2003Interface<string>(oss, className, this->getName());
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceDeclaration_(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName()+"_");
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceBody_(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceBody<string>(oss, className, this->getName());
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceDeclaration(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName());
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName()+"_");
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceGetBody_(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceGetBody<string>(oss, className, this->getName());
-  }
-
-  template <typename T>
-  void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration(ostream& oss,const string& className)
-  {
-    CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName());
-  }
+      StdString CAttributeEnum<T>::_toString(void) const
+      {
+          StdOStringStream oss;
+          if (!CEnum<T>::isEmpty() && this->hasId())
+              oss << this->getName() << "=\"" << CEnum<T>::toString() << "\"";
+          return (oss.str());
+      }
+
+  template <class T>
+      void CAttributeEnum<T>::_fromString(const StdString & str)
+      {
+          CEnum<T>::fromString(str);
+      }
+
+  template <class T>
+      bool CAttributeEnum<T>::_toBuffer (CBufferOut& buffer) const
+      {
+          return CEnum<T>::toBuffer(buffer);
+      }
+
+  template <class T>
+      bool CAttributeEnum<T>::_fromBuffer(CBufferIn& buffer)
+      {
+          return CEnum<T>::fromBuffer(buffer);
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateCInterface(ostream& oss,const string& className)
+      {
+          CInterface::AttributeCInterface<CEnumBase>(oss, className, this->getName());
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortran2003Interface(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortran2003Interface<string>(oss, className, this->getName());
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceDeclaration_(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName()+"_");
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceBody_(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceBody<string>(oss, className, this->getName());
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceDeclaration(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceDeclaration<string>(oss, className, this->getName());
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration_(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName()+"_");
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceGetBody_(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceGetBody<string>(oss, className, this->getName());
+      }
+
+  template <typename T>
+      void CAttributeEnum<T>::generateFortranInterfaceGetDeclaration(ostream& oss,const string& className)
+      {
+          CInterface::AttributeFortranInterfaceGetDeclaration<string>(oss, className, this->getName());
+      }
 } // namespace xios
 
 #endif // __XIOS_ATTRIBUTE_ENUM_IMPL_HPP__
+
Index: /XIOS/dev/branch_yushan_merged/src/attribute_map.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/attribute_map.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/attribute_map.hpp	(revision 1134)
@@ -76,4 +76,5 @@
             /// PropriÃ©tÃ© statique ///
             static CAttributeMap * Current;
+            #pragma omp threadprivate (Current)
 
       };  // class CAttributeMap
Index: /XIOS/dev/branch_yushan_merged/src/buffer_client.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/buffer_client.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/buffer_client.cpp	(revision 1134)
@@ -25,4 +25,5 @@
     buffer[1] = new char[bufferSize];
     retBuffer = new CBufferOut(buffer[current], bufferSize);
+    #pragma omp critical (_output)
     info(10) << "CClientBuffer: allocated 2 x " << bufferSize << " bytes for server " << serverRank << " with a maximum of " << maxBufferedEvents << " buffered events" << endl;
   }
Index: /XIOS/dev/branch_yushan_merged/src/buffer_client.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/buffer_client.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/buffer_client.hpp	(revision 1134)
@@ -6,4 +6,7 @@
 #include "mpi.hpp"
 #include "cxios.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace xios
@@ -13,4 +16,5 @@
     public:
       static size_t maxRequestSize;
+      #pragma omp threadprivate(maxRequestSize)
 
       CClientBuffer(MPI_Comm intercomm, int serverRank, StdSize bufferSize, StdSize maxBufferedEvents);
Index: /XIOS/dev/branch_yushan_merged/src/buffer_server.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/buffer_server.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/buffer_server.hpp	(revision 1134)
@@ -4,5 +4,5 @@
 #include "xios_spl.hpp"
 #include "buffer.hpp"
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "cxios.hpp"
 
Index: /XIOS/dev/branch_yushan_merged/src/calendar.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/calendar.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/calendar.cpp	(revision 1134)
@@ -117,5 +117,6 @@
       const CDate& CCalendar::update(int step)
       {
-        info(20) << "update step : " << step << " timestep " << this->timestep << std::endl;
+        #pragma omp critical (_output)
+        info(80)<< "update step : " << step << " timestep " << this->timestep << std::endl;
         return (this->currentDate = this->getInitDate() + step * this->timestep);
       }
Index: /XIOS/dev/branch_yushan_merged/src/client.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client.cpp	(revision 1134)
@@ -11,11 +11,15 @@
 #include "timer.hpp"
 #include "buffer_client.hpp"
+#include "log.hpp"
+
 
 namespace xios
 {
+    extern int test_omp_rank;
+    #pragma omp threadprivate(test_omp_rank)
 
     MPI_Comm CClient::intraComm ;
     MPI_Comm CClient::interComm ;
-    std::list<MPI_Comm> CClient::contextInterComms;
+    std::list<MPI_Comm> *CClient::contextInterComms_ptr = 0;
     int CClient::serverLeader ;
     bool CClient::is_MPI_Initialized ;
@@ -24,5 +28,7 @@
     StdOFStream CClient::m_errorStream;
 
-    void CClient::initialize(const string& codeId,MPI_Comm& localComm,MPI_Comm& returnComm)
+    StdOFStream CClient::array_infoStream[10];
+
+    void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
     {
       int initialized ;
@@ -35,9 +41,13 @@
       {
 // localComm doesn't given
+
         if (localComm == MPI_COMM_NULL)
         {
           if (!is_MPI_Initialized)
           {
-            MPI_Init(NULL, NULL);
+            //MPI_Init(NULL, NULL);
+            int return_level;
+            MPI_Init_thread(NULL, NULL, 3, &return_level);
+            assert(return_level == 3);
           }
           CTimer::get("XIOS").resume() ;
@@ -51,8 +61,8 @@
           int myColor ;
           int i,c ;
-          MPI_Comm newComm ;
-
-          MPI_Comm_size(CXios::globalComm,&size) ;
+
+          MPI_Comm_size(CXios::globalComm,&size);
           MPI_Comm_rank(CXios::globalComm,&rank);
+       
 
           hashAll=new unsigned long[size] ;
@@ -96,7 +106,16 @@
             MPI_Comm_size(intraComm,&intraCommSize) ;
             MPI_Comm_rank(intraComm,&intraCommRank) ;
-            info(50)<<"intercommCreate::client "<<rank<<" intraCommSize : "<<intraCommSize
-                 <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< serverLeader<<endl ;
+            
+            #pragma omp critical(_output)
+            {
+              info(10)<<"intercommCreate::client "<<test_omp_rank<< " "<< &test_omp_rank <<" intraCommSize : "<<intraCommSize
+                 <<" intraCommRank :"<<intraCommRank<<"  serverLeader "<< serverLeader
+                 <<" globalComm : "<< &(CXios::globalComm) << endl ;  
+            }
+
+            
+            
             MPI_Intercomm_create(intraComm,0,CXios::globalComm,serverLeader,0,&interComm) ;
+
           }
           else
@@ -148,4 +167,5 @@
 
       MPI_Comm_dup(intraComm,&returnComm) ;
+
     }
 
@@ -154,5 +174,9 @@
     {
       CContext::setCurrent(id) ;
-      CContext* context=CContext::create(id);
+      CContext* context = CContext::create(id);
+
+      int tmp_rank;
+      MPI_Comm_rank(contextComm,&tmp_rank) ;
+      
       StdString idServer(id);
       idServer += "_server";
@@ -161,6 +185,6 @@
       {
         int size,rank,globalRank ;
-        size_t message_size ;
-        int leaderRank ;
+        //size_t message_size ;
+        //int leaderRank ;
         MPI_Comm contextInterComm ;
 
@@ -173,5 +197,5 @@
         CMessage msg ;
         msg<<idServer<<size<<globalRank ;
-//        msg<<id<<size<<globalRank ;
+
 
         int messageSize=msg.size() ;
@@ -184,5 +208,8 @@
 
         MPI_Intercomm_create(contextComm,0,CXios::globalComm,serverLeader,10+globalRank,&contextInterComm) ;
-        info(10)<<"Register new Context : "<<id<<endl ;
+        
+        #pragma omp critical(_output)
+        info(10)<<" RANK "<< tmp_rank<<" Register new Context : "<<id<<endl ;
+
 
         MPI_Comm inter ;
@@ -190,7 +217,11 @@
         MPI_Barrier(inter) ;
 
+        
         context->initClient(contextComm,contextInterComm) ;
 
-        contextInterComms.push_back(contextInterComm);
+        
+        if(contextInterComms_ptr == NULL) contextInterComms_ptr = new std::list<MPI_Comm>;
+        contextInterComms_ptr->push_back(contextInterComm);
+        
         MPI_Comm_free(&inter);
       }
@@ -209,6 +240,8 @@
         // Finally, we should return current context to context client
         CContext::setCurrent(id);
-
-        contextInterComms.push_back(contextInterComm);
+        
+        if(contextInterComms_ptr == NULL) contextInterComms_ptr = new std::list<MPI_Comm>;
+        contextInterComms_ptr->push_back(contextInterComm);
+
       }
     }
@@ -220,5 +253,5 @@
 
       MPI_Comm_rank(intraComm,&rank) ;
- 
+
       if (!CXios::isServer)
       {
@@ -230,6 +263,7 @@
       }
 
-      for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
+      for (std::list<MPI_Comm>::iterator it = contextInterComms_ptr->begin(); it != contextInterComms_ptr->end(); ++it)
         MPI_Comm_free(&(*it));
+      
       MPI_Comm_free(&interComm);
       MPI_Comm_free(&intraComm);
@@ -241,15 +275,21 @@
       {
         if (CXios::usingOasis) oasis_finalize();
-        else MPI_Finalize() ;
-      }
-      
-      info(20) << "Client side context is finalized"<<endl ;
-      report(0) <<" Performance report : total time spent for XIOS : "<< CTimer::get("XIOS").getCumulatedTime()<<" s"<<endl ;
-      report(0)<< " Performance report : time spent for waiting free buffer : "<< CTimer::get("Blocking time").getCumulatedTime()<<" s"<<endl ;
-      report(0)<< " Performance report : Ratio : "<< CTimer::get("Blocking time").getCumulatedTime()/CTimer::get("XIOS").getCumulatedTime()*100.<<" %"<<endl ;
-      report(0)<< " Performance report : This ratio must be close to zero. Otherwise it may be usefull to increase buffer size or numbers of server"<<endl ;
-//      report(0)<< " Memory report : Current buffer_size : "<<CXios::bufferSize<<endl ;
-      report(0)<< " Memory report : Minimum buffer size required : " << CClientBuffer::maxRequestSize << " bytes" << endl ;
-      report(0)<< " Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
+        else MPI_Finalize();
+      }
+      
+      #pragma omp critical (_output)
+      info(20) << "Client "<<rank<<" : Client side context is finalized "<< endl ;
+
+  /*    #pragma omp critical (_output)
+      {
+         report(0) <<"     Performance report : total time spent for XIOS : "<< CTimer::get("XIOS").getCumulatedTime()<<" s"<<endl ;
+         report(0)<< "     Performance report : time spent for waiting free buffer : "<< CTimer::get("Blocking time").getCumulatedTime()<<" s"<<endl ;
+         report(0)<< "     Performance report : Ratio : "<< CTimer::get("Blocking time").getCumulatedTime()/CTimer::get("XIOS").getCumulatedTime()*100.<<" %"<<endl ;
+         report(0)<< "     Performance report : This ratio must be close to zero. Otherwise it may be usefull to increase buffer size or numbers of server"<<endl ;
+         report(0)<< "     Memory report : Current buffer_size : "<<CXios::bufferSize<<endl ;
+         report(0)<< "     Memory report : Minimum buffer size required : " << CClientBuffer::maxRequestSize << " bytes" << endl ;
+         report(0)<< "     Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
+       }      
+*/
    }
 
@@ -280,8 +320,9 @@
 
       fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << getRank() << ext;
+      
       fb->open(fileNameClient.str().c_str(), std::ios::out);
       if (!fb->is_open())
         ERROR("void CClient::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb)",
-              << std::endl << "Can not open <" << fileNameClient << "> file to write the client log(s).");
+            << std::endl << "Can not open <" << fileNameClient << "> file to write the client log(s).");
     }
 
@@ -294,9 +335,13 @@
     void CClient::openInfoStream(const StdString& fileName)
     {
-      std::filebuf* fb = m_infoStream.rdbuf();
-      openStream(fileName, ".out", fb);
-
-      info.write2File(fb);
-      report.write2File(fb);
+      //std::filebuf* fb = m_infoStream.rdbuf();
+
+      info_FB[omp_get_thread_num()] = array_infoStream[omp_get_thread_num()].rdbuf();
+          
+      openStream(fileName, ".out", info_FB[omp_get_thread_num()]);
+
+      info.write2File(info_FB[omp_get_thread_num()]);
+      report.write2File(info_FB[omp_get_thread_num()]);
+      
     }
 
Index: /XIOS/dev/branch_yushan_merged/src/client.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client.hpp	(revision 1134)
@@ -12,11 +12,22 @@
         static void initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm);
         static void finalize(void);
-        static void registerContext(const string& id, MPI_Comm contextComm);
+        static void registerContext(const string& id, ep_lib::MPI_Comm contextComm);
 
         static MPI_Comm intraComm;
+        #pragma omp threadprivate(intraComm)
+
         static MPI_Comm interComm;
-        static std::list<MPI_Comm> contextInterComms;
+        #pragma omp threadprivate(interComm)
+
+        //static std::list<MPI_Comm> contextInterComms;
+        
+        static std::list<MPI_Comm> * contextInterComms_ptr;
+        #pragma omp threadprivate(contextInterComms_ptr)
+
         static int serverLeader;
+        #pragma omp threadprivate(serverLeader)
+
         static bool is_MPI_Initialized ;
+        #pragma omp threadprivate(is_MPI_Initialized)
 
         //! Get rank of the current process
@@ -39,6 +50,13 @@
       protected:
         static int rank;
+        #pragma omp threadprivate(rank)
+
         static StdOFStream m_infoStream;
+        #pragma omp threadprivate(m_infoStream) 
+
         static StdOFStream m_errorStream;
+        #pragma omp threadprivate(m_errorStream)
+
+        static StdOFStream array_infoStream[10];
 
         static void openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb);
Index: /XIOS/dev/branch_yushan_merged/src/client_client_dht_template.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client_client_dht_template.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client_client_dht_template.hpp	(revision 1134)
@@ -13,5 +13,5 @@
 #include "xios_spl.hpp"
 #include "array_new.hpp"
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "policy.hpp"
 #include <boost/unordered_map.hpp>
@@ -40,8 +40,8 @@
   public:
     CClientClientDHTTemplate(const Index2InfoTypeMap& indexInfoInitMap,
-                             const MPI_Comm& clientIntraComm);
+                             const ep_lib::MPI_Comm& clientIntraComm);
 
     CClientClientDHTTemplate(const Index2VectorInfoTypeMap& indexInfoInitMap,
-                             const MPI_Comm& clientIntraComm);
+                             const ep_lib::MPI_Comm& clientIntraComm);
 
     void computeIndexInfoMapping(const CArray<size_t,1>& indices);
@@ -55,5 +55,5 @@
 
   protected:
-    CClientClientDHTTemplate(const MPI_Comm& clientIntraComm);
+    CClientClientDHTTemplate(const ep_lib::MPI_Comm& clientIntraComm);
 
   protected:
@@ -62,9 +62,9 @@
     // Redistribute index and info among clients
     void computeDistributedIndex(const Index2InfoTypeMap& indexInfoInitMap,
-                                 const MPI_Comm& intraCommLevel,
+                                 const ep_lib::MPI_Comm& intraCommLevel,
                                  int level);
 
     void computeDistributedIndex(const Index2VectorInfoTypeMap& indexInfoInitMap,
-                                 const MPI_Comm& intraCommLevel,
+                                 const ep_lib::MPI_Comm& intraCommLevel,
                                  int level);
 
@@ -73,5 +73,5 @@
 
     void computeIndexInfoMappingLevel(const CArray<size_t,1>& indices,
-                                      const MPI_Comm& intraCommLevel,
+                                      const ep_lib::MPI_Comm& intraCommLevel,
                                       int level);
 
@@ -85,19 +85,19 @@
     // Send information to clients
     void sendInfoToClients(int clientDestRank, unsigned char* info, int infoSize,
-                           const MPI_Comm& clientIntraComm,
-                           std::vector<MPI_Request>& requestSendInfo);
+                           const ep_lib::MPI_Comm& clientIntraComm,
+                           std::vector<ep_lib::MPI_Request>& requestSendInfo);
 
     void recvInfoFromClients(int clientSrcRank, unsigned char* info, int infoSize,
-                            const MPI_Comm& clientIntraComm,
-                            std::vector<MPI_Request>& requestRecvInfo);
+                            const ep_lib::MPI_Comm& clientIntraComm,
+                            std::vector<ep_lib::MPI_Request>& requestRecvInfo);
 
     // Send global index to clients
     void sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
-                            const MPI_Comm& clientIntraComm,
-                            std::vector<MPI_Request>& requestSendIndexGlobal);
+                            const ep_lib::MPI_Comm& clientIntraComm,
+                            std::vector<ep_lib::MPI_Request>& requestSendIndexGlobal);
 
     void recvIndexFromClients(int clientSrcRank, size_t* indices, size_t indiceSize,
-                             const MPI_Comm& clientIntraComm,
-                             std::vector<MPI_Request>& requestRecvIndex);
+                             const ep_lib::MPI_Comm& clientIntraComm,
+                             std::vector<ep_lib::MPI_Request>& requestRecvIndex);
 
     void sendRecvOnReturn(const std::vector<int>& sendNbRank, std::vector<int>& sendNbElements,
Index: /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp	(revision 1134)
@@ -10,9 +10,13 @@
 #include "utils.hpp"
 #include "mpi_tag.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+
 
 namespace xios
 {
 template<typename T, typename H>
-CClientClientDHTTemplate<T,H>::CClientClientDHTTemplate(const MPI_Comm& clientIntraComm)
+CClientClientDHTTemplate<T,H>::CClientClientDHTTemplate(const ep_lib::MPI_Comm& clientIntraComm)
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
 {
@@ -34,5 +38,5 @@
 template<typename T, typename H>
 CClientClientDHTTemplate<T,H>::CClientClientDHTTemplate(const Index2InfoTypeMap& indexInfoMap,
-                                                        const MPI_Comm& clientIntraComm)
+                                                        const ep_lib::MPI_Comm& clientIntraComm)
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
 {
@@ -59,5 +63,5 @@
 template<typename T, typename H>
 CClientClientDHTTemplate<T,H>::CClientClientDHTTemplate(const Index2VectorInfoTypeMap& indexInfoMap,
-                                                        const MPI_Comm& clientIntraComm)
+                                                        const ep_lib::MPI_Comm& clientIntraComm)
   : H(clientIntraComm), index2InfoMapping_(), indexToInfoMappingLevel_(), nbClient_(0)
 {
@@ -95,5 +99,5 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::computeIndexInfoMappingLevel(const CArray<size_t,1>& indices,
-                                                                 const MPI_Comm& commLevel,
+                                                                 const ep_lib::MPI_Comm& commLevel,
                                                                  int level)
 {
@@ -169,5 +173,5 @@
     recvIndexBuff = new unsigned long[recvNbIndexCount];
 
-  std::vector<MPI_Request> request;
+  std::vector<ep_lib::MPI_Request> request;
   std::vector<int>::iterator itbRecvIndex = recvRankClient.begin(), itRecvIndex,
                              iteRecvIndex = recvRankClient.end(),
@@ -179,5 +183,7 @@
   {
     if (0 != recvNbIndexClientCount[idx])
+    {
       recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+    }
     currentIndex += recvNbIndexClientCount[idx];
   }
@@ -188,5 +194,5 @@
     sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
 
-  std::vector<MPI_Status> status(request.size());
+  std::vector<ep_lib::MPI_Status> status(request.size());
   MPI_Waitall(request.size(), &request[0], &status[0]);
 
@@ -242,5 +248,5 @@
   }
 
-  std::vector<MPI_Request> requestOnReturn;
+  std::vector<ep_lib::MPI_Request> requestOnReturn;
   currentIndex = 0;
   for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
@@ -293,5 +299,5 @@
   }
 
-  std::vector<MPI_Status> statusOnReturn(requestOnReturn.size());
+  std::vector<ep_lib::MPI_Status> statusOnReturn(requestOnReturn.size());
   MPI_Waitall(requestOnReturn.size(), &requestOnReturn[0], &statusOnReturn[0]);
 
@@ -360,5 +366,5 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::computeDistributedIndex(const Index2VectorInfoTypeMap& indexInfoMap,
-                                                            const MPI_Comm& commLevel,
+                                                            const ep_lib::MPI_Comm& commLevel,
                                                             int level)
 {
@@ -412,5 +418,4 @@
     {
       client2ClientIndex[indexClient + groupRankBegin][sendNbIndexBuff[indexClient]] = it->first;;
-  //          ProcessDHTElement<InfoType>::packElement(it->second, client2ClientInfo[indexClient + groupRankBegin], sendNbInfo[indexClient]);
       ProcessDHTElement<InfoType>::packElement(infoTmp[idx], client2ClientInfo[indexClient + groupRankBegin], sendNbInfo[indexClient]);
       ++sendNbIndexBuff[indexClient];
@@ -439,5 +444,5 @@
   // it will send a message to the correct clients.
   // Contents of the message are index and its corresponding informatioin
-  std::vector<MPI_Request> request;
+  std::vector<ep_lib::MPI_Request> request;
   int currentIndex = 0;
   int nbRecvClient = recvRankClient.size();
@@ -458,11 +463,18 @@
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
+  {
     sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+  }
+
   boost::unordered_map<int, unsigned char*>::iterator itbInfo = client2ClientInfo.begin(), itInfo,
                                                       iteInfo = client2ClientInfo.end();
   for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
+  {
     sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
 
-  std::vector<MPI_Status> status(request.size());
+  }
+
+  std::vector<ep_lib::MPI_Status> status(request.size());
+
   MPI_Waitall(request.size(), &request[0], &status[0]);
 
@@ -518,8 +530,8 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::sendIndexToClients(int clientDestRank, size_t* indices, size_t indiceSize,
-                                                       const MPI_Comm& clientIntraComm,
-                                                       std::vector<MPI_Request>& requestSendIndex)
-{
-  MPI_Request request;
+                                                       const ep_lib::MPI_Comm& clientIntraComm,
+                                                       std::vector<ep_lib::MPI_Request>& requestSendIndex)
+{
+  ep_lib::MPI_Request request;
   requestSendIndex.push_back(request);
   MPI_Isend(indices, indiceSize, MPI_UNSIGNED_LONG,
@@ -536,8 +548,8 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::recvIndexFromClients(int clientSrcRank, size_t* indices, size_t indiceSize,
-                                                         const MPI_Comm& clientIntraComm,
-                                                         std::vector<MPI_Request>& requestRecvIndex)
-{
-  MPI_Request request;
+                                                         const ep_lib::MPI_Comm& clientIntraComm,
+                                                         std::vector<ep_lib::MPI_Request>& requestRecvIndex)
+{
+  ep_lib::MPI_Request request;
   requestRecvIndex.push_back(request);
   MPI_Irecv(indices, indiceSize, MPI_UNSIGNED_LONG,
@@ -555,10 +567,9 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::sendInfoToClients(int clientDestRank, unsigned char* info, int infoSize,
-                                                      const MPI_Comm& clientIntraComm,
-                                                      std::vector<MPI_Request>& requestSendInfo)
-{
-  MPI_Request request;
+                                                      const ep_lib::MPI_Comm& clientIntraComm,
+                                                      std::vector<ep_lib::MPI_Request>& requestSendInfo)
+{
+  ep_lib::MPI_Request request;
   requestSendInfo.push_back(request);
-
   MPI_Isend(info, infoSize, MPI_CHAR,
             clientDestRank, MPI_DHT_INFO, clientIntraComm, &(requestSendInfo.back()));
@@ -575,8 +586,8 @@
 template<typename T, typename H>
 void CClientClientDHTTemplate<T,H>::recvInfoFromClients(int clientSrcRank, unsigned char* info, int infoSize,
-                                                        const MPI_Comm& clientIntraComm,
-                                                        std::vector<MPI_Request>& requestRecvInfo)
-{
-  MPI_Request request;
+                                                        const ep_lib::MPI_Comm& clientIntraComm,
+                                                        std::vector<ep_lib::MPI_Request>& requestRecvInfo)
+{
+  ep_lib::MPI_Request request;
   requestRecvInfo.push_back(request);
 
@@ -651,6 +662,6 @@
 {
   recvNbElements.resize(recvNbRank.size());
-  std::vector<MPI_Request> request(sendNbRank.size()+recvNbRank.size());
-  std::vector<MPI_Status> requestStatus(sendNbRank.size()+recvNbRank.size());
+  std::vector<ep_lib::MPI_Request> request(sendNbRank.size()+recvNbRank.size());
+  std::vector<ep_lib::MPI_Status> requestStatus(sendNbRank.size()+recvNbRank.size());
 
   int nRequest = 0;
@@ -696,6 +707,6 @@
   std::vector<int> recvBuff(recvBuffSize*2,0);
 
-  std::vector<MPI_Request> request(sendBuffSize+recvBuffSize);
-  std::vector<MPI_Status> requestStatus(sendBuffSize+recvBuffSize);
+  std::vector<ep_lib::MPI_Request> request(sendBuffSize+recvBuffSize);
+  std::vector<ep_lib::MPI_Status> requestStatus(sendBuffSize+recvBuffSize);
 
   int nRequest = 0;
@@ -721,5 +732,8 @@
   }
 
+  //MPI_Barrier(this->internalComm_);
+
   MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
+
   int nbRecvRank = 0, nbRecvElements = 0;
   recvNbRank.clear();
Index: /XIOS/dev/branch_yushan_merged/src/client_server_mapping.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client_server_mapping.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client_server_mapping.hpp	(revision 1134)
@@ -14,4 +14,8 @@
 #include "mpi.hpp"
 #include <boost/unordered_map.hpp>
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+
 
 namespace xios {
@@ -37,5 +41,5 @@
 
     static std::map<int,int> computeConnectedClients(int nbServer, int nbClient,
-                                                     MPI_Comm& clientIntraComm,
+                                                     ep_lib::MPI_Comm& clientIntraComm,
                                                      const std::vector<int>& connectedServerRank);
 
Index: /XIOS/dev/branch_yushan_merged/src/client_server_mapping_distributed.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client_server_mapping_distributed.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/client_server_mapping_distributed.hpp	(revision 1134)
@@ -35,5 +35,5 @@
     /** Default constructor */
     CClientServerMappingDistributed(const boost::unordered_map<size_t,int>& globalIndexOfServer,
-                                    const MPI_Comm& clientIntraComm,
+                                    const ep_lib::MPI_Comm& clientIntraComm,
                                     bool isDataDistributed = true);
 
Index: /XIOS/dev/branch_yushan_merged/src/context_client.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/context_client.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/context_client.cpp	(revision 1134)
@@ -20,5 +20,5 @@
     \cxtSer [in] cxtSer Pointer to context of server side. (It is only used on case of attached mode)
     */
-    CContextClient::CContextClient(CContext* parent, MPI_Comm intraComm_, MPI_Comm interComm_, CContext* cxtSer)
+    CContextClient::CContextClient(CContext* parent, ep_lib::MPI_Comm intraComm_, ep_lib::MPI_Comm interComm_, CContext* cxtSer)
      : mapBufferSize_(), parentServer(cxtSer), maxBufferedEvents(4)
     {
@@ -291,6 +291,10 @@
        if (ratio < minBufferSizeEventSizeRatio) minBufferSizeEventSizeRatio = ratio;
      }
+     #ifdef _usingMPI
      MPI_Allreduce(MPI_IN_PLACE, &minBufferSizeEventSizeRatio, 1, MPI_DOUBLE, MPI_MIN, intraComm);
-
+     #elif _usingEP
+     MPI_Allreduce(&minBufferSizeEventSizeRatio, &minBufferSizeEventSizeRatio, 1, MPI_DOUBLE, MPI_MIN, intraComm);
+     #endif
+     
      if (minBufferSizeEventSizeRatio < 1.0)
        ERROR("void CContextClient::setBufferSize(const std::map<int,StdSize>& mapSize, const std::map<int,StdSize>& maxEventSize)",
@@ -392,9 +396,9 @@
      for (itMap = itbMap; itMap != iteMap; ++itMap)
      {
-       report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
-                  << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
+       //report(10) << " Memory report : Context <" << context->getId() << "> : client side : memory used for buffer of each connection to server" << endl
+       //           << "  +) To server with rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
        totalBuf += itMap->second;
      }
-     report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
+     //report(0) << " Memory report : Context <" << context->getId() << "> : client side : total memory used for buffer " << totalBuf << " bytes" << endl;
 
      releaseBuffers();
Index: /XIOS/dev/branch_yushan_merged/src/context_client.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/context_client.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/context_client.hpp	(revision 1134)
@@ -27,5 +27,5 @@
     public:
       // Contructor
-      CContextClient(CContext* parent, MPI_Comm intraComm, MPI_Comm interComm, CContext* parentServer = 0);
+      CContextClient(CContext* parent, ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* parentServer = 0);
 
       // Send event to server
@@ -67,7 +67,7 @@
       int serverSize; //!< Size of server group
 
-      MPI_Comm interComm; //!< Communicator of server group
+      ep_lib::MPI_Comm interComm; //!< Communicator of server group
 
-      MPI_Comm intraComm; //!< Communicator of client group
+      ep_lib::MPI_Comm intraComm; //!< Communicator of client group
 
       map<int,CClientBuffer*> buffers; //!< Buffers for connection to servers
Index: /XIOS/dev/branch_yushan_merged/src/context_server.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/context_server.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/context_server.cpp	(revision 1134)
@@ -10,5 +10,5 @@
 #include "file.hpp"
 #include "grid.hpp"
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "tracer.hpp"
 #include "timer.hpp"
@@ -23,5 +23,5 @@
 {
 
-  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
+  CContextServer::CContextServer(CContext* parent, ep_lib::MPI_Comm intraComm_, ep_lib::MPI_Comm interComm_)
   {
     context=parent;
@@ -72,5 +72,5 @@
     int count;
     char * addr;
-    MPI_Status status;
+    ep_lib::MPI_Status status;
     map<int,CServerBuffer*>::iterator it;
 
@@ -80,5 +80,5 @@
       {
         traceOff();
-        MPI_Iprobe(rank,20,interComm,&flag,&status);
+        ep_lib::MPI_Iprobe(rank,20,interComm,&flag,&status);
         traceOn();
         if (flag==true)
@@ -88,5 +88,5 @@
           {
             StdSize buffSize = 0;
-            MPI_Recv(&buffSize, 1, MPI_LONG, rank, 20, interComm, &status);
+            ep_lib::MPI_Recv(&buffSize, 1, MPI_LONG, rank, 20, interComm, &status);
             mapBufferSize_.insert(std::make_pair(rank, buffSize));
             it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(buffSize)))).first;
@@ -94,9 +94,10 @@
           else
           {
-            MPI_Get_count(&status,MPI_CHAR,&count);
+            
+            ep_lib::MPI_Get_count(&status,MPI_CHAR,&count);
             if (it->second->isBufferFree(count))
             {
               addr=(char*)it->second->getBuffer(count);
-              MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
+              ep_lib::MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
               bufferRequest[rank]=addr;
             }
@@ -109,5 +110,5 @@
   void CContextServer::checkPendingRequest(void)
   {
-    map<int,MPI_Request>::iterator it;
+    map<int,ep_lib::MPI_Request>::iterator it;
     list<int> recvRequest;
     list<int>::iterator itRecv;
@@ -115,16 +116,16 @@
     int flag;
     int count;
-    MPI_Status status;
-
-    for(it=pendingRequest.begin();it!=pendingRequest.end();it++)
+    ep_lib::MPI_Status status;
+
+    for(it=pendingRequest.begin();it!=pendingRequest.end();++it)
     {
       rank=it->first;
       traceOff();
-      MPI_Test(& it->second, &flag, &status);
+      ep_lib::MPI_Test(& it->second, &flag, &status);
       traceOn();
       if (flag==true)
       {
         recvRequest.push_back(rank);
-        MPI_Get_count(&status,MPI_CHAR,&count);
+        ep_lib::MPI_Get_count(&status,MPI_CHAR,&count);
         processRequest(rank,bufferRequest[rank],count);
       }
@@ -219,4 +220,5 @@
     {
       finished=true;
+      #pragma omp critical (_output)
       info(20)<<"Server Side context <"<<context->getId()<<"> finalized"<<endl;
       std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
@@ -225,10 +227,10 @@
       for (itMap = itbMap; itMap != iteMap; ++itMap)
       {
-        report(10)<< " Memory report : Context <"<<context->getId()<<"> : server side : memory used for buffer of each connection to client" << endl
-                  << "  +) With client of rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
+        //report(10)<< " Memory report : Context <"<<context->getId()<<"> : server side : memory used for buffer of each connection to client" << endl
+        //          << "  +) With client of rank " << itMap->first << " : " << itMap->second << " bytes " << endl;
         totalBuf += itMap->second;
       }
       context->finalize();
-      report(0)<< " Memory report : Context <"<<context->getId()<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
+      //report(0)<< " Memory report : Context <"<<context->getId()<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
     }
     else if (event.classId==CContext::GetType()) CContext::dispatchEvent(event);
Index: /XIOS/dev/branch_yushan_merged/src/context_server.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/context_server.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/context_server.hpp	(revision 1134)
@@ -14,5 +14,5 @@
     public:
 
-    CContextServer(CContext* parent,MPI_Comm intraComm,MPI_Comm interComm) ;
+    CContextServer(CContext* parent, ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm) ;
     bool eventLoop(bool enableEventsProcessing = true);
     void listen(void) ;
@@ -25,13 +25,13 @@
     bool hasFinished(void);
 
-    MPI_Comm intraComm ;
+    ep_lib::MPI_Comm intraComm ;
     int intraCommSize ;
     int intraCommRank ;
 
-    MPI_Comm interComm ;
+    ep_lib::MPI_Comm interComm ;
     int commSize ;
 
     map<int,CServerBuffer*> buffers ;
-    map<int,MPI_Request> pendingRequest ;
+    map<int,ep_lib::MPI_Request> pendingRequest ;
     map<int,char*> bufferRequest ;
 
Index: /XIOS/dev/branch_yushan_merged/src/cxios.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/cxios.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/cxios.cpp	(revision 1134)
@@ -14,17 +14,30 @@
 namespace xios
 {
-  string CXios::rootFile="./iodef.xml" ;
-  string CXios::xiosCodeId="xios.x" ;
-  string CXios::clientFile="./xios_client";
-  string CXios::serverFile="./xios_server";
+
+  extern int test_omp_rank;
+  #pragma omp threadprivate(test_omp_rank)
+
+  const string CXios::rootFile="./iodef.xml" ;
+  const string CXios::xiosCodeId="xios.x" ;
+  const string CXios::clientFile="./xios_client";
+  const string CXios::serverFile="./xios_server";
+
 
   bool CXios::isClient ;
   bool CXios::isServer ;
+
+
   MPI_Comm CXios::globalComm ;
+
+  
   bool CXios::usingOasis ;
   bool CXios::usingServer = false;
+
+
   double CXios::bufferSizeFactor = 1.0;
   const double CXios::defaultBufferSizeFactor = 1.0;
   StdSize CXios::minBufferSize = 1024 * sizeof(double);
+
+
   bool CXios::printLogs2Files;
   bool CXios::isOptPerformance = true;
@@ -36,5 +49,11 @@
   {
     set_new_handler(noMemory);
-    parseFile(rootFile);
+    
+    
+    #pragma omp critical
+    {
+      parseFile(rootFile);  
+    }
+    #pragma omp barrier
     parseXiosConfig();
   }
@@ -68,5 +87,35 @@
       ERROR("CXios::parseXiosConfig()", "recv_field_timeout cannot be negative.");
 
-    globalComm=MPI_COMM_WORLD ;
+ 
+    int num_ep;
+    if(isClient)  
+    { 
+      num_ep = omp_get_num_threads();
+    }
+    
+    if(isServer) 
+    { 
+      num_ep = omp_get_num_threads();
+    }
+    
+    MPI_Info info;
+    #pragma omp master
+    {
+      MPI_Comm *ep_comm;
+      MPI_Comm_create_endpoints(MPI_COMM_WORLD, num_ep, info, ep_comm);  // servers should reach here too.
+      passage = ep_comm;  
+    }
+    
+    #pragma omp barrier
+
+      
+    CXios::globalComm = passage[omp_get_thread_num()];
+
+    int tmp_rank;
+    MPI_Comm_rank(CXios::globalComm, &tmp_rank);
+
+    
+    test_omp_rank = tmp_rank;
+    
   }
 
@@ -79,9 +128,10 @@
   void CXios::initClientSide(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
   {
+    isClient = true;
+    
     initialize() ;
 
-    isClient = true;
-
     CClient::initialize(codeId,localComm,returnComm) ;
+
     if (CClient::getRank()==0) globalRegistry = new CRegistry(returnComm) ;
 
@@ -92,4 +142,5 @@
     if (printLogs2Files)
     {
+      #pragma omp critical
       CClient::openInfoStream(clientFile);
       CClient::openErrorStream(clientFile);
@@ -107,4 +158,5 @@
      if (CClient::getRank()==0)
      {
+       #pragma omp critical (_output)
        info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
        globalRegistry->toFile("xios_registry.bin") ;
@@ -123,4 +175,15 @@
   void CXios::initServer()
   {
+    int initialized;
+    MPI_Initialized(&initialized);
+    if (initialized) CServer::is_MPI_Initialized=true ;
+    else CServer::is_MPI_Initialized=false ;
+      
+ 
+    if(!CServer::is_MPI_Initialized)
+    {
+      MPI_Init(NULL, NULL);
+    }
+      
     set_new_handler(noMemory);
     std::set<StdString> parseList;
@@ -133,8 +196,11 @@
   void CXios::initServerSide(void)
   {
-    initServer();
+    
     isClient = false;
     isServer = true;
-
+    
+    initServer();
+    
+    
     // Initialize all aspects MPI
     CServer::initialize();
@@ -162,5 +228,7 @@
        delete globalRegistry ;
      }
+
     CServer::finalize();
+        
     CServer::closeInfoStream();
   }
Index: /XIOS/dev/branch_yushan_merged/src/cxios.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/cxios.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/cxios.hpp	(revision 1134)
@@ -5,4 +5,5 @@
 #include "mpi.hpp"
 #include "registry.hpp"
+#include "log.hpp"
 
 namespace xios
@@ -14,37 +15,43 @@
   {
     public:
-     static void initialize(void) ;
-     static void initClientSide(const string & codeId, MPI_Comm& localComm, MPI_Comm& returnComm) ;
-     static void initServerSide(void) ;
-     static void clientFinalize(void) ;
-     static void parseFile(const string& filename) ;
+      static void initialize(void) ;
+      static void initClientSide(const string & codeId, ep_lib::MPI_Comm& localComm, ep_lib::MPI_Comm& returnComm) ;
+      static void initServerSide(void) ;
+      static void clientFinalize(void) ;
+      static void parseFile(const string& filename) ;
 
-     template <typename T>
-     static T getin(const string& id,const T& defaultValue) ;
+      template <typename T>
+      static T getin(const string& id,const T& defaultValue) ;
 
-     template <typename T>
-     static T getin(const string& id) ;
+      template <typename T>
+      static T getin(const string& id) ;
 
     public:
-     static string rootFile ; //!< Configuration filename
-     static string xiosCodeId ; //!< Identity for XIOS
-     static string clientFile; //!< Filename template for client
-     static string serverFile; //!< Filename template for server
+      static const string rootFile; //!< Configuration filename
+      static const string xiosCodeId ; //!< Identity for XIOS
+      static const string clientFile; //!< Filename template for client
+      static const string serverFile; //!< Filename template for server
 
-     static bool isClient ; //!< Check if xios is client
-     static bool isServer ; //!< Check if xios is server
+      static bool isClient ; //!< Check if xios is client
+      static bool isServer ; //!< Check if xios is server
+      #pragma omp threadprivate(isClient, isServer)
 
-     static MPI_Comm globalComm ; //!< Global communicator
+      static MPI_Comm globalComm ; //!< Global communicator
+      #pragma omp threadprivate(globalComm)
 
-     static bool printLogs2Files; //!< Printing out logs into files
-     static bool usingOasis ; //!< Using Oasis
-     static bool usingServer ; //!< Using server (server mode)
-     static double bufferSizeFactor; //!< Factor used to tune the buffer size
-     static const double defaultBufferSizeFactor; //!< Default factor value
-     static StdSize minBufferSize; //!< Minimum buffer size
-     static bool isOptPerformance; //!< Check if buffer size is for performance (as large as possible)
-     static CRegistry* globalRegistry ; //!< global registry which is wrote by the root process of the servers
-     static double recvFieldTimeout; //!< Time to wait for data before issuing an error when receiving a field
-
+      static bool printLogs2Files; //!< Printing out logs into files
+      static bool usingOasis ; //!< Using Oasis
+      static bool usingServer ; //!< Using server (server mode)
+      static double bufferSizeFactor; //!< Factor used to tune the buffer size
+      static const double defaultBufferSizeFactor; //!< Default factor value
+      static StdSize minBufferSize; //!< Minimum buffer size
+      static bool isOptPerformance; //!< Check if buffer size is for performance (as large as possible)
+      #pragma omp threadprivate(printLogs2Files, usingOasis, usingServer, bufferSizeFactor, minBufferSize, isOptPerformance)
+      
+      static CRegistry* globalRegistry ; //!< global registry which is wrote by the root process of the servers
+      static double recvFieldTimeout;
+      #pragma omp threadprivate(recvFieldTimeout)
+      
+      
     public:
      //! Setting xios to use server mode
Index: /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.cpp	(revision 1134)
@@ -22,5 +22,5 @@
 
   CDHTAutoIndexing::CDHTAutoIndexing(const CArray<size_t,1>& hashValue,
-                                     const MPI_Comm& clientIntraComm)
+                                     const ep_lib::MPI_Comm& clientIntraComm)
     : CClientClientDHTTemplate<size_t>(clientIntraComm)
   {
@@ -58,5 +58,5 @@
   */
   CDHTAutoIndexing::CDHTAutoIndexing(Index2VectorInfoTypeMap& hashInitMap,
-                                     const MPI_Comm& clientIntraComm)
+                                     const ep_lib::MPI_Comm& clientIntraComm)
     : CClientClientDHTTemplate<size_t>(clientIntraComm)
   {
Index: /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/dht_auto_indexing.hpp	(revision 1134)
@@ -12,4 +12,7 @@
 
 #include "client_client_dht_template.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace xios
@@ -25,8 +28,8 @@
 
     CDHTAutoIndexing(const CArray<size_t,1>& hashValue,
-                     const MPI_Comm& clientIntraComm);
+                     const ep_lib::MPI_Comm& clientIntraComm);
 
     CDHTAutoIndexing(Index2VectorInfoTypeMap& hashInitMap,
-                     const MPI_Comm& clientIntraComm);
+                     const ep_lib::MPI_Comm& clientIntraComm);
 
     size_t getNbIndexesGlobal() const;
Index: /XIOS/dev/branch_yushan_merged/src/event_scheduler.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/event_scheduler.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/event_scheduler.cpp	(revision 1134)
@@ -132,9 +132,17 @@
     while(received)
     {
+      #ifdef _usingEP
+      MPI_Iprobe(-1,1,communicator,&received, &status) ;
+      #else
       MPI_Iprobe(MPI_ANY_SOURCE,1,communicator,&received, &status) ;
+      #endif
       if (received)
       {
         recvRequest=new SPendingRequest ;
+        #ifdef _usingEP
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -1, 1, communicator, &(recvRequest->request)) ;
+        #else
         MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator, &(recvRequest->request)) ;
+        #endif
         pendingRecvParentRequest.push(recvRequest) ;
       }
@@ -174,9 +182,17 @@
     while(received)
     {
+      #ifdef _usingEP
+      MPI_Iprobe(-1,0,communicator,&received, &status) ;
+      #else
       MPI_Iprobe(MPI_ANY_SOURCE,0,communicator,&received, &status) ;
+      #endif
       if (received)
       {
         recvRequest=new SPendingRequest ;
+        #ifdef _usingEP
+        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, -1, 0, communicator, &recvRequest->request) ;
+        #else
         MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator, &recvRequest->request) ;
+        #endif
         pendingRecvChildRequest.push_back(recvRequest) ;
       }
Index: /XIOS/dev/branch_yushan_merged/src/event_scheduler.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/event_scheduler.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/event_scheduler.hpp	(revision 1134)
@@ -4,4 +4,8 @@
 #include "xios_spl.hpp"
 #include "mpi.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+
 
 namespace xios
Index: /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.cpp	(revision 1134)
@@ -65,5 +65,5 @@
   }
 
-  std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > CSpatialTransformFilterEngine::engines;
+  std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *CSpatialTransformFilterEngine::engines_ptr = 0;
 
   CSpatialTransformFilterEngine* CSpatialTransformFilterEngine::get(CGridTransformation* gridTransformation)
@@ -73,9 +73,11 @@
             "Impossible to get the requested engine, the grid transformation is invalid.");
 
-    std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines.find(gridTransformation);
-    if (it == engines.end())
+    if(engines_ptr == NULL) engines_ptr = new std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >;
+
+    std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines_ptr->find(gridTransformation);
+    if (it == engines_ptr->end())
     {
       boost::shared_ptr<CSpatialTransformFilterEngine> engine(new CSpatialTransformFilterEngine(gridTransformation));
-      it = engines.insert(std::make_pair(gridTransformation, engine)).first;
+      it = engines_ptr->insert(std::make_pair(gridTransformation, engine)).first;
     }
 
@@ -153,5 +155,5 @@
 
       idxSendBuff = 0;
-      std::vector<MPI_Request> sendRecvRequest;
+      std::vector<ep_lib::MPI_Request> sendRecvRequest;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
       {
@@ -163,5 +165,5 @@
           sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
         }
-        sendRecvRequest.push_back(MPI_Request());
+        sendRecvRequest.push_back(ep_lib::MPI_Request());
         MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest.back());
       }
@@ -181,9 +183,9 @@
         int srcRank = itRecv->first;
         int countSize = itRecv->second.size();
-        sendRecvRequest.push_back(MPI_Request());
+        sendRecvRequest.push_back(ep_lib::MPI_Request());
         MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest.back());
         currentBuff += countSize;
       }
-      std::vector<MPI_Status> status(sendRecvRequest.size());
+      std::vector<ep_lib::MPI_Status> status(sendRecvRequest.size());
       MPI_Waitall(sendRecvRequest.size(), &sendRecvRequest[0], &status[0]);
 
Index: /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.hpp	(revision 1134)
@@ -104,5 +104,8 @@
 
       //! The allocated engines
-      static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > engines;
+      //static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > engines;
+      static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *engines_ptr;
+      #pragma omp threadprivate(engines_ptr)
+      
   }; // class CSpatialTransformFilterEngine
 } // namespace xios
Index: /XIOS/dev/branch_yushan_merged/src/group_factory.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/group_factory.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/group_factory.cpp	(revision 1134)
@@ -4,14 +4,15 @@
 {
    /// ////////////////////// DÃ©finitions ////////////////////// ///
-   StdString CGroupFactory::CurrContext("");
+   StdString *CGroupFactory::CurrContext_ptr = new StdString;
 
    void CGroupFactory::SetCurrentContextId(const StdString & context)
-   { 
-      CGroupFactory::CurrContext = context;
+   {  
+      if(CGroupFactory::CurrContext_ptr == NULL ) CGroupFactory::CurrContext_ptr = new StdString; 
+      CGroupFactory::CurrContext_ptr->assign(context);
    }
 
    StdString & CGroupFactory::GetCurrentContextId(void)
    { 
-      return (CGroupFactory::CurrContext);
+      return (*CGroupFactory::CurrContext_ptr);
    }
 
Index: /XIOS/dev/branch_yushan_merged/src/group_factory.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/group_factory.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/group_factory.hpp	(revision 1134)
@@ -69,5 +69,6 @@
 
          /// PropriÃ©tÃ©s statiques ///
-         static StdString CurrContext;
+         static StdString *CurrContext_ptr;
+         #pragma omp threadprivate(CurrContext_ptr)
 
    }; // class CGroupFactory
Index: /XIOS/dev/branch_yushan_merged/src/indent.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/indent.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/indent.hpp	(revision 1134)
@@ -10,5 +10,9 @@
     public:
     static int defaultIncSize;
+    #pragma omp threadprivate(defaultIncSize)
+    
     static int index ;
+    #pragma omp threadprivate(index)
+    
     int incSize ;
     int offset ;
Index: /XIOS/dev/branch_yushan_merged/src/indent_xml.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/indent_xml.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/indent_xml.cpp	(revision 1134)
@@ -15,4 +15,6 @@
    {
       static unsigned int LineNB = 1;
+      #pragma omp threadprivate(LineNB)
+      
       if (CIndent::WithLine) out << LineNB++ << ". ";
       for(unsigned int i = 0; i < CIndent::Indent; out << CIndent::Increm , i++){}
Index: /XIOS/dev/branch_yushan_merged/src/indent_xml.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/indent_xml.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/indent_xml.hpp	(revision 1134)
@@ -22,6 +22,11 @@
          /// PropriÃ©tÃ©s  statiques ///
          static unsigned int Indent;
+         #pragma omp threadprivate(Indent)
+
          static StdString    Increm;
+         #pragma omp threadprivate(Increm)
+
          static bool         WithLine;
+         #pragma omp threadprivate(WithLine)
 
    }; // class CIndent
Index: /XIOS/dev/branch_yushan_merged/src/interface/c/icdata.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/interface/c/icdata.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/interface/c/icdata.cpp	(revision 1134)
@@ -11,5 +11,5 @@
 
 #include "xios.hpp"
-#include "oasis_cinterface.hpp"
+//#include "oasis_cinterface.hpp"
 
 #include "attribute_template.hpp"
@@ -23,5 +23,5 @@
 #include "context.hpp"
 #include "context_client.hpp"
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "timer.hpp"
 #include "array_new.hpp"
@@ -55,6 +55,8 @@
    {
       std::string str;
-      MPI_Comm local_comm;
-      MPI_Comm return_comm;
+      ep_lib::MPI_Comm local_comm;
+      ep_lib::MPI_Comm return_comm;
+      
+      ep_lib::fc_comm_map.clear();
 
       if (!cstr2string(client_id, len_client_id, str)) return;
@@ -62,8 +64,23 @@
       int initialized;
       MPI_Initialized(&initialized);
+
+      #ifdef _usingEP
+      if (initialized) local_comm = ep_lib::EP_Comm_f2c(static_cast< int >(*f_local_comm));
+      else local_comm = MPI_COMM_NULL;
+      #else
       if (initialized) local_comm=MPI_Comm_f2c(*f_local_comm);
-      else local_comm=MPI_COMM_NULL;
+      else local_comm = MPI_COMM_NULL;
+      #endif
+      
+     
+
       CXios::initClientSide(str, local_comm, return_comm);
-      *f_return_comm=MPI_Comm_c2f(return_comm);
+
+      #ifdef _usingEP
+      *f_return_comm = ep_lib::EP_Comm_c2f(return_comm);
+      #else
+      *f_return_comm = MPI_Comm_c2f(return_comm);
+      #endif
+
       CTimer::get("XIOS init").suspend();
       CTimer::get("XIOS").suspend();
@@ -73,11 +90,13 @@
    {
      std::string str;
-     MPI_Comm comm;
+     ep_lib::MPI_Comm comm;
 
      if (!cstr2string(context_id, len_context_id, str)) return;
      CTimer::get("XIOS").resume();
      CTimer::get("XIOS init context").resume();
-     comm=MPI_Comm_f2c(*f_comm);
-     CClient::registerContext(str, comm);
+     comm = ep_lib::EP_Comm_f2c(static_cast< int >(*f_comm));
+
+     CClient::registerContext(str,comm);
+          
      CTimer::get("XIOS init context").suspend();
      CTimer::get("XIOS").suspend();
@@ -100,5 +119,7 @@
      CTimer::get("XIOS close definition").resume();
      CContext* context = CContext::getCurrent();
+     
      context->closeDefinition();
+     
      CTimer::get("XIOS close definition").suspend();
      CTimer::get("XIOS").suspend();
@@ -109,4 +130,7 @@
      CTimer::get("XIOS").resume();
      CTimer::get("XIOS context finalize").resume();
+     
+     
+     
      CContext* context = CContext::getCurrent();
      context->finalize();
@@ -429,5 +453,7 @@
       CContext* context = CContext::getCurrent();
       if (!context->hasServer && !context->client->isAttachedModeEnabled())
-        context->checkBuffersAndListen();
+      {
+        context->checkBuffersAndListen();
+      }  
 
       CArray<double, 3>data(data_k8, shape(data_Xsize, data_Ysize, data_Zsize), neverDeleteData);
Index: /XIOS/dev/branch_yushan_merged/src/interface/c/oasis_cinterface.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/interface/c/oasis_cinterface.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/interface/c/oasis_cinterface.cpp	(revision 1134)
@@ -1,5 +1,5 @@
 #include "oasis_cinterface.hpp"
 #include <string>
-#include "mpi.hpp"
+//#include "mpi_std.hpp"
 
 namespace xios
@@ -26,5 +26,9 @@
     
     fxios_oasis_get_localcomm(&f_comm) ;
+    #ifdef _usingEP
+    comm=EP_Comm_f2c(f_comm.mpi_fint) ;
+    #else
     comm=MPI_Comm_f2c(f_comm) ;
+    #endif
   }
  
@@ -34,5 +38,9 @@
     
     fxios_oasis_get_intracomm(&f_comm,server_id.data(),server_id.size()) ;
+    #ifdef _usingEP
+    comm_client_server=EP_Comm_f2c(f_comm.mpi_fint) ;
+    #else
     comm_client_server=MPI_Comm_f2c(f_comm) ;
+    #endif
   }
  
@@ -42,5 +50,9 @@
     
     fxios_oasis_get_intercomm(&f_comm,server_id.data(),server_id.size()) ;
+    #ifdef _usingEP
+    comm_client_server=EP_Comm_f2c(f_comm.mpi_fint) ;
+    #else
     comm_client_server=MPI_Comm_f2c(f_comm) ;
+    #endif
   }
 }
Index: /XIOS/dev/branch_yushan_merged/src/interface/fortran/idata.F90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/interface/fortran/idata.F90	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/interface/fortran/idata.F90	(revision 1134)
@@ -465,4 +465,5 @@
    INTEGER :: f_return_comm
 
+
       IF (PRESENT(local_comm)) THEN
         f_local_comm=local_comm
Index: /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.cpp	(revision 1134)
@@ -18,9 +18,10 @@
     }
     mpi = comm && !multifile;
+    MPI_Info m_info;
 
     // The file format will be detected automatically by NetCDF, it is safe to always set NC_MPIIO
     // even if Parallel NetCDF ends up being used.
     if (mpi)
-      CNetCdfInterface::openPar(filename, NC_NOWRITE | NC_MPIIO, *comm, MPI_INFO_NULL, this->ncidp);
+      CNetCdfInterface::openPar(filename, NC_NOWRITE | NC_MPIIO, *comm, m_info, this->ncidp);
     else
       CNetCdfInterface::open(filename, NC_NOWRITE, this->ncidp);
Index: /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/inetcdf4.hpp	(revision 1134)
@@ -7,5 +7,5 @@
 #include "array_new.hpp"
 
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "netcdf.hpp"
 
Index: /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.cpp	(revision 1134)
@@ -28,5 +28,5 @@
       CNc4DataOutput::CNc4DataOutput
          (CFile* file, const StdString & filename, bool exist, bool useClassicFormat, bool useCFConvention,
-          MPI_Comm comm_file, bool multifile, bool isCollective, const StdString& timeCounterName)
+          ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective, const StdString& timeCounterName)
             : SuperClass()
             , SuperClassWriter(filename, exist, useClassicFormat, useCFConvention, &comm_file, multifile, timeCounterName)
@@ -78,5 +78,5 @@
 
 
-        StdString dimXid, dimYid ;
+         StdString dimXid, dimYid ;
 
         nc_type typePrec ;
@@ -463,5 +463,5 @@
       StdString domainName = domain->name;
       domain->assignMesh(domainName, domain->nvertex);
-      domain->mesh->createMeshEpsilon(server->intraComm, domain->lonvalue_srv, domain->latvalue_srv, domain->bounds_lon_srv, domain->bounds_lat_srv);
+      domain->mesh->createMeshEpsilon(static_cast<MPI_Comm>(server->intraComm.mpi_comm), domain->lonvalue_srv, domain->latvalue_srv, domain->bounds_lon_srv, domain->bounds_lat_srv);
 
       StdString node_x = domainName + "_node_x";
Index: /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/nc4_data_output.hpp	(revision 1134)
@@ -27,5 +27,5 @@
                (CFile* file, const StdString & filename, bool exist, bool useClassicFormat,
                 bool useCFConvention,
-                MPI_Comm comm_file, bool multifile, bool isCollective = true,
+                ep_lib::MPI_Comm comm_file, bool multifile, bool isCollective = true,
                 const StdString& timeCounterName = "time_counter");
 
@@ -116,5 +116,5 @@
 
             /// PropriÃ©tÃ©s privÃ©es ///
-            MPI_Comm comm_file;
+            ep_lib::MPI_Comm comm_file;
             const StdString filename;
             std::map<Time, StdSize> timeToRecordCache;
Index: /XIOS/dev/branch_yushan_merged/src/io/netCdfInterface.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/netCdfInterface.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/netCdfInterface.hpp	(revision 1134)
@@ -16,5 +16,5 @@
 #endif
 
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "netcdf.hpp"
 
Index: /XIOS/dev/branch_yushan_merged/src/io/netcdf.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/netcdf.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/netcdf.hpp	(revision 1134)
@@ -1,5 +1,5 @@
 #ifndef __XIOS_NETCDF_HPP__
 #define __XIOS_NETCDF_HPP__
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #define MPI_INCLUDED
 #include <netcdf.h>
@@ -18,5 +18,5 @@
 extern "C"
 {
-#  include <netcdf_par.h>
+  #include <netcdf_par.h>
 }
 #  endif
@@ -30,8 +30,8 @@
 namespace xios
 {
-  inline int nc_create_par(const char *path, int cmode, MPI_Comm comm, MPI_Info info,int *ncidp)
+  inline int nc_create_par(const char *path, int cmode, ep_lib::MPI_Comm comm, MPI_Info info,int *ncidp)
   {
 #if defined(USING_NETCDF_PAR)
-    return ::nc_create_par(path, cmode, comm, info, ncidp) ;
+    return ::nc_create_par(path, cmode, static_cast<MPI_Comm>(comm.mpi_comm), info, ncidp) ;
 #else
     ERROR("int nc_create_par(const char *path, int cmode, MPI_Comm comm, MPI_Info info,int *ncidp)",
@@ -41,8 +41,8 @@
   }
 
-  inline int nc_open_par(const char *path, int mode, MPI_Comm comm, MPI_Info info,int *ncidp)
+  inline int nc_open_par(const char *path, int mode, ep_lib::MPI_Comm comm, MPI_Info info,int *ncidp)
   {
 #if defined(USING_NETCDF_PAR)
-    return ::nc_open_par(path, mode, comm, info, ncidp) ;
+    return ::nc_open_par(path, mode, static_cast<MPI_Comm>(comm.mpi_comm), info, ncidp) ;
 #else
     ERROR("int nc_open_par(const char *path, int mode, MPI_Comm comm, MPI_Info info,int *ncidp)",
Index: /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.cpp	(revision 1134)
@@ -3,5 +3,4 @@
 #include "onetcdf4.hpp"
 #include "group_template.hpp"
-#include "mpi.hpp"
 #include "netcdf.hpp"
 #include "netCdfInterface.hpp"
@@ -12,7 +11,6 @@
       /// ////////////////////// DÃ©finitions ////////////////////// ///
 
-      CONetCDF4::CONetCDF4(const StdString& filename, bool append, bool useClassicFormat,
-							bool useCFConvention,
-                           const MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
+      CONetCDF4::CONetCDF4(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention, 
+                           const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
         : path()
         , wmpi(false)
@@ -32,5 +30,5 @@
 
       void CONetCDF4::initialize(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention, 
-                                 const MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
+                                 const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName)
       {
          this->useClassicFormat = useClassicFormat;
@@ -56,5 +54,5 @@
          {
             if (wmpi)
-               CNetCdfInterface::createPar(filename, mode, *comm, MPI_INFO_NULL, this->ncidp);
+               CNetCdfInterface::createPar(filename, mode, static_cast<MPI_Comm>(comm->mpi_comm), MPI_INFO_NULL_STD, this->ncidp);           
             else
                CNetCdfInterface::create(filename, mode, this->ncidp);
@@ -66,5 +64,5 @@
             mode |= NC_WRITE;
             if (wmpi)
-               CNetCdfInterface::openPar(filename, mode, *comm, MPI_INFO_NULL, this->ncidp);
+               CNetCdfInterface::openPar(filename, mode, static_cast<MPI_Comm>(comm->mpi_comm), MPI_INFO_NULL_STD, this->ncidp);
             else
                CNetCdfInterface::open(filename, mode, this->ncidp);
@@ -535,6 +533,7 @@
                                  const std::vector<StdSize>& scount, const int* data)
       {
-          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
-      }
+         CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+      }
+
       //---------------------------------------------------------------
 
@@ -544,5 +543,5 @@
                                  const std::vector<StdSize>& scount, const float* data)
       {
-          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+         CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
       }
 
Index: /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/io/onetcdf4.hpp	(revision 1134)
@@ -7,5 +7,5 @@
 #include "data_output.hpp"
 #include "array_new.hpp"
-#include "mpi.hpp"
+#include "mpi_std.hpp"
 #include "netcdf.hpp"
 
@@ -28,5 +28,5 @@
             CONetCDF4(const StdString& filename, bool append, bool useClassicFormat = false,
             		  bool useCFConvention = true,
-                      const MPI_Comm* comm = NULL, bool multifile = true,
+                      const ep_lib::MPI_Comm* comm = NULL, bool multifile = true,
                       const StdString& timeCounterName = "time_counter");
 
@@ -37,5 +37,5 @@
             /// Initialisation ///
             void initialize(const StdString& filename, bool append, bool useClassicFormat, bool useCFConvention,
-                            const MPI_Comm* comm, bool multifile, const StdString& timeCounterName);
+                            const ep_lib::MPI_Comm* comm, bool multifile, const StdString& timeCounterName);
             void close(void);
             void sync(void);
Index: /XIOS/dev/branch_yushan_merged/src/log.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/log.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/log.cpp	(revision 1134)
@@ -1,8 +1,38 @@
 #include "log.hpp"
+#include <string>
+#include <iostream>
+#include <string>
 
 namespace xios
 {
+
+  std::filebuf* info_FB[10];
+
+
   CLog info("info") ;
   CLog report("report") ;
   CLog error("error", cerr.rdbuf()) ;
+
+  
+  CLog& CLog::operator()(int l)
+    {
+      if (l<=level)
+      {
+        omp_set_lock( &mutex );
+        //rdbuf(strBuf_); 
+        rdbuf(strBuf_array[omp_get_thread_num()]); 
+        *this<<"-> "<<name<<" : " ;
+        omp_unset_lock( &mutex );
+      }
+      else rdbuf(NULL) ;
+      return *this;
+    }
+
+
+
+  int test_omp_rank;
+  #pragma omp threadprivate(test_omp_rank)
+
+  
+
 }
Index: /XIOS/dev/branch_yushan_merged/src/log.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/log.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/log.hpp	(revision 1134)
@@ -5,4 +5,6 @@
 #include <iostream>
 #include <string>
+#include <stdio.h>
+#include <omp.h>
 
 namespace xios
@@ -14,17 +16,20 @@
     public :
     CLog(const string& name_, std::streambuf* sBuff = cout.rdbuf())
-      : ostream(sBuff), level(0), name(name_), strBuf_(sBuff) {}
-    CLog& operator()(int l)
+      : ostream(cout.rdbuf()), level(0), name(name_), strBuf_(sBuff) 
     {
-      if (l<=level)
-      {
-        rdbuf(strBuf_);
-        *this<<"-> "<<name<<" : " ;
-      }
-      else rdbuf(NULL) ;
-      return *this;
+      omp_init_lock( &mutex );
+      for(int i=0; i<10; i++)
+        strBuf_array[i] = sBuff;
     }
+
+    ~CLog()
+    {
+      omp_destroy_lock( &mutex );
+    }
+
+
+    CLog& operator()(int l);
     void setLevel(int l) {level=l; }
-    int getLevel() {return level ;}
+    int  getLevel() {return level ;}
     bool isActive(void) { if (rdbuf()==NULL) return true ; else return false ;}
     bool isActive(int l) {if (l<=level) return true ; else return false ; }
@@ -46,9 +51,16 @@
      * \param [in] pointer to new streambuf
     */
-    void changeStreamBuff(std::streambuf* sBuff) { strBuf_ = sBuff; rdbuf(sBuff); }
+    void changeStreamBuff(std::streambuf* sBuff) 
+    { 
+      strBuf_ = sBuff; 
+      strBuf_array[omp_get_thread_num()] = sBuff;
+      rdbuf(sBuff);
+    }
 
     int level ;
     string name ;
     std::streambuf* strBuf_;
+    std::streambuf* strBuf_array[10];
+    omp_lock_t mutex;
   };
 
@@ -56,4 +68,9 @@
   extern CLog report;
   extern CLog error;
+
+
+  extern std::filebuf* info_FB[10];
+
+
 }
 #endif
Index: /XIOS/dev/branch_yushan_merged/src/memtrack.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/memtrack.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/memtrack.cpp	(revision 1134)
@@ -68,4 +68,5 @@
         private:    // static member variables
             static BlockHeader *ourFirstNode;
+            #pragma omp threadprivate(ourFirstNode)
     
         private:    // member variables
Index: /XIOS/dev/branch_yushan_merged/src/mpi.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/mpi.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/mpi.hpp	(revision 1134)
@@ -11,5 +11,12 @@
 #define OMPI_SKIP_MPICXX
 
-#include <mpi.h>
+#ifdef _usingEP
+  #include <omp.h>
+  #include "../extern/src_ep_dev/ep_lib.hpp"
+  using namespace ep_lib;
+#elif _usingMPI
+  #include <mpi.h>
+#endif
+
 
 #endif
Index: /XIOS/dev/branch_yushan_merged/src/mpi_std.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/mpi_std.hpp	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/src/mpi_std.hpp	(revision 1134)
@@ -0,0 +1,18 @@
+#ifndef __XIOS_MPI_HPP__
+#define __XIOS_MPI_HPP__
+
+/* skip C++ Binding for mpich , intel MPI */
+#define MPICH_SKIP_MPICXX
+
+/* skip C++ Binding for SGI MPI library */
+#define MPI_NO_CPPBIND
+
+/* skip C++ Binding for OpenMPI */
+#define OMPI_SKIP_MPICXX
+
+#include "../extern/src_ep_dev/ep_lib.hpp"
+#include <mpi.h>
+
+
+
+#endif
Index: /XIOS/dev/branch_yushan_merged/src/node/axis.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/axis.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/axis.cpp	(revision 1134)
@@ -44,6 +44,7 @@
    { /* Ne rien faire de plus */ }
 
-   std::map<StdString, ETranformationType> CAxis::transformationMapList_ = std::map<StdString, ETranformationType>();
-   bool CAxis::dummyTransformationMapList_ = CAxis::initializeTransformationMap(CAxis::transformationMapList_);
+   std::map<StdString, ETranformationType> *CAxis::transformationMapList_ptr = 0; //new std::map<StdString, ETranformationType>();  
+   //bool CAxis::dummyTransformationMapList_ = CAxis::initializeTransformationMap(CAxis::transformationMapList_ptr);
+
    bool CAxis::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
    {
@@ -54,4 +55,16 @@
      m["extract_domain"] = TRANS_EXTRACT_DOMAIN_TO_AXIS;
    }
+
+
+   bool CAxis::initializeTransformationMap()
+   {
+     if(CAxis::transformationMapList_ptr == 0) CAxis::transformationMapList_ptr = new std::map<StdString, ETranformationType>();
+     (*CAxis::transformationMapList_ptr)["zoom_axis"]        = TRANS_ZOOM_AXIS;
+     (*CAxis::transformationMapList_ptr)["interpolate_axis"] = TRANS_INTERPOLATE_AXIS;
+     (*CAxis::transformationMapList_ptr)["inverse_axis"]     = TRANS_INVERSE_AXIS;
+     (*CAxis::transformationMapList_ptr)["reduce_domain"]    = TRANS_REDUCE_DOMAIN_TO_AXIS;
+     (*CAxis::transformationMapList_ptr)["extract_domain"]   = TRANS_EXTRACT_DOMAIN_TO_AXIS;
+   }
+
 
    ///---------------------------------------------------------------
@@ -780,6 +793,6 @@
       CContextServer* server = CContext::getCurrent()->server;
       axis->numberWrittenIndexes_ = axis->indexesToWrite.size();
-      MPI_Allreduce(&axis->numberWrittenIndexes_, &axis->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-      MPI_Scan(&axis->numberWrittenIndexes_, &axis->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      ep_lib::MPI_Allreduce(&axis->numberWrittenIndexes_, &axis->totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+      ep_lib::MPI_Scan(&axis->numberWrittenIndexes_, &axis->offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
       axis->offsetWrittenIndexes_ -= axis->numberWrittenIndexes_;
     }
@@ -1030,4 +1043,6 @@
   }
 
+
+
   void CAxis::duplicateTransformation(CAxis* src)
   {
@@ -1073,6 +1088,10 @@
 
         nodeElementName = node.getElementName();
-        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
-        it = transformationMapList_.find(nodeElementName);
+
+        if(transformationMapList_ptr == 0) initializeTransformationMap();
+        //transformationMapList_ptr = new std::map<StdString, ETranformationType>();
+
+        std::map<StdString, ETranformationType>::const_iterator ite = (*CAxis::transformationMapList_ptr).end(), it;
+        it = (*CAxis::transformationMapList_ptr).find(nodeElementName);
         if (ite != it)
         {
@@ -1096,2 +1115,3 @@
 
 } // namespace xios
+
Index: /XIOS/dev/branch_yushan_merged/src/node/axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/axis.hpp	(revision 1134)
@@ -169,6 +169,14 @@
        private:
          static bool initializeTransformationMap(std::map<StdString, ETranformationType>& m);
-         static std::map<StdString, ETranformationType> transformationMapList_;
-         static bool dummyTransformationMapList_;
+         //static bool initializeTransformationMap(std::map<StdString, ETranformationType>* m);
+         static bool initializeTransformationMap();
+
+         //static std::map<StdString, ETranformationType> transformationMapList_;
+         
+         static std::map<StdString, ETranformationType> *transformationMapList_ptr;
+         #pragma omp threadprivate(transformationMapList_ptr)
+
+         //static bool dummyTransformationMapList_;
+         //#pragma omp threadprivate(dummyTransformationMapList_)
 
          DECLARE_REF_FUNC(Axis,axis)
@@ -182,2 +190,3 @@
 
 #endif // __XIOS_CAxis__
+
Index: /XIOS/dev/branch_yushan_merged/src/node/compute_connectivity_domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/compute_connectivity_domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/compute_connectivity_domain.hpp	(revision 1134)
@@ -60,4 +60,6 @@
       static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
+      
   }; // class CComputeConnectivityDomain
 
Index: /XIOS/dev/branch_yushan_merged/src/node/context.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/context.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/context.cpp	(revision 1134)
@@ -18,5 +18,6 @@
 namespace xios {
 
-  shared_ptr<CContextGroup> CContext::root;
+  //shared_ptr<CContextGroup> CContext::root;
+  shared_ptr<CContextGroup> * CContext::root_ptr = 0;
 
    /// ////////////////////// DÃ©finitions ////////////////////// ///
@@ -54,6 +55,12 @@
    CContextGroup* CContext::getRoot(void)
    {
-      if (root.get()==NULL) root=shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
-      return root.get();
+      //if (root.get()==NULL) root=shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
+      //return root.get();
+
+      //static shared_ptr<CContextGroup> *root_ptr;
+      if(root_ptr == 0) //root_ptr = new shared_ptr<CContextGroup>;
+      // if (root_ptr->get()==NULL) 
+      root_ptr = new shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
+      return root_ptr->get();
    }
 
@@ -236,8 +243,14 @@
 
    //! Initialize client side
-   void CContext::initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer /*= 0*/)
+   void CContext::initClient(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtServer /*= 0*/)
    {
      hasClient=true;
-     client = new CContextClient(this,intraComm, interComm, cxtServer);
+     client = new CContextClient(this, intraComm, interComm, cxtServer);
+
+     int tmp_rank;
+     MPI_Comm_rank(intraComm, &tmp_rank);
+     MPI_Barrier(intraComm);
+     
+
      registryIn=new CRegistry(intraComm);
      registryIn->setPath(getId()) ;
@@ -248,5 +261,5 @@
      registryOut->setPath(getId()) ;
 
-     MPI_Comm intraCommServer, interCommServer;
+     ep_lib::MPI_Comm intraCommServer, interCommServer;
      if (cxtServer) // Attached mode
      {
@@ -311,5 +324,5 @@
 
    //! Initialize server
-   void CContext::initServer(MPI_Comm intraComm,MPI_Comm interComm, CContext* cxtClient /*= 0*/)
+   void CContext::initServer(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtClient /*= 0*/)
    {
      hasServer=true;
@@ -323,5 +336,5 @@
      registryOut->setPath(getId()) ;
 
-     MPI_Comm intraCommClient, interCommClient;
+     ep_lib::MPI_Comm intraCommClient, interCommClient;
      if (cxtClient) // Attached mode
      {
@@ -372,5 +385,5 @@
         }
 
-        for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
+        for (std::list<ep_lib::MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
           MPI_Comm_free(&(*it));
         comms.clear();
@@ -812,4 +825,7 @@
    void CContext::postProcessing()
    {
+     int myRank;
+     MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
      if (isPostProcessed) return;
 
@@ -1191,7 +1207,5 @@
    void CContext::updateCalendar(int step)
    {
-      info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
       calendar->update(step);
-      info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
 
       if (hasClient)
@@ -1241,5 +1255,6 @@
     CContext* context = CObjectFactory::CreateObject<CContext>(id).get();
     getRoot();
-    if (!hasctxt) CGroupFactory::AddChild(root, context->getShared());
+    //if (!hasctxt) CGroupFactory::AddChild(root, context->getShared());
+    if (!hasctxt) CGroupFactory::AddChild(*root_ptr, context->getShared());
 
 #define DECLARE_NODE(Name_, name_) \
Index: /XIOS/dev/branch_yushan_merged/src/node/context.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/context.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/context.hpp	(revision 1134)
@@ -88,6 +88,6 @@
       public :
          // Initialize server or client
-         void initServer(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtClient = 0);
-         void initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer = 0);
+         void initServer(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtClient = 0);
+         void initClient(ep_lib::MPI_Comm intraComm, ep_lib::MPI_Comm interComm, CContext* cxtServer = 0);
          bool isInitialized(void);
 
@@ -206,5 +206,8 @@
 
          // Context root
-         static shared_ptr<CContextGroup> root;
+         //static shared_ptr<CContextGroup> root;
+
+         static shared_ptr<CContextGroup> *root_ptr;
+         #pragma omp threadprivate(root_ptr)
 
          // Determine context on client or not
@@ -219,6 +222,9 @@
          // Concrete contex client
          CContextClient* client;
+
+
          CRegistry* registryIn ;  //!< input registry which is read from file
          CRegistry* registryOut ; //!< output registry which will be wrote on file at the finalize
+         
 
       private:
@@ -227,5 +233,5 @@
          StdString idServer_;
          CGarbageCollector garbageCollector;
-         std::list<MPI_Comm> comms; //!< Communicators allocated internally
+         std::list<ep_lib::MPI_Comm> comms; //!< Communicators allocated internally
 
       public: // Some function maybe removed in the near future
Index: /XIOS/dev/branch_yushan_merged/src/node/domain.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/domain.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/domain.cpp	(revision 1134)
@@ -65,6 +65,8 @@
    }
 
-   std::map<StdString, ETranformationType> CDomain::transformationMapList_ = std::map<StdString, ETranformationType>();
-   bool CDomain::_dummyTransformationMapList = CDomain::initializeTransformationMap(CDomain::transformationMapList_);
+   //std::map<StdString, ETranformationType> CDomain::transformationMapList_ = std::map<StdString, ETranformationType>();
+   //bool CDomain::_dummyTransformationMapList = CDomain::initializeTransformationMap(CDomain::transformationMapList_);
+
+   std::map<StdString, ETranformationType> *CDomain::transformationMapList_ptr = 0;
 
    bool CDomain::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
@@ -76,4 +78,15 @@
      m["expand_domain"] = TRANS_EXPAND_DOMAIN;
    }
+
+   bool CDomain::initializeTransformationMap()
+   {
+     CDomain::transformationMapList_ptr = new std::map<StdString, ETranformationType>();
+     (*CDomain::transformationMapList_ptr)["zoom_domain"] = TRANS_ZOOM_DOMAIN;
+     (*CDomain::transformationMapList_ptr)["interpolate_domain"] = TRANS_INTERPOLATE_DOMAIN;
+     (*CDomain::transformationMapList_ptr)["generate_rectilinear_domain"] = TRANS_GENERATE_RECTILINEAR_DOMAIN;
+     (*CDomain::transformationMapList_ptr)["compute_connectivity_domain"] = TRANS_COMPUTE_CONNECTIVITY_DOMAIN;
+     (*CDomain::transformationMapList_ptr)["expand_domain"] = TRANS_EXPAND_DOMAIN;
+   }
+
 
    const std::set<StdString> & CDomain::getRelFiles(void) const
@@ -623,5 +636,5 @@
    {
 	  CContext* context = CContext::getCurrent();
-      CContextClient* client = context->client;
+          CContextClient* client = context->client;
 	  lon_g.resize(ni_glo) ;
 	  lat_g.resize(nj_glo) ;
@@ -1713,4 +1726,5 @@
                                                                                 client->intraComm);
     clientServerMap->computeServerIndexMapping(globalIndexDomain);
+    
     const CClientServerMapping::GlobalIndexMap& globalIndexDomainOnServer = clientServerMap->getGlobalIndexOnServer();
 
@@ -2350,6 +2364,7 @@
 
         nodeElementName = node.getElementName();
-        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
-        it = transformationMapList_.find(nodeElementName);
+        if(transformationMapList_ptr == 0) initializeTransformationMap();
+        std::map<StdString, ETranformationType>::const_iterator ite = (*transformationMapList_ptr).end(), it;
+        it = (*transformationMapList_ptr).find(nodeElementName);
         if (ite != it)
         {
Index: /XIOS/dev/branch_yushan_merged/src/node/domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/domain.hpp	(revision 1134)
@@ -216,6 +216,12 @@
        private:
          static bool initializeTransformationMap(std::map<StdString, ETranformationType>& m);
-         static std::map<StdString, ETranformationType> transformationMapList_;
-         static bool _dummyTransformationMapList;
+         static bool initializeTransformationMap();
+         //static std::map<StdString, ETranformationType> transformationMapList_;
+ 
+         static std::map<StdString, ETranformationType> *transformationMapList_ptr;
+         #pragma omp threadprivate(transformationMapList_ptr)
+
+         //static bool _dummyTransformationMapList;
+         //#pragma omp threadprivate(_dummyTransformationMapList)
 
          DECLARE_REF_FUNC(Domain,domain)
Index: /XIOS/dev/branch_yushan_merged/src/node/expand_domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/expand_domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/expand_domain.hpp	(revision 1134)
@@ -60,4 +60,6 @@
       static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
+      
   }; // class CExpandDomain
 
Index: /XIOS/dev/branch_yushan_merged/src/node/extract_axis_to_scalar.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/extract_axis_to_scalar.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/extract_axis_to_scalar.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CExtractAxisToScalar
 
Index: /XIOS/dev/branch_yushan_merged/src/node/extract_domain_to_axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/extract_domain_to_axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/extract_domain_to_axis.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CExtractDomainToAxis
 
Index: /XIOS/dev/branch_yushan_merged/src/node/field.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/field.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/field.cpp	(revision 1134)
@@ -953,5 +953,5 @@
 
        const bool ignoreMissingValue = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
-       
+
        boost::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
                                                                              CContext::getCurrent()->getCalendar()->getInitDate(),
Index: /XIOS/dev/branch_yushan_merged/src/node/file.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/file.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/file.cpp	(revision 1134)
@@ -579,6 +579,6 @@
 
       if (isOpen) data_out->closeFile();
-      if (time_counter_name.isEmpty()) data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective));
-      else data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), fileComm, multifile, isCollective, time_counter_name));
+      if (time_counter_name.isEmpty()) data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast<MPI_Comm>(fileComm.mpi_comm), multifile, isCollective));
+      else data_in = shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), static_cast<MPI_Comm>(fileComm.mpi_comm), multifile, isCollective, time_counter_name));
       isOpen = true;
     }
Index: /XIOS/dev/branch_yushan_merged/src/node/file.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/file.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/file.hpp	(revision 1134)
@@ -12,4 +12,7 @@
 #include "attribute_enum_impl.hpp"
 #include "mpi.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace xios {
@@ -156,5 +159,5 @@
          bool isOpen;
          bool allDomainEmpty;
-         MPI_Comm fileComm;
+         ep_lib::MPI_Comm fileComm;
 
       private :
Index: /XIOS/dev/branch_yushan_merged/src/node/generate_rectilinear_domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/generate_rectilinear_domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/generate_rectilinear_domain.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CGenerateRectilinearDomain
 
Index: /XIOS/dev/branch_yushan_merged/src/node/grid.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/grid.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/grid.cpp	(revision 1134)
@@ -1114,5 +1114,5 @@
           outLocalIndexToServer(idx) = itIndex->second;
         }
-        
+
         const std::list<int>& ranks = client->getRanksServerLeader();
         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
Index: /XIOS/dev/branch_yushan_merged/src/node/interpolate_axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/interpolate_axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/interpolate_axis.hpp	(revision 1134)
@@ -62,4 +62,6 @@
       static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
+      
   }; // class CInterpolateAxis
 
Index: /XIOS/dev/branch_yushan_merged/src/node/interpolate_domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/interpolate_domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/interpolate_domain.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CInterpolateDomain
 
Index: /XIOS/dev/branch_yushan_merged/src/node/inverse_axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/inverse_axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/inverse_axis.hpp	(revision 1134)
@@ -59,4 +59,5 @@
       static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
 
   }; // class CInverseAxis
Index: /XIOS/dev/branch_yushan_merged/src/node/mesh.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/mesh.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/mesh.cpp	(revision 1134)
@@ -34,4 +34,8 @@
   std::map <StdString, vector<int> > CMesh::domainList = std::map <StdString, vector<int> >();
 
+  std::map <StdString, CMesh> *CMesh::meshList_ptr = 0;
+  std::map <StdString, vector<int> > *CMesh::domainList_ptr = 0;
+
+
 ///---------------------------------------------------------------
 /*!
@@ -41,4 +45,6 @@
  * \param [in] nvertex Number of verteces (1 for nodes, 2 for edges, 3 and up for faces).
  */
+
+/* bkp
   CMesh* CMesh::getMesh (StdString meshName, int nvertex)
   {
@@ -64,4 +70,34 @@
       CMesh::meshList.insert( make_pair(meshName, newMesh) );
       return &meshList[meshName];
+    }
+  }
+*/
+
+  CMesh* CMesh::getMesh (StdString meshName, int nvertex)
+  {
+    if(CMesh::domainList_ptr == NULL) CMesh::domainList_ptr = new std::map <StdString, vector<int> >();
+    if(CMesh::meshList_ptr == NULL)   CMesh::meshList_ptr   = new std::map <StdString, CMesh>();
+
+    (*CMesh::domainList_ptr)[meshName].push_back(nvertex);
+
+    if ( (*CMesh::meshList_ptr).begin() != (*CMesh::meshList_ptr).end() )
+    {
+      for (std::map<StdString, CMesh>::iterator it=(*CMesh::meshList_ptr).begin(); it!=(*CMesh::meshList_ptr).end(); ++it)
+      {
+        if (it->first == meshName)
+          return &((*CMesh::meshList_ptr)[meshName]);
+        else
+        {
+          CMesh newMesh;
+          (*CMesh::meshList_ptr).insert( make_pair(meshName, newMesh) );
+          return &((*CMesh::meshList_ptr)[meshName]);
+        }
+      }
+    }
+    else
+    {
+      CMesh newMesh;
+      (*CMesh::meshList_ptr).insert( make_pair(meshName, newMesh) );
+      return &((*CMesh::meshList_ptr)[meshName]);
     }
   }
@@ -488,5 +524,5 @@
  * \param [in] bounds_lat Array of boundary latitudes. Its size depends on the element type.
  */
-  void CMesh::createMeshEpsilon(const MPI_Comm& comm,
+  void CMesh::createMeshEpsilon(const ep_lib::MPI_Comm& comm,
                                 const CArray<double, 1>& lonvalue, const CArray<double, 1>& latvalue,
                                 const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat)
@@ -1688,5 +1724,5 @@
    */
 
-  void CMesh::getGloNghbFacesNodeType(const MPI_Comm& comm, const CArray<int, 1>& face_idx,
+  void CMesh::getGloNghbFacesNodeType(const ep_lib::MPI_Comm& comm, const CArray<int, 1>& face_idx,
                                const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,
                                CArray<int, 2>& nghbFaces)
@@ -1844,5 +1880,5 @@
    */
 
-  void CMesh::getGloNghbFacesEdgeType(const MPI_Comm& comm, const CArray<int, 1>& face_idx,
+  void CMesh::getGloNghbFacesEdgeType(const ep_lib::MPI_Comm& comm, const CArray<int, 1>& face_idx,
                                const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,
                                CArray<int, 2>& nghbFaces)
@@ -2025,5 +2061,5 @@
    */
 
-  void CMesh::getGlobalNghbFaces(const int nghbType, const MPI_Comm& comm,
+  void CMesh::getGlobalNghbFaces(const int nghbType, const ep_lib::MPI_Comm& comm,
                                  const CArray<int, 1>& face_idx,
                                  const CArray<double, 2>& bounds_lon, const CArray<double, 2>& bounds_lat,
Index: /XIOS/dev/branch_yushan_merged/src/node/mesh.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/mesh.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/mesh.hpp	(revision 1134)
@@ -60,9 +60,9 @@
                       const CArray<double, 2>&, const CArray<double, 2>& );
                         
-      void createMeshEpsilon(const MPI_Comm&,
+      void createMeshEpsilon(const ep_lib::MPI_Comm&,
                              const CArray<double, 1>&, const CArray<double, 1>&,
                              const CArray<double, 2>&, const CArray<double, 2>& );
 
-      void getGlobalNghbFaces(const int, const MPI_Comm&, const CArray<int, 1>&,
+      void getGlobalNghbFaces(const int, const ep_lib::MPI_Comm&, const CArray<int, 1>&,
                               const CArray<double, 2>&, const CArray<double, 2>&,
                               CArray<int, 2>&);
@@ -82,8 +82,13 @@
       static std::map <StdString, CMesh> meshList;
       static std::map <StdString, vector<int> > domainList;
+
+      static std::map <StdString, CMesh> *meshList_ptr;
+      static std::map <StdString, vector<int> > *domainList_ptr;
+      #pragma omp threadprivate(meshList_ptr, domainList_ptr)
+   
       CClientClientDHTSizet* pNodeGlobalIndex;                    // pointer to a map <nodeHash, nodeIdxGlo>
       CClientClientDHTSizet* pEdgeGlobalIndex;                    // pointer to a map <edgeHash, edgeIdxGlo>
-      void getGloNghbFacesNodeType(const MPI_Comm&, const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&);
-      void getGloNghbFacesEdgeType(const MPI_Comm&, const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&);
+      void getGloNghbFacesNodeType(const ep_lib::MPI_Comm&, const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&);
+      void getGloNghbFacesEdgeType(const ep_lib::MPI_Comm&, const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&);
       void getLocNghbFacesNodeType(const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&, CArray<int, 1>&);
       void getLocNghbFacesEdgeType(const CArray<int, 1>&, const CArray<double, 2>&, const CArray<double, 2>&, CArray<int, 2>&, CArray<int, 1>&);
Index: /XIOS/dev/branch_yushan_merged/src/node/reduce_axis_to_scalar.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/reduce_axis_to_scalar.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/reduce_axis_to_scalar.hpp	(revision 1134)
@@ -59,4 +59,5 @@
       static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CReduceAxisToScalar
 
Index: /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_axis.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CReduceDomainToAxis
 
Index: /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_scalar.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_scalar.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/reduce_domain_to_scalar.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CScalar>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CReduceDomainToScalar
 
Index: /XIOS/dev/branch_yushan_merged/src/node/scalar.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/scalar.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/scalar.cpp	(revision 1134)
@@ -27,6 +27,9 @@
    { /* Ne rien faire de plus */ }
 
-   std::map<StdString, ETranformationType> CScalar::transformationMapList_ = std::map<StdString, ETranformationType>();
-   bool CScalar::dummyTransformationMapList_ = CScalar::initializeTransformationMap(CScalar::transformationMapList_);
+   //std::map<StdString, ETranformationType> CScalar::transformationMapList_ = std::map<StdString, ETranformationType>();
+   //bool CScalar::dummyTransformationMapList_ = CScalar::initializeTransformationMap(CScalar::transformationMapList_);
+
+   std::map<StdString, ETranformationType> *CScalar::transformationMapList_ptr = 0;
+   
    bool CScalar::initializeTransformationMap(std::map<StdString, ETranformationType>& m)
    {
@@ -34,4 +37,12 @@
      m["extract_axis"]  = TRANS_EXTRACT_AXIS_TO_SCALAR;
      m["reduce_domain"] = TRANS_REDUCE_DOMAIN_TO_SCALAR;
+   }
+
+   bool CScalar::initializeTransformationMap()
+   {
+     CScalar::transformationMapList_ptr = new std::map<StdString, ETranformationType>();
+     (*CScalar::transformationMapList_ptr)["reduce_axis"]   = TRANS_REDUCE_AXIS_TO_SCALAR;
+     (*CScalar::transformationMapList_ptr)["extract_axis"]  = TRANS_EXTRACT_AXIS_TO_SCALAR;
+     (*CScalar::transformationMapList_ptr)["reduce_domain"] = TRANS_REDUCE_DOMAIN_TO_SCALAR;
    }
 
@@ -164,6 +175,7 @@
 
         nodeElementName = node.getElementName();
-        std::map<StdString, ETranformationType>::const_iterator ite = transformationMapList_.end(), it;
-        it = transformationMapList_.find(nodeElementName);
+        if(CScalar::transformationMapList_ptr == 0) initializeTransformationMap();
+        std::map<StdString, ETranformationType>::const_iterator ite = (*CScalar::transformationMapList_ptr).end(), it;
+        it = (*CScalar::transformationMapList_ptr).find(nodeElementName);
         if (ite != it)
         {
Index: /XIOS/dev/branch_yushan_merged/src/node/scalar.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/scalar.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/scalar.hpp	(revision 1134)
@@ -87,6 +87,10 @@
        private:
            static bool initializeTransformationMap(std::map<StdString, ETranformationType>& m);
-           static std::map<StdString, ETranformationType> transformationMapList_;
-           static bool dummyTransformationMapList_;
+           static bool initializeTransformationMap();
+           
+           //static bool dummyTransformationMapList_;
+
+           static std::map<StdString, ETranformationType> *transformationMapList_ptr;
+           #pragma omp threadprivate(transformationMapList_ptr)
 
 
Index: /XIOS/dev/branch_yushan_merged/src/node/zoom_axis.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/zoom_axis.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/zoom_axis.hpp	(revision 1134)
@@ -60,4 +60,5 @@
       static CTransformation<CAxis>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
   }; // class CZoomAxis
 
Index: /XIOS/dev/branch_yushan_merged/src/node/zoom_domain.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/node/zoom_domain.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/node/zoom_domain.hpp	(revision 1134)
@@ -60,4 +60,6 @@
       static CTransformation<CDomain>* create(const StdString& id, xml::CXMLNode* node);
       static bool _dummyRegistered;
+      #pragma omp threadprivate(_dummyRegistered)
+      
   }; // class CZoomDomain
 
Index: /XIOS/dev/branch_yushan_merged/src/object_factory.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/object_factory.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/object_factory.cpp	(revision 1134)
@@ -5,11 +5,16 @@
    /// ////////////////////// DÃ©finitions ////////////////////// ///
 
-   StdString CObjectFactory::CurrContext("");
+   StdString *CObjectFactory::CurrContext_ptr = new StdString;
 
    void CObjectFactory::SetCurrentContextId(const StdString & context)
-   { CObjectFactory::CurrContext = context; }
+   {
+     if(CObjectFactory::CurrContext_ptr == NULL ) CObjectFactory::CurrContext_ptr = new StdString;
+     CObjectFactory::CurrContext_ptr->assign(context); 
+   }
 
    StdString & CObjectFactory::GetCurrentContextId(void)
-   { return (CObjectFactory::CurrContext); }
+   { 
+     return (*CObjectFactory::CurrContext_ptr); 
+   }
 
 } // namespace xios
Index: /XIOS/dev/branch_yushan_merged/src/object_factory.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/object_factory.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/object_factory.hpp	(revision 1134)
@@ -59,5 +59,6 @@
 
          /// PropriÃ©tÃ©s statiques ///
-         static StdString CurrContext;
+         static StdString *CurrContext_ptr;
+         #pragma omp threadprivate(CurrContext_ptr)
 
    }; // class CObjectFactory
Index: /XIOS/dev/branch_yushan_merged/src/object_factory_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/object_factory_impl.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/object_factory_impl.hpp	(revision 1134)
@@ -10,8 +10,12 @@
        int CObjectFactory::GetObjectNum(void)
    {
-      if (CurrContext.size() == 0)
+      if (CurrContext_ptr->size() == 0)
          ERROR("CObjectFactory::GetObjectNum(void)",
                << "please define current context id !");
-      return (U::AllVectObj[CObjectFactory::CurrContext].size());
+
+      if(U::AllVectObj == NULL) return 0;
+      
+      
+      return (*U::AllVectObj)[*CObjectFactory::CurrContext_ptr].size();
    }
 
@@ -19,8 +23,12 @@
       int CObjectFactory::GetObjectIdNum(void)
    {
-      if (CurrContext.size() == 0)
+      if (CurrContext_ptr->size() == 0)
          ERROR("CObjectFactory::GetObjectIdNum(void)",
                << "please define current context id !");
-      return (U::AllMapObj[CObjectFactory::CurrContext].size());
+      if(U::AllMapObj  == NULL) return 0;
+
+      
+
+      return (* U::AllMapObj) [*CObjectFactory::CurrContext_ptr].size();
    }
 
@@ -28,9 +36,14 @@
       bool CObjectFactory::HasObject(const StdString & id)
    {
-      if (CurrContext.size() == 0)
+      if (CurrContext_ptr->size() == 0)
          ERROR("CObjectFactory::HasObject(const StdString & id)",
                << "[ id = " << id << " ] please define current context id !");
-      return (U::AllMapObj[CObjectFactory::CurrContext].find(id) !=
-              U::AllMapObj[CObjectFactory::CurrContext].end());
+      
+      if(U::AllMapObj  == NULL)  return false;
+
+      
+
+      return ((*U::AllMapObj)[*CObjectFactory::CurrContext_ptr].find(id) !=
+              (*U::AllMapObj)[*CObjectFactory::CurrContext_ptr].end());
    }
 
@@ -38,6 +51,13 @@
       bool CObjectFactory::HasObject(const StdString & context, const StdString & id)
    {
-      if (U::AllMapObj.find(context) == U::AllMapObj.end()) return false ;
-      else return (U::AllMapObj[context].find(id) !=  U::AllMapObj[context].end());
+      if(U::AllMapObj  == NULL) return false;
+
+      if (U::AllMapObj->find(context) == U::AllMapObj->end()) return false ;
+
+      else
+      {
+         return ((*U::AllMapObj)[context].find(id) !=  (*U::AllMapObj)[context].end());
+      } 
+         
    }
 
@@ -45,9 +65,11 @@
       boost::shared_ptr<U> CObjectFactory::GetObject(const U * const object)
    {
-      if (CurrContext.size() == 0)
+      if(U::AllVectObj == NULL) return (boost::shared_ptr<U>());
+   
+      if (CurrContext_ptr->size() == 0)
          ERROR("CObjectFactory::GetObject(const U * const object)",
                << "please define current context id !");
       std::vector<boost::shared_ptr<U> > & vect =
-                     U::AllVectObj[CObjectFactory::CurrContext];
+                     (*U::AllVectObj)[*CObjectFactory::CurrContext_ptr];
 
       typename std::vector<boost::shared_ptr<U> >::const_iterator
@@ -70,5 +92,7 @@
       boost::shared_ptr<U> CObjectFactory::GetObject(const StdString & id)
    {
-      if (CurrContext.size() == 0)
+      if(U::AllMapObj  == NULL) return (boost::shared_ptr<U>());
+
+      if (CurrContext_ptr->size() == 0)
          ERROR("CObjectFactory::GetObject(const StdString & id)",
                << "[ id = " << id << " ] please define current context id !");
@@ -77,5 +101,5 @@
                << "[ id = " << id << ", U = " << U::GetName() << " ] "
                << "object was not found.");
-      return (U::AllMapObj[CObjectFactory::CurrContext][id]);
+      return (*U::AllMapObj)[*CObjectFactory::CurrContext_ptr][id];
    }
 
@@ -83,9 +107,12 @@
       boost::shared_ptr<U> CObjectFactory::GetObject(const StdString & context, const StdString & id)
    {
+      if(U::AllMapObj  == NULL) return (boost::shared_ptr<U>());
+
       if (!CObjectFactory::HasObject<U>(context,id))
          ERROR("CObjectFactory::GetObject(const StdString & id)",
                << "[ id = " << id << ", U = " << U::GetName() <<", context = "<<context<< " ] "
                << "object was not found.");
-      return (U::AllMapObj[context][id]);
+
+      return (*U::AllMapObj)[context][id];
    }
 
@@ -93,5 +120,9 @@
    boost::shared_ptr<U> CObjectFactory::CreateObject(const StdString& id)
    {
-      if (CurrContext.empty())
+      if(U::AllVectObj == NULL) U::AllVectObj = new xios_map<StdString, std::vector<boost::shared_ptr<U> > >;
+      if(U::AllMapObj  == NULL) U::AllMapObj  = new xios_map<StdString, xios_map<StdString, boost::shared_ptr<U> > >;
+
+      
+      if (CurrContext_ptr->empty())
          ERROR("CObjectFactory::CreateObject(const StdString& id)",
                << "[ id = " << id << " ] please define current context id !");
@@ -105,6 +136,6 @@
          boost::shared_ptr<U> value(new U(id.empty() ? CObjectFactory::GenUId<U>() : id));
 
-         U::AllVectObj[CObjectFactory::CurrContext].insert(U::AllVectObj[CObjectFactory::CurrContext].end(), value);
-         U::AllMapObj[CObjectFactory::CurrContext].insert(std::make_pair(value->getId(), value));
+         (* U::AllVectObj)[*CObjectFactory::CurrContext_ptr].insert((*U::AllVectObj)[*CObjectFactory::CurrContext_ptr].end(), value);
+         (* U::AllMapObj) [*CObjectFactory::CurrContext_ptr].insert(std::make_pair(value->getId(), value));
 
          return value;
@@ -116,5 +147,7 @@
          CObjectFactory::GetObjectVector(const StdString & context)
    {
-      return (U::AllVectObj[context]);
+      if(U::AllVectObj != NULL) 
+      
+      return (*U::AllVectObj)[context];
    }
 
@@ -130,5 +163,6 @@
    {
       StdOStringStream oss;
-      oss << GetUIdBase<U>() << U::GenId[CObjectFactory::CurrContext]++;
+      if(U::GenId == NULL) U::GenId = new xios_map< StdString, long int >;
+      oss << GetUIdBase<U>() << (*U::GenId)[*CObjectFactory::CurrContext_ptr]++;
       return oss.str();
    }
Index: /XIOS/dev/branch_yushan_merged/src/object_template.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/object_template.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/object_template.hpp	(revision 1134)
@@ -100,11 +100,18 @@
 
          /// PropriÃ©tÃ©s statiques ///
-         static xios_map<StdString,
-                xios_map<StdString,
-                boost::shared_ptr<DerivedType> > > AllMapObj;
-         static xios_map<StdString,
-                std::vector<boost::shared_ptr<DerivedType> > > AllVectObj;
+         // bkp
+         // static xios_map<StdString,
+         //        xios_map<StdString,
+         //        boost::shared_ptr<DerivedType> > > AllMapObj;
+         // static xios_map<StdString,
+         //        std::vector<boost::shared_ptr<DerivedType> > > AllVectObj;
 
-         static xios_map< StdString, long int > GenId ;
+         // static xios_map< StdString, long int > GenId ;
+
+
+         static xios_map<StdString, xios_map<StdString, boost::shared_ptr<DerivedType> > > *AllMapObj;
+         static xios_map<StdString, std::vector<boost::shared_ptr<DerivedType> > > *AllVectObj;
+         static xios_map< StdString, long int > *GenId;
+         #pragma omp threadprivate(AllMapObj, AllVectObj, GenId)
 
    }; // class CObjectTemplate
Index: /XIOS/dev/branch_yushan_merged/src/object_template_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/object_template_impl.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/object_template_impl.hpp	(revision 1134)
@@ -24,12 +24,12 @@
       xios_map<StdString,
       xios_map<StdString,
-      boost::shared_ptr<T> > > CObjectTemplate<T>::AllMapObj;
+      boost::shared_ptr<T> > > *CObjectTemplate<T>::AllMapObj = 0;
 
    template <class T>
       xios_map<StdString,
-      std::vector<boost::shared_ptr<T> > > CObjectTemplate<T>::AllVectObj;
-
-   template <class T>
-      xios_map<StdString,long int> CObjectTemplate<T>::GenId;
+      std::vector<boost::shared_ptr<T> > > *CObjectTemplate<T>::AllVectObj = 0;
+
+   template <class T>
+      xios_map<StdString,long int> *CObjectTemplate<T>::GenId = 0;
 
    template <class T>
@@ -66,5 +66,5 @@
          CObjectTemplate<T>::GetAllVectobject(const StdString & contextId)
    {
-      return (CObjectTemplate<T>::AllVectObj[contextId]);
+      return (CObjectTemplate<T>::AllVectObj->at(contextId));
    }
 
Index: /XIOS/dev/branch_yushan_merged/src/parse_expr/lex_parser.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/parse_expr/lex_parser.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/parse_expr/lex_parser.cpp	(revision 1134)
@@ -347,4 +347,5 @@
 extern char *yytext;
 #define yytext_ptr yytext
+
 static yyconst flex_int16_t yy_nxt[][128] =
     {
Index: /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.cpp	(revision 1134)
@@ -80,15 +80,18 @@
 }
 
-  IFilterExprNode* parsed;
-  std::string globalInputText;
-  size_t globalReadOffset = 0;
-
+  static IFilterExprNode* parsed;
+  static std::string globalInputText;
+  static std::string *globalInputText_ptr = 0;
+  static size_t globalReadOffset = 0;
+  #pragma omp threadprivate(parsed, globalInputText_ptr, globalReadOffset)
+  
   int readInputForLexer(char* buffer, size_t* numBytesRead, size_t maxBytesToRead)
   {
+    if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
     size_t numBytesToRead = maxBytesToRead;
-    size_t bytesRemaining = globalInputText.length()-globalReadOffset;
+    size_t bytesRemaining = (*globalInputText_ptr).length()-globalReadOffset;
     size_t i;
     if (numBytesToRead > bytesRemaining) numBytesToRead = bytesRemaining;
-    for (i = 0; i < numBytesToRead; i++) buffer[i] = globalInputText.c_str()[globalReadOffset + i];
+    for (i = 0; i < numBytesToRead; i++) buffer[i] = (*globalInputText_ptr).c_str()[globalReadOffset + i];
     *numBytesRead = numBytesToRead;
     globalReadOffset += numBytesToRead;
@@ -2002,7 +2005,11 @@
   IFilterExprNode* parseExpr(const string& strExpr)
   {
-    globalInputText = strExpr;
-    globalReadOffset = 0;
-    yyparse();
+    #pragma omp critical (_parser)
+    {
+      if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
+      (*globalInputText_ptr).assign (strExpr);
+      globalReadOffset = 0;
+      yyparse();
+    }
     return parsed;
   }
@@ -2010,2 +2017,3 @@
 
 
+
Index: /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.yacc
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.yacc	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/parse_expr/yacc_parser.yacc	(revision 1134)
@@ -15,15 +15,18 @@
 }
 
-  IFilterExprNode* parsed;
-  std::string globalInputText;
-  size_t globalReadOffset = 0;
-
+  static IFilterExprNode* parsed;
+  static std::string globalInputText;
+  static std::string *globalInputText_ptr = 0;
+  static size_t globalReadOffset = 0;
+  #pragma omp threadprivate(parsed, globalInputText_ptr, globalReadOffset)
+  
   int readInputForLexer(char* buffer, size_t* numBytesRead, size_t maxBytesToRead)
   {
+    if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
     size_t numBytesToRead = maxBytesToRead;
-    size_t bytesRemaining = globalInputText.length()-globalReadOffset;
+    size_t bytesRemaining = (*globalInputText_ptr).length()-globalReadOffset;
     size_t i;
     if (numBytesToRead > bytesRemaining) numBytesToRead = bytesRemaining;
-    for (i = 0; i < numBytesToRead; i++) buffer[i] = globalInputText.c_str()[globalReadOffset + i];
+    for (i = 0; i < numBytesToRead; i++) buffer[i] = (*globalInputText_ptr).c_str()[globalReadOffset + i];
     *numBytesRead = numBytesToRead;
     globalReadOffset += numBytesToRead;
@@ -145,7 +148,11 @@
   IFilterExprNode* parseExpr(const string& strExpr)
   {
-    globalInputText = strExpr;
-    globalReadOffset = 0;
-    yyparse();
+    #pragma omp critical (_parser)
+    {
+      if(globalInputText_ptr == 0) globalInputText_ptr = new std::string;
+      (*globalInputText_ptr).assign (strExpr);
+      globalReadOffset = 0;
+      yyparse();
+    }
     return parsed;
   }
Index: /XIOS/dev/branch_yushan_merged/src/policy.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/policy.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/policy.hpp	(revision 1134)
@@ -31,5 +31,5 @@
 {
 protected:
-  DivideAdaptiveComm(const MPI_Comm& mpiComm);
+  DivideAdaptiveComm(const ep_lib::MPI_Comm& mpiComm);
 
   void computeMPICommLevel();
@@ -41,5 +41,5 @@
 
 protected:
-  const MPI_Comm& internalComm_;
+  const ep_lib::MPI_Comm& internalComm_;
   std::vector<std::vector<int> > groupParentsBegin_;
   std::vector<std::vector<int> > nbInGroupParents_;
Index: /XIOS/dev/branch_yushan_merged/src/registry.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/registry.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/registry.cpp	(revision 1134)
@@ -1,5 +1,4 @@
 #include "registry.hpp"
 #include "type.hpp"
-#include <mpi.hpp>
 #include <fstream>
 #include <sstream>
@@ -258,5 +257,6 @@
   void CRegistry::hierarchicalGatherRegistry(void)
   {
-    hierarchicalGatherRegistry(communicator) ;
+   // hierarchicalGatherRegistry(communicator) ;
+    gatherRegistry(communicator) ;
   }
 
Index: /XIOS/dev/branch_yushan_merged/src/registry.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/registry.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/registry.hpp	(revision 1134)
@@ -6,4 +6,8 @@
 #include "mpi.hpp"
 #include "message.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+
 
 // Those two headers can be replaced by the C++11 equivalent in the future
@@ -23,5 +27,5 @@
 
 /** Constructor, the communicator is used for bcast or gather operation between MPI processes */
-      CRegistry(const MPI_Comm& comm=MPI_COMM_WORLD) : communicator(comm) {}
+      CRegistry(const ep_lib::MPI_Comm& comm=MPI_COMM_WORLD) : communicator(comm) {}
 
 /** Copy constructor */
@@ -120,5 +124,5 @@
 
 /** MPI communicator used for broadcast and gather operation */
-      MPI_Comm communicator ;
+      ep_lib::MPI_Comm communicator ;
   } ;
 
Index: /XIOS/dev/branch_yushan_merged/src/server.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/server.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/server.cpp	(revision 1134)
@@ -9,5 +9,4 @@
 #include <boost/functional/hash.hpp>
 #include <boost/algorithm/string.hpp>
-#include "mpi.hpp"
 #include "tracer.hpp"
 #include "timer.hpp"
@@ -26,21 +25,14 @@
     bool CServer::finished=false ;
     bool CServer::is_MPI_Initialized ;
+
+    
     CEventScheduler* CServer::eventScheduler = 0;
    
     void CServer::initialize(void)
     {
-      int initialized ;
-      MPI_Initialized(&initialized) ;
-      if (initialized) is_MPI_Initialized=true ;
-      else is_MPI_Initialized=false ;
-
       // Not using OASIS
       if (!CXios::usingOasis)
       {
 
-        if (!is_MPI_Initialized)
-        {
-          MPI_Init(NULL, NULL);
-        }
         CTimer::get("XIOS").resume() ;
 
@@ -50,5 +42,5 @@
         unsigned long* hashAll ;
 
-//        int rank ;
+
         int size ;
         int myColor ;
@@ -77,11 +69,14 @@
 
         myColor=colors[hashServer] ;
-        MPI_Comm_split(MPI_COMM_WORLD,myColor,rank,&intraComm) ;
-
+
+
+        MPI_Comm_split(CXios::globalComm,myColor,rank,&intraComm) ;
+
+        
         int serverLeader=leaders[hashServer] ;
         int clientLeader;
 
          serverLeader=leaders[hashServer] ;
-         for(it=leaders.begin();it!=leaders.end();it++)
+         for(it=leaders.begin();it!=leaders.end();++it)
          {
            if (it->first!=hashServer)
@@ -104,5 +99,4 @@
       else
       {
-//        int rank ,size;
         int size;
         if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId);
@@ -135,5 +129,4 @@
       }
 
-//      int rank;
       MPI_Comm_rank(intraComm,&rank) ;
       if (rank==0) isRoot=true;
@@ -149,8 +142,9 @@
       delete eventScheduler ;
 
-      for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
+      for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); ++it)
         MPI_Comm_free(&(*it));
-      for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); it++)
+      for (std::list<MPI_Comm>::iterator it = interComm.begin(); it != interComm.end(); ++it)
         MPI_Comm_free(&(*it));
+
       MPI_Comm_free(&intraComm);
 
@@ -158,6 +152,8 @@
       {
         if (CXios::usingOasis) oasis_finalize();
-        else MPI_Finalize() ;
+        //else  {MPI_Finalize() ;}
       }
+
+      
       report(0)<<"Performance report : Time spent for XIOS : "<<CTimer::get("XIOS server").getCumulatedTime()<<endl  ;
       report(0)<<"Performance report : Time spent in processing events : "<<CTimer::get("Process events").getCumulatedTime()<<endl  ;
@@ -180,11 +176,17 @@
          {
            listenRootContext();
-           if (!finished) listenRootFinalize() ;
+           if (!finished) 
+           {
+             listenRootFinalize() ;
+           }
          }
 
          contextEventLoop() ;
          if (finished && contextList.empty()) stop=true ;
+         
          eventScheduler->checkEvent() ;
        }
+       
+       
        CTimer::get("XIOS server").suspend() ;
      }
@@ -196,5 +198,5 @@
         int flag ;
 
-        for(it=interComm.begin();it!=interComm.end();it++)
+        for(it=interComm.begin();it!=interComm.end();++it)
         {
            MPI_Status status ;
@@ -206,4 +208,5 @@
               MPI_Recv(&msg,1,MPI_INT,0,0,*it,&status) ;
               info(20)<<" CServer : Receive client finalize"<<endl ;
+
               MPI_Comm_free(&(*it));
               interComm.erase(it) ;
@@ -259,9 +262,18 @@
        {
          traceOff() ;
+         #ifdef _usingEP
+         MPI_Iprobe(-1,1,CXios::globalComm, &flag, &status) ;
+         #else
          MPI_Iprobe(MPI_ANY_SOURCE,1,CXios::globalComm, &flag, &status) ;
+         #endif
          traceOn() ;
+         
          if (flag==true)
          {
+           #ifdef _usingMPI
            rank=status.MPI_SOURCE ;
+           #elif _usingEP
+           rank= status.ep_src ;
+           #endif
            MPI_Get_count(&status,MPI_CHAR,&count) ;
            buffer=new char[count] ;
@@ -277,5 +289,9 @@
          if (flag==true)
          {
+           #ifdef _usingMPI
            rank=status.MPI_SOURCE ;
+           #elif _usingEP
+           rank= status.ep_src ;
+           #endif
            MPI_Get_count(&status,MPI_CHAR,&count) ;
            recvContextMessage((void*)buffer,count) ;
@@ -399,5 +415,5 @@
        bool finished ;
        map<string,CContext*>::iterator it ;
-       for(it=contextList.begin();it!=contextList.end();it++)
+       for(it=contextList.begin();it!=contextList.end();++it)
        {
          finished=it->second->checkBuffersAndListen();
Index: /XIOS/dev/branch_yushan_merged/src/server.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/server.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/server.hpp	(revision 1134)
@@ -7,4 +7,8 @@
 #include "mpi.hpp"
 #include "event_scheduler.hpp"
+
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace xios
Index: /XIOS/dev/branch_yushan_merged/src/test/test_client.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/test/test_client.f90	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/test/test_client.f90	(revision 1134)
@@ -35,6 +35,8 @@
 
   CALL MPI_INIT(ierr)
+  CALL init_wait
 
-  CALL init_wait
+  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
+  if(rank < 2) then
 
 !!! XIOS Initialization (get the local communicator)
@@ -72,4 +74,5 @@
 
   CALL xios_context_initialize("test",comm)
+
   CALL xios_get_handle("test",ctx_hdl)
   CALL xios_set_current_context(ctx_hdl)
@@ -125,11 +128,15 @@
   CALL xios_is_defined_field_attr("field_A",enabled=ok)
   PRINT *,"field_A : attribute enabled is defined ? ",ok
+  
   CALL xios_close_context_definition()
 
   PRINT*,"field field_A is active ? ",xios_field_is_active("field_A")
+
+  call MPI_Barrier(comm, ierr)
+
   DO ts=1,24*10
     CALL xios_update_calendar(ts)
     CALL xios_send_field("field_A",field_A)
-    CALL wait_us(5000) ;
+    CALL wait_us(5000)
   ENDDO
 
@@ -141,4 +148,13 @@
 
   CALL xios_finalize()
+  print *, "Client : xios_finalize "
+
+    else
+
+    CALL xios_init_server
+    print *, "Server : xios_finalize "
+  
+    endif
+    
 
   CALL MPI_FINALIZE(ierr)
Index: /XIOS/dev/branch_yushan_merged/src/test/test_complete.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/test/test_complete.f90	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/test/test_complete.f90	(revision 1134)
@@ -5,5 +5,5 @@
   IMPLICIT NONE
   INCLUDE "mpif.h"
-  INTEGER :: rank
+  INTEGER :: rank, size
   INTEGER :: size_loc
   INTEGER :: ierr
@@ -28,11 +28,21 @@
   INTEGER, ALLOCATABLE :: kindex(:)
   INTEGER :: ni,ibegin,iend,nj,jbegin,jend
-  INTEGER :: i,j,l,ts,n, nb_pt
+  INTEGER :: i,j,l,ts,n, nb_pt, provided
 
 !!! MPI Initialization
 
-  CALL MPI_INIT(ierr)
+  CALL MPI_INIT_THREAD(3, provided, ierr)
+    if(provided .NE. 3) then
+      print*, "provided thread level = ", provided
+      call MPI_Abort()
+    endif 
+
+  
 
   CALL init_wait
+
+  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
+  CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
+  if(rank < size-1) then
 
 !!! XIOS Initialization (get the local communicator)
@@ -220,5 +230,6 @@
 !####################################################################################
 
-    DO ts=1,24*10
+    DO ts=1,24*2
+    !DO ts=1,24
 
       CALL xios_get_handle("atmosphere",ctx_hdl)
@@ -255,17 +266,42 @@
 !!! Fin des contextes
 
-    CALL xios_context_finalize()
-    CALL xios_get_handle("atmosphere",ctx_hdl)
+
+    CALL xios_get_handle("surface",ctx_hdl) 
+
     CALL xios_set_current_context(ctx_hdl)
     CALL xios_context_finalize()
 
+    print *, "xios_context_finalize(surface)" 
+
+     CALL xios_get_handle("atmosphere",ctx_hdl)
+
+     CALL xios_set_current_context(ctx_hdl)
+
+     CALL xios_context_finalize()
+
+     print *, "xios_context_finalize(atmosphere)"
+
+    
+
+!!! Fin de XIOS
+
+    
+
+    CALL xios_finalize()
+
     DEALLOCATE(lon, lat, field_A_atm, lonvalue)
     DEALLOCATE(kindex, field_A_srf)
 
-!!! Fin de XIOS
+     print *, "Client : xios_finalize "
 
     CALL MPI_COMM_FREE(comm, ierr)
 
-    CALL xios_finalize()
+  else
+
+    CALL xios_init_server
+    print *, "Server : xios_finalize "
+  
+    endif
+
 
     CALL MPI_FINALIZE(ierr)
Index: /XIOS/dev/branch_yushan_merged/src/test/test_complete_omp.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/test/test_complete_omp.f90	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/src/test/test_complete_omp.f90	(revision 1134)
@@ -0,0 +1,345 @@
+PROGRAM test_complete_omp
+
+  USE xios
+  USE mod_wait
+  use omp_lib
+  IMPLICIT NONE
+  INCLUDE "mpif.h"
+  INTEGER :: rank, size
+  INTEGER :: size_loc
+  INTEGER :: ierr
+
+  CHARACTER(len=*),PARAMETER :: id="client"
+  INTEGER :: comm, comm2
+  TYPE(xios_duration)  :: dtime
+  TYPE(xios_context) :: ctx_hdl
+  INTEGER,PARAMETER :: ni_glo=100
+  INTEGER,PARAMETER :: nj_glo=100
+  INTEGER,PARAMETER :: llm=5
+  DOUBLE PRECISION  :: lval(llm)=1
+  TYPE(xios_field) :: field_hdl
+  TYPE(xios_fieldgroup) :: fieldgroup_hdl
+  TYPE(xios_file) :: file_hdl
+  LOGICAL :: ok
+  CHARACTER(len=256) :: crname
+
+  DOUBLE PRECISION,DIMENSION(ni_glo,nj_glo) :: lon_glo,lat_glo
+  DOUBLE PRECISION :: field_A_glo(ni_glo,nj_glo,llm)
+  DOUBLE PRECISION,ALLOCATABLE :: lon(:,:),lat(:,:),field_A_atm(:,:,:), field_A_srf(:,:), lonvalue(:,:)
+  INTEGER, ALLOCATABLE :: kindex(:)
+  INTEGER :: ni,ibegin,iend,nj,jbegin,jend
+  INTEGER :: i,j,l,ts,n, nb_pt, provided
+
+!!! MPI Initialization
+
+  CALL MPI_INIT_THREAD(3, provided, ierr)
+    if(provided .NE. 3) then
+      print*, "provided thread level = ", provided
+      call MPI_Abort()
+    endif 
+
+  
+
+  CALL init_wait
+
+  CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
+  CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
+  if(rank < size-2) then
+
+  !$omp parallel default(private)
+
+!!! XIOS Initialization (get the local communicator)
+
+  CALL xios_initialize(id,return_comm=comm)
+
+  CALL MPI_COMM_RANK(comm,rank,ierr)
+  CALL MPI_COMM_SIZE(comm,size_loc,ierr)
+
+  size_loc = size_loc*omp_get_num_threads()
+  rank = rank*omp_get_num_threads() + omp_get_thread_num()
+
+
+!###########################################################################
+! Contexte ATM
+!###########################################################################
+
+!!! Initialisation des coordonnes globales et locales pour la grille rgulire
+
+  DO j=1,nj_glo
+    DO i=1,ni_glo
+      lon_glo(i,j)=(i-1)+(j-1)*ni_glo
+      lat_glo(i,j)=1000+(i-1)+(j-1)*ni_glo
+      DO l=1,llm
+        field_A_glo(i,j,l)=(i-1)+(j-1)*ni_glo+10000*l
+      ENDDO
+    ENDDO
+  ENDDO
+  ni=ni_glo ; ibegin=0
+
+  jbegin=0
+  DO n=0,size_loc-1
+    nj=nj_glo/size_loc
+    IF (n<MOD(nj_glo,size_loc)) nj=nj+1
+    IF (n==rank) exit
+    jbegin=jbegin+nj
+  ENDDO
+
+  iend=ibegin+ni-1 ; jend=jbegin+nj-1
+
+  ALLOCATE(lon(ni,nj),lat(ni,nj),field_A_atm(0:ni+1,-1:nj+2,llm),lonvalue(ni,nj))
+  lon(:,:)=lon_glo(ibegin+1:iend+1,jbegin+1:jend+1)
+  lat(:,:)=lat_glo(ibegin+1:iend+1,jbegin+1:jend+1)
+  field_A_atm(1:ni,1:nj,:)=field_A_glo(ibegin+1:iend+1,jbegin+1:jend+1,:)
+
+
+!!! Context ATMOSPHERE
+
+  CALL xios_context_initialize("atmosphere",comm)
+
+  CALL xios_get_handle("atmosphere",ctx_hdl)
+  CALL xios_set_current_context(ctx_hdl)
+
+  CALL xios_define_calendar(type="Gregorian", &
+                            start_date=xios_date(2000, 01, 01, 00, 00, 00), &
+                            time_origin=xios_date(1999, 01, 01, 15, 00, 00))
+
+  CALL xios_set_axis_attr("axis_atm",n_glo=llm ,value=lval) ;
+
+  CALL xios_set_domain_attr("domain_atm",ni_glo=ni_glo, nj_glo=nj_glo, ibegin=ibegin, ni=ni,jbegin=jbegin,nj=nj, type='curvilinear')
+  CALL xios_set_domain_attr("domain_atm",data_dim=2, data_ibegin=-1, data_ni=ni+2, data_jbegin=-2, data_nj=nj+4)
+  CALL xios_set_domain_attr("domain_atm",lonvalue_2D=lon,latvalue_2D=lat)
+
+  CALL xios_set_domain_attr("domain_atm_zoom",ni_glo=ni_glo, nj_glo=nj_glo, ibegin=ibegin, ni=ni,jbegin=jbegin,nj=nj)
+  CALL xios_set_domain_attr("domain_atm_zoom",data_dim=2, data_ibegin=-1, data_ni=ni+2, data_jbegin=-2, data_nj=nj+4)
+  CALL xios_set_domain_attr("domain_atm_zoom",lonvalue_2D=lon,latvalue_2D=lat)
+
+
+!!! Activation du groupe field_definition
+
+  CALL xios_set_fieldgroup_attr("field_definition",enabled=.TRUE.)
+
+!!! Cration d un nouveau champ
+
+  CALL xios_get_handle("field_definition",fieldgroup_hdl)
+  CALL xios_add_child(fieldgroup_hdl,field_hdl,"field_B_atm")
+
+!!! Heritage des attributs d un autre champ
+
+  CALL xios_set_attr(field_hdl,field_ref="field_A_atm_zoom",name="field_B_atm")
+
+!!! Affectation de ce nouveau champ au fichier avec un nouveau nom
+
+  CALL xios_get_handle("output_atmosphere",file_hdl)
+  CALL xios_add_child(file_hdl,field_hdl)
+  CALL xios_set_attr(field_hdl,field_ref="field_B_atm",name="field_C_atm")
+
+!!! Definition du timestep
+
+  dtime%second=3600
+  CALL xios_set_timestep(timestep=dtime)
+
+!!! Recupration des valeurs des longitudes et de taille des domaines locaux (pour test de fonctionnalit)
+
+  ni=0 ; lonvalue(:,:)=0
+  CALL xios_get_domain_attr("domain_atm",ni=ni,lonvalue_2D=lonvalue)
+
+  !PRINT *,"ni",ni
+  !PRINT *,"lonvalue",lonvalue;
+
+!!! Fin de la definition du contexte
+
+  CALL xios_close_context_definition()
+
+   print *, "xios_close_context_definition(atmosphere)"
+
+!!! Test des valeurs des champs/fichiers
+
+  !!! Attribut defini ?
+
+  CALL xios_is_defined_field_attr("field_A_atm",enabled=ok)
+  PRINT *,"field_A_atm : attribute enabled is defined ? ",ok
+
+  !!! Recuperer la valeur d un attribut
+
+  CALL xios_get_field_attr("field_A_atm",name=crname)
+  PRINT *,"field_A_atm : attribute name is : ",TRIM(crname)
+
+  !!! Champ actif (besoin de fournir la valeur) ?
+
+    PRINT*,"field field_A_atm is active ? ",xios_field_is_active("field_A_atm")
+
+  !!! Champ defini ?
+
+    PRINT*,"field field_A_atm is valid ?",xios_is_valid_field("field_A_atm")
+
+
+!###########################################################################
+! Contexte SRF
+!###########################################################################
+
+!!! Initialisation des coordonnes globales et locales pour la grille indexee (1 point sur 2)
+
+    nb_pt=ni*nj/2
+    ALLOCATE(kindex(nb_pt),field_A_srf(nb_pt,llm))
+    DO i=1,nb_pt
+      kindex(i)=2*i-1
+    ENDDO
+    field_A_srf(1:nb_pt,:)=RESHAPE(field_A_glo(ibegin+1:iend+1:2,jbegin+1:jend+1,:),(/ nb_pt,llm /))
+
+  CALL xios_context_initialize("surface",comm)
+
+
+  CALL xios_get_handle("surface",ctx_hdl)
+  CALL xios_set_current_context(ctx_hdl)
+
+  CALL xios_define_calendar(type="Gregorian", &
+                            start_date=xios_date(2000, 01, 01, 00, 00, 00), &
+                            time_origin=xios_date(1999, 01, 01, 15, 00, 00))
+
+  CALL xios_set_axis_attr("axis_srf",n_glo=llm ,value=lval)
+  CALL xios_set_domain_attr("domain_srf",ni_glo=ni_glo, nj_glo=nj_glo, ibegin=ibegin, ni=ni,jbegin=jbegin,nj=nj, type='curvilinear')
+  CALL xios_set_domain_attr("domain_srf",data_dim=1, data_ibegin=0, data_ni=nb_pt)
+  CALL xios_set_domain_attr("domain_srf",data_i_index=kindex)
+  CALL xios_set_domain_attr("domain_srf",lonvalue_2D=lon,latvalue_2D=lat)
+
+!!! Cration d un nouveau champ
+
+  CALL xios_get_handle("field_definition",fieldgroup_hdl)
+  CALL xios_add_child(fieldgroup_hdl,field_hdl,"field_B_srf")
+
+!!! Heritage des attributs d un autre champ
+
+  CALL xios_set_attr(field_hdl,field_ref="field_A_srf",name="field_B_srf")
+
+!!! Affectation de ce nouveau champ au fichier avec un nouveau nom
+
+  CALL xios_get_handle("output_surface",file_hdl)
+  CALL xios_add_child(file_hdl,field_hdl)
+  CALL xios_set_attr(field_hdl,field_ref="field_B_srf",name="field_C_srf")
+
+!!! Definition du timestep
+
+  dtime%second=1800
+  CALL xios_set_timestep(timestep=dtime)
+
+!!! Recupration des valeurs des longitudes et de taille des domaines locaux (pour test de fonctionnalit)
+
+  ni=0 ; lonvalue(:,:)=0
+  CALL xios_get_domain_attr("domain_srf",ni=ni,lonvalue_2D=lonvalue)
+
+  !PRINT *,"ni",ni
+  !PRINT *,"lonvalue",lonvalue ;
+
+!!! Fin de la definition du contexte SRF
+
+  CALL xios_get_handle("surface",ctx_hdl)
+  CALL xios_set_current_context(ctx_hdl)
+
+  CALL xios_close_context_definition()
+
+  print *, "xios_close_context_definition(surface)", rank, size_loc 
+
+
+
+!####################################################################################
+!!! Boucle temporelle
+!####################################################################################
+
+    DO ts=1,24*2
+    !DO ts=1,24
+
+      CALL xios_get_handle("atmosphere",ctx_hdl)
+      CALL xios_set_current_context(ctx_hdl)
+
+!!! Mise a jour du pas de temps
+
+      CALL xios_update_calendar(ts)
+
+!!! On donne la valeur du champ atm
+
+      CALL xios_send_field("field_A_atm",field_A_atm)
+
+!!! On change de contexte
+
+      CALL xios_get_handle("surface",ctx_hdl)
+      CALL xios_set_current_context(ctx_hdl)
+
+!!! Mise a jour du pas de temps
+
+      CALL xios_update_calendar(ts)
+
+!!! On donne la valeur du champ srf
+
+      CALL xios_send_field("field_A_srf",field_A_srf)
+
+      CALL wait_us(5000) ;
+
+
+    ENDDO
+
+     print *, "end temporal loop"
+
+!####################################################################################
+!!! Finalisation
+!####################################################################################
+
+!!! Fin des contextes
+
+     CALL xios_get_handle("surface",ctx_hdl) 
+
+     CALL xios_set_current_context(ctx_hdl)
+
+     CALL xios_context_finalize()
+
+     print *, "xios_context_finalize(surface)" 
+
+     CALL xios_get_handle("atmosphere",ctx_hdl)
+
+     CALL xios_set_current_context(ctx_hdl)
+
+     CALL xios_context_finalize()
+
+     print *, "xios_context_finalize(atmosphere)"
+
+    
+
+!!! Fin de XIOS
+
+    
+
+    CALL xios_finalize()
+
+    DEALLOCATE(lon, lat, field_A_atm, lonvalue)
+    DEALLOCATE(kindex, field_A_srf)
+
+    print *, "Client : xios_finalize "
+
+    !$omp barrier
+
+    !$omp master 
+    CALL MPI_COMM_FREE(comm, ierr)
+    !$omp end master
+
+    !$omp barrier
+
+   
+
+  !$omp end parallel
+
+  else
+
+    CALL xios_init_server
+    print *, "Server : xios_finalize "
+  
+    endif
+
+
+    CALL MPI_FINALIZE(ierr)
+
+  END PROGRAM test_complete_omp
+
+
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90	(revision 1134)
+++ /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90	(revision 1134)
@@ -0,0 +1,194 @@
+PROGRAM test_omp
+
+  USE xios
+  USE mod_wait
+  use omp_lib
+  IMPLICIT NONE
+  INCLUDE "mpif.h"
+  INTEGER :: rank
+  INTEGER :: size
+  INTEGER :: ierr
+
+  CHARACTER(len=*),PARAMETER :: id="client"
+  INTEGER :: comm
+  TYPE(xios_duration) :: dtime
+  CHARACTER(len=20) :: dtime_str
+  TYPE(xios_date) :: date
+  CHARACTER(len=20) :: date_str
+  CHARACTER(len=15) :: calendar_type
+  TYPE(xios_context) :: ctx_hdl
+  INTEGER,PARAMETER :: ni_glo=100
+  INTEGER,PARAMETER :: nj_glo=100
+  INTEGER,PARAMETER :: llm=5
+  DOUBLE PRECISION  :: lval(llm)=1
+  TYPE(xios_field) :: field_hdl
+  TYPE(xios_fieldgroup) :: fieldgroup_hdl
+  TYPE(xios_file) :: file_hdl
+  LOGICAL :: ok
+
+  DOUBLE PRECISION,DIMENSION(ni_glo,nj_glo) :: lon_glo,lat_glo
+  DOUBLE PRECISION :: field_A_glo(ni_glo,nj_glo,llm)
+  DOUBLE PRECISION,ALLOCATABLE :: lon(:,:),lat(:,:),field_A(:,:,:), lonvalue(:,:) ;
+  INTEGER :: ni,ibegin,iend,nj,jbegin,jend
+  INTEGER :: i,j,l,ts,n, provided
+
+!!! MPI Initialization   
+
+    CALL MPI_INIT_THREAD(3, provided, ierr)
+    if(provided .NE. 3) then
+      print*, "provided thread level = ", provided
+      call MPI_Abort()
+    endif
+
+    CALL init_wait
+
+    CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
+    CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
+    if(rank < size-2) then
+
+    !$omp parallel default(private)
+  
+    CALL xios_initialize(id,return_comm=comm)
+  
+    CALL MPI_COMM_RANK(comm,rank,ierr)
+    CALL MPI_COMM_SIZE(comm,size,ierr)
+
+    size = size*omp_get_num_threads()
+    rank = rank*omp_get_num_threads() + omp_get_thread_num()
+
+    DO j=1,nj_glo
+      DO i=1,ni_glo
+        lon_glo(i,j)=(i-1)+(j-1)*ni_glo
+        lat_glo(i,j)=1000+(i-1)+(j-1)*ni_glo
+        DO l=1,llm
+          field_A_glo(i,j,l)=(i-1)+(j-1)*ni_glo+10000*l
+        ENDDO
+      ENDDO
+    ENDDO
+    ni=ni_glo ; ibegin=0
+
+    jbegin=0
+    DO n=0,size-1
+      nj=nj_glo/size
+      IF (n<MOD(nj_glo,size)) nj=nj+1
+      IF (n==rank) exit
+      jbegin=jbegin+nj
+    ENDDO
+
+    iend=ibegin+ni-1 ; jend=jbegin+nj-1
+
+    ALLOCATE(lon(ni,nj),lat(ni,nj),field_A(0:ni+1,-1:nj+2,llm),lonvalue(ni,nj))
+    lon(:,:)=lon_glo(ibegin+1:iend+1,jbegin+1:jend+1)
+    lat(:,:)=lat_glo(ibegin+1:iend+1,jbegin+1:jend+1)
+    field_A(1:ni,1:nj,:)=field_A_glo(ibegin+1:iend+1,jbegin+1:jend+1,:)
+
+    !print*, "xios init OK", rank, size
+
+    CALL xios_context_initialize("test",comm)
+
+    !print*, "xios_context_initialize OK", rank, size
+
+    CALL xios_get_handle("test",ctx_hdl)
+    CALL xios_set_current_context(ctx_hdl)
+  
+  
+    CALL xios_get_calendar_type(calendar_type)
+    !print*, "xios_get_calendar_type OK", rank, size
+
+    CALL xios_set_axis_attr("axis_A",n_glo=llm ,value=lval) ;
+    CALL xios_set_domain_attr("domain_A",ni_glo=ni_glo, nj_glo=nj_glo, ibegin=ibegin, ni=ni,jbegin=jbegin,nj=nj,type='curvilinear')
+    CALL xios_set_domain_attr("domain_A",data_dim=2, data_ibegin=-1, data_ni=ni+2, data_jbegin=-2, data_nj=nj+4)
+    CALL xios_set_domain_attr("domain_A",lonvalue_2D=lon,latvalue_2D=lat)
+    CALL xios_set_fieldgroup_attr("field_definition",enabled=.TRUE.)
+    !print*, "test block OK", rank, size
+
+    CALL xios_get_handle("field_definition",fieldgroup_hdl)
+    CALL xios_add_child(fieldgroup_hdl,field_hdl,"field_B")
+    CALL xios_set_attr(field_hdl,field_ref="field_A",name="field_B")
+
+    CALL xios_get_handle("output",file_hdl)
+    CALL xios_add_child(file_hdl,field_hdl)
+    CALL xios_set_attr(field_hdl,field_ref="field_A_zoom",name="field_C")
+
+    dtime%second = 3600
+    CALL xios_set_timestep(dtime)
+    !print*, "xios_set_timestep OK", rank, size    
+
+   ! The calendar is created as soon as the calendar type is defined. This way
+   ! calendar operations can be used before the context definition is closed
+   CALL xios_get_time_origin(date)
+   !PRINT *, "--> year length = ", xios_get_year_length_in_seconds(date%year)
+   !PRINT *, "--> day length = ", xios_get_day_length_in_seconds()
+   CALL xios_date_convert_to_string(date, date_str)
+   !PRINT *, "time_origin = ", date_str
+   !PRINT *, "xios_date_get_second_of_year(time_origin) = ", xios_date_get_second_of_year(date)
+   !PRINT *, "xios_date_get_day_of_year(time_origin) = ", xios_date_get_day_of_year(date)
+   !PRINT *, "xios_date_get_fraction_of_year(time_origin) = ", xios_date_get_fraction_of_year(date)
+   !PRINT *, "xios_date_get_second_of_day(time_origin) = ", xios_date_get_second_of_day(date)
+   !PRINT *, "xios_date_get_fraction_of_day(time_origin) = ", xios_date_get_fraction_of_day(date)
+   dtime%timestep = 1
+   dtime = 0.5 * dtime
+   CALL xios_duration_convert_to_string(dtime, dtime_str)
+   !PRINT *, "duration = ", dtime_str
+   date = date + 3 * (dtime + dtime)
+   CALL xios_date_convert_to_string(date, date_str)
+   !PRINT *, "date = time_origin + 3 * (duration + duration) = ", date_str
+   !PRINT *, "xios_date_convert_to_seconds(date) = ", xios_date_convert_to_seconds(date)
+   !PRINT *, "xios_date_convert_to_seconds(date - 2.5h) = ", xios_date_convert_to_seconds(date - 2.5 * xios_hour)
+
+   ni=0 ; lonvalue(:,:)=0;
+   CALL xios_get_domain_attr("domain_A",ni=ni,lonvalue_2D=lonvalue)
+   !print *,"ni",ni
+   !print *,"lonvalue",lonvalue;
+
+   CALL xios_is_defined_field_attr("field_A",enabled=ok)
+   !PRINT *,"field_A : attribute enabled is defined ? ",ok 
+
+   CALL xios_close_context_definition()
+   !print*, "xios_close_context_definition OK"  
+
+   PRINT*,"field field_A is active ? ",xios_field_is_active("field_A")
+
+   DO ts=1,6
+    CALL xios_update_calendar(ts)
+    CALL xios_send_field("field_A",field_A)
+    CALL wait_us(5000)
+   ENDDO
+
+    CALL xios_context_finalize()
+    !print*, "xios_context_finalize OK", rank, size
+
+    CALL xios_finalize()
+    print*, "xios finalize OK", rank, size
+
+    DEALLOCATE(lon, lat, field_A, lonvalue)
+    !$omp master 
+    !call MPI_Barrier(comm)
+    CALL MPI_COMM_FREE(comm, ierr)
+    !$omp end master
+
+     !$omp barrier
+
+     !print*, "MPI_COMM_FREE OK", rank, size
+
+
+
+  !$omp end parallel
+
+
+    else
+
+    CALL xios_init_server
+    print *, "Server : xios_finalize "
+  
+    endif
+    
+
+  CALL MPI_FINALIZE(ierr)
+
+END PROGRAM test_omp
+
+
+
+
+
Index: /XIOS/dev/branch_yushan_merged/src/timer.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/timer.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/timer.cpp	(revision 1134)
@@ -8,4 +8,5 @@
 {
   std::map<std::string,CTimer> CTimer::allTimer;
+  std::map<std::string,CTimer> *CTimer::allTimer_ptr = 0;
   
   CTimer::CTimer(const std::string& name_) : name(name_) 
@@ -52,7 +53,15 @@
   CTimer& CTimer::get(const std::string name)
   {
-    std::map<std::string,CTimer>::iterator it = allTimer.find(name);
-    if (it == allTimer.end())
-      it = allTimer.insert(std::make_pair(name, CTimer(name))).first;
+    // bkp
+    // std::map<std::string,CTimer>::iterator it = allTimer.find(name);
+    // if (it == allTimer.end())
+    //   it = allTimer.insert(std::make_pair(name, CTimer(name))).first;
+    // return it->second;
+
+    if(allTimer_ptr == NULL) allTimer_ptr = new std::map<std::string,CTimer>;
+
+    std::map<std::string,CTimer>::iterator it = (*allTimer_ptr).find(name);
+    if (it == (*allTimer_ptr).end())
+      it = (*allTimer_ptr).insert(std::make_pair(name, CTimer(name))).first;
     return it->second;
   }
Index: /XIOS/dev/branch_yushan_merged/src/timer.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/timer.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/timer.hpp	(revision 1134)
@@ -21,4 +21,8 @@
       double getCumulatedTime(void);
       static std::map<std::string,CTimer> allTimer;
+
+      static std::map<std::string,CTimer> *allTimer_ptr;
+      #pragma omp threadprivate(allTimer_ptr)
+      
       static double getTime(void);
       static CTimer& get(std::string name);
Index: /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.cpp	(revision 1134)
@@ -9,5 +9,7 @@
 
 CReductionAlgorithm::CallBackMap* CReductionAlgorithm::reductionCreationCallBacks_ = 0;
-std::map<StdString,EReductionType> CReductionAlgorithm::ReductionOperations = std::map<StdString,EReductionType>();
+//std::map<StdString,EReductionType> CReductionAlgorithm::ReductionOperations = std::map<StdString,EReductionType>();
+std::map<StdString,EReductionType> *CReductionAlgorithm::ReductionOperations_ptr = 0; 
+
 bool CReductionAlgorithm::initReductionOperation(std::map<StdString,EReductionType>& m)
 {
@@ -29,9 +31,34 @@
 }
 
-bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation(CReductionAlgorithm::ReductionOperations);
+
+bool CReductionAlgorithm::initReductionOperation()
+{
+  if(CReductionAlgorithm::ReductionOperations_ptr == NULL) CReductionAlgorithm::ReductionOperations_ptr = new std::map<StdString,EReductionType>();
+  // So so stupid way to intialize operation but it works ...
+  (*CReductionAlgorithm::ReductionOperations_ptr)["sum"] = TRANS_REDUCE_SUM;
+  CSumReductionAlgorithm::registerTrans();
+
+  (*CReductionAlgorithm::ReductionOperations_ptr)["min"] = TRANS_REDUCE_MIN;
+  CMinReductionAlgorithm::registerTrans();
+
+  (*CReductionAlgorithm::ReductionOperations_ptr)["max"] = TRANS_REDUCE_MAX;
+  CMaxReductionAlgorithm::registerTrans();
+
+  (*CReductionAlgorithm::ReductionOperations_ptr)["extract"] = TRANS_REDUCE_EXTRACT;
+  CExtractReductionAlgorithm::registerTrans();
+
+  (*CReductionAlgorithm::ReductionOperations_ptr)["average"] = TRANS_REDUCE_AVERAGE;
+  CAverageReductionAlgorithm::registerTrans();
+}
+
+//bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation(CReductionAlgorithm::ReductionOperations);
+bool CReductionAlgorithm::_dummyInit = CReductionAlgorithm::initReductionOperation();
 
 CReductionAlgorithm* CReductionAlgorithm::createOperation(EReductionType reduceType)
 {
   int reduceTypeInt = reduceType;
+  //if (0 == reductionCreationCallBacks_)
+  //  reductionCreationCallBacks_ = new CallBackMap();
+
   CallBackMap::const_iterator it = (*reductionCreationCallBacks_).find(reduceType);
   if ((*reductionCreationCallBacks_).end() == it)
Index: /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/Functions/reduction.hpp	(revision 1134)
@@ -23,5 +23,7 @@
 {
 public:
-  static std::map<StdString,EReductionType> ReductionOperations;
+  //static std::map<StdString,EReductionType> ReductionOperations;
+  static std::map<StdString,EReductionType> *ReductionOperations_ptr;
+  #pragma omp threadprivate(ReductionOperations_ptr)
 
 public:
@@ -60,4 +62,5 @@
   typedef std::map<EReductionType, CreateOperationCallBack> CallBackMap;
   static CallBackMap* reductionCreationCallBacks_;
+  #pragma omp threadprivate(reductionCreationCallBacks_)
 
   static bool registerOperation(EReductionType reduceType, CreateOperationCallBack createFn);
@@ -66,5 +69,7 @@
 protected:
   static bool initReductionOperation(std::map<StdString,EReductionType>& m);
+  static bool initReductionOperation();
   static bool _dummyInit;
+  #pragma omp threadprivate(_dummyInit)
 };
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_extract_domain.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_extract_domain.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_extract_domain.cpp	(revision 1134)
@@ -62,5 +62,5 @@
 
   pos_ = algo->position;
-  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
 }
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.cpp	(revision 1134)
@@ -173,6 +173,6 @@
 
   // Sending global index of grid source to corresponding process as well as the corresponding mask
-  std::vector<MPI_Request> requests;
-  std::vector<MPI_Status> status;
+  std::vector<ep_lib::MPI_Request> requests;
+  std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc;
   boost::unordered_map<int, double* > sendValueToDest;
@@ -184,5 +184,5 @@
     sendValueToDest[recvRank] = new double [recvSize];
 
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests.back());
   }
@@ -206,5 +206,5 @@
 
     // Send global index source and mask
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests.back());
   }
@@ -213,7 +213,6 @@
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
 
-
-  std::vector<MPI_Request>().swap(requests);
-  std::vector<MPI_Status>().swap(status);
+  std::vector<ep_lib::MPI_Request>().swap(requests);
+  std::vector<ep_lib::MPI_Status>().swap(status);
 
   // Okie, on destination side, we will wait for information of masked index of source
@@ -223,5 +222,5 @@
     int recvSize = itSend->second;
 
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Irecv(recvValueFromSrc[recvRank], recvSize, MPI_DOUBLE, recvRank, 48, client->intraComm, &requests.back());
   }
@@ -241,10 +240,9 @@
     }
     // Okie, now inform the destination which source index are masked
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendValueToDest[recvRank], recvSize, MPI_DOUBLE, recvRank, 48, client->intraComm, &requests.back());
   }
   status.resize(requests.size());
   MPI_Waitall(requests.size(), &requests[0], &status[0]);
-
 
   size_t nGloAxisDest = axisDest_->n_glo.getValue() - 1;
Index: /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_inverse.hpp	(revision 1134)
@@ -12,5 +12,8 @@
 #include "axis_algorithm_transformation.hpp"
 #include "transformation.hpp"
-
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
+   
 namespace xios {
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_reduce_domain.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_reduce_domain.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/axis_algorithm_reduce_domain.cpp	(revision 1134)
@@ -70,5 +70,5 @@
 
   dir_ = (CReduceDomainToAxis::direction_attr::iDir == algo->direction)  ? iDir : jDir;
-  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
 }
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.cpp	(revision 1134)
@@ -405,10 +405,10 @@
   CContextClient* client=context->client;
 
-  MPI_Comm poleComme(MPI_COMM_NULL);
-  MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? MPI_UNDEFINED : 1, 0, &poleComme);
+  ep_lib::MPI_Comm poleComme(MPI_COMM_NULL);
+  ep_lib::MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? MPI_UNDEFINED : 1, 0, &poleComme);
   if (MPI_COMM_NULL != poleComme)
   {
     int nbClientPole;
-    MPI_Comm_size(poleComme, &nbClientPole);
+    ep_lib::MPI_Comm_size(poleComme, &nbClientPole);
 
     std::map<int,std::vector<std::pair<int,double> > >::iterator itePole = interMapValuePole.end(), itPole,
@@ -573,5 +573,5 @@
   double* sendWeightBuff = new double [sendBuffSize];
 
-  std::vector<MPI_Request> sendRequest;
+  std::vector<ep_lib::MPI_Request> sendRequest;
 
   int sendOffSet = 0, l = 0;
@@ -594,5 +594,5 @@
     }
 
-    sendRequest.push_back(MPI_Request());
+    sendRequest.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendIndexDestBuff + sendOffSet,
              k,
@@ -602,5 +602,5 @@
              client->intraComm,
              &sendRequest.back());
-    sendRequest.push_back(MPI_Request());
+    sendRequest.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendIndexSrcBuff + sendOffSet,
              k,
@@ -610,5 +610,5 @@
              client->intraComm,
              &sendRequest.back());
-    sendRequest.push_back(MPI_Request());
+    sendRequest.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendWeightBuff + sendOffSet,
              k,
@@ -629,5 +629,5 @@
   while (receivedSize < recvBuffSize)
   {
-    MPI_Status recvStatus;
+    ep_lib::MPI_Status recvStatus;
     MPI_Recv((recvIndexDestBuff + receivedSize),
              recvBuffSize,
@@ -640,6 +640,9 @@
     int countBuff = 0;
     MPI_Get_count(&recvStatus, MPI_INT, &countBuff);
+    #ifdef _usingMPI
     clientSrcRank = recvStatus.MPI_SOURCE;
-
+    #elif _usingEP
+    clientSrcRank = recvStatus.ep_src;
+    #endif
     MPI_Recv((recvIndexSrcBuff + receivedSize),
              recvBuffSize,
@@ -666,6 +669,7 @@
   }
 
-  std::vector<MPI_Status> requestStatus(sendRequest.size());
-  MPI_Waitall(sendRequest.size(), &sendRequest[0], MPI_STATUS_IGNORE);
+  std::vector<ep_lib::MPI_Status> requestStatus(sendRequest.size());
+  ep_lib::MPI_Status stat_ignore;
+  MPI_Waitall(sendRequest.size(), &sendRequest[0], &stat_ignore);
 
   delete [] sendIndexDestBuff;
@@ -758,5 +762,5 @@
 
   MPI_Allreduce(&localNbWeight, &globalNbWeight, 1, MPI_LONG, MPI_SUM, client->intraComm);
-  MPI_Scan(&localNbWeight, &startIndex, 1, MPI_LONG, MPI_SUM, client->intraComm);
+  ep_lib::MPI_Scan(&localNbWeight, &startIndex, 1, MPI_LONG, MPI_SUM, client->intraComm);
   
   if (0 == globalNbWeight)
@@ -771,6 +775,6 @@
   std::vector<StdSize> start(1, startIndex - localNbWeight);
   std::vector<StdSize> count(1, localNbWeight);
-  
-  WriteNetCdf netCdfWriter(filename, client->intraComm);  
+
+  WriteNetCdf netCdfWriter(filename, static_cast<MPI_Comm>(client->intraComm.mpi_comm));
 
   // Define some dimensions
Index: /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.hpp	(revision 1134)
@@ -13,4 +13,7 @@
 #include "transformation.hpp"
 #include "nc4_data_output.hpp"
+#ifdef _usingEP
+#include "ep_declaration.hpp"
+#endif
 
 namespace xios {
Index: /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp	(revision 1134)
@@ -473,6 +473,6 @@
 
   // Sending global index of grid source to corresponding process as well as the corresponding mask
-  std::vector<MPI_Request> requests;
-  std::vector<MPI_Status> status;
+  std::vector<ep_lib::MPI_Request> requests;
+  std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned char* > recvMaskDst;
   boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc;
@@ -484,7 +484,7 @@
     recvGlobalIndexSrc[recvRank] = new unsigned long [recvSize];
 
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests.back());
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests.back());
   }
@@ -522,7 +522,7 @@
 
     // Send global index source and mask
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests.back());
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests.back());
   }
@@ -532,6 +532,6 @@
 
   // Okie, now use the mask to identify which index source we need to send, then also signal the destination which masked index we will return
-  std::vector<MPI_Request>().swap(requests);
-  std::vector<MPI_Status>().swap(status);
+  std::vector<ep_lib::MPI_Request>().swap(requests);
+  std::vector<ep_lib::MPI_Status>().swap(status);
   // Okie, on destination side, we will wait for information of masked index of source
   for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend)
@@ -540,5 +540,5 @@
     int recvSize = itSend->second;
 
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back());
   }
@@ -577,5 +577,5 @@
 
     // Okie, now inform the destination which source index are masked
-    requests.push_back(MPI_Request());
+    requests.push_back(ep_lib::MPI_Request());
     MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back());
   }
Index: /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation_factory_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation_factory_impl.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation_factory_impl.hpp	(revision 1134)
@@ -57,7 +57,10 @@
   typedef std::map<ETranformationType, CreateTransformationCallBack> CallBackMap;
   static CallBackMap* transformationCreationCallBacks_;
+  #pragma omp threadprivate(transformationCreationCallBacks_)
+  
   static bool registerTransformation(ETranformationType transType, CreateTransformationCallBack createFn);
   static bool unregisterTransformation(ETranformationType transType);
   static bool initializeTransformation_;
+  #pragma omp threadprivate(initializeTransformation_)
 };
 
@@ -79,4 +82,6 @@
                                                                                std::map<int, int>& elementPositionInGridDst2DomainPosition)
 {
+  if (0 == transformationCreationCallBacks_)
+    transformationCreationCallBacks_ = new CallBackMap();
   typename CallBackMap::const_iterator it = (*transformationCreationCallBacks_).find(transType);
   if ((*transformationCreationCallBacks_).end() == it)
Index: /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_extract_axis.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_extract_axis.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_extract_axis.cpp	(revision 1134)
@@ -49,5 +49,5 @@
   StdString op = "extract";
   pos_ = algo->position;
-  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
 }
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_axis.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_axis.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_axis.cpp	(revision 1134)
@@ -75,5 +75,5 @@
   }
   
-  if (CReductionAlgorithm::ReductionOperations.end() == CReductionAlgorithm::ReductionOperations.find(op))
+  if ((*CReductionAlgorithm::ReductionOperations_ptr).end() == (*CReductionAlgorithm::ReductionOperations_ptr).find(op))
     ERROR("CScalarAlgorithmReduceAxis::CScalarAlgorithmReduceAxis(CAxis* axisDestination, CAxis* axisSource, CReduceAxisToScalar* algo)",
        << "Operation '" << op << "' not found. Please make sure to use a supported one"
@@ -81,5 +81,5 @@
        << "Scalar destination " << scalarDestination->getId());
 
-  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
 }
 
Index: /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_domain.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_domain.cpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/scalar_algorithm_reduce_domain.cpp	(revision 1134)
@@ -69,5 +69,5 @@
   }
   
-  if (CReductionAlgorithm::ReductionOperations.end() == CReductionAlgorithm::ReductionOperations.find(op))
+  if ((*CReductionAlgorithm::ReductionOperations_ptr).end() == (*CReductionAlgorithm::ReductionOperations_ptr).find(op))
     ERROR("CScalarAlgorithmReduceDomain::CScalarAlgorithmReduceDomain(CDomain* domainDestination, CDomain* domainSource, CReduceDomainToScalar* algo)",
        << "Operation '" << op << "' not found. Please make sure to use a supported one"
@@ -75,5 +75,5 @@
        << "Scalar destination " << scalarDestination->getId());
 
-  reduction_ = CReductionAlgorithm::createOperation(CReductionAlgorithm::ReductionOperations[op]);
+  reduction_ = CReductionAlgorithm::createOperation((*CReductionAlgorithm::ReductionOperations_ptr)[op]);
 }
 
Index: /XIOS/dev/branch_yushan_merged/src/type/type.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/type/type.hpp	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/type/type.hpp	(revision 1134)
@@ -94,5 +94,5 @@
     const CType_ref& operator = (CType<T>& val) const ;
     const CType_ref& operator = (const CType_ref& val) const;
-    operator T&() const;    
+    operator T&() const;
 
     inline virtual CBaseType* clone(void) const   { return _clone(); }
Index: /XIOS/dev/branch_yushan_merged/src/xios_server.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/xios_server.f90	(revision 1133)
+++ /XIOS/dev/branch_yushan_merged/src/xios_server.f90	(revision 1134)
@@ -1,9 +1,15 @@
 PROGRAM server_main
   USE xios
+  USE mod_wait
   IMPLICIT NONE
   INCLUDE "mpif.h"
-  INTEGER :: ierr
-  
+  INTEGER :: ierr, th_level
+    
+    CALL MPI_INIT(ierr)
+    !CALL MPI_INIT_thread(3, th_level, ierr)
+    CALL init_wait
     CALL xios_init_server
+    
+    CALL MPI_FINALIZE(ierr)
 
   END PROGRAM server_main
