Index: /XIOS/dev/branch_yushan_merged/bld.cfg
===================================================================
--- /XIOS/dev/branch_yushan_merged/bld.cfg	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/bld.cfg	(revision 1196)
@@ -38,5 +38,5 @@
 #bld::target test_expand_domain.exe
 #bld::target test_new_features.exe test_unstruct_complete.exe 
-bld::target test_omp.exe #test_complete_omp.exe test_remap_omp.exe test_unstruct_omp.exe
+bld::target test_omp.exe test_complete_omp.exe test_remap_omp.exe test_unstruct_omp.exe
 #bld::target test_client.exe test_complete.exe #test_xios2_cmip6.exe
 #bld::target test_connectivity_expand.exe
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_fortran.cpp	(revision 1196)
@@ -31,5 +31,5 @@
       {
         fc_comm_map.insert(std::make_pair( std::make_pair( fint, omp_get_thread_num()) , comm));
-        //printf("EP_Comm_c2f : MAP insert: %d, %d, %p\n", fint, omp_get_thread_num(), comm.ep_comm_ptr);
+        //printf("EP_Comm_c2f : MAP %p insert: %d, %d, %p\n", &fc_comm_map, fint, omp_get_thread_num(), comm.ep_comm_ptr);
       }
     }
@@ -54,5 +54,5 @@
       MPI_Comm comm_ptr;
       comm_ptr = it->second;
-      //printf("EP_Comm_f2c : MAP find: %d, %d, %p\n", it->first.first, it->first.second, comm_ptr.ep_comm_ptr);
+      //printf("EP_Comm_f2c : MAP %p find: %d, %d, %p\n", &fc_comm_map, it->first.first, it->first.second, comm_ptr.ep_comm_ptr);
       return  comm_ptr;
     }
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm.cpp	(revision 1196)
@@ -43,6 +43,7 @@
       MPI_Isend(&leader_ranks[0], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_s);
       MPI_Status status;
+      MPI_Wait(&req_s, &status);
+      
       MPI_Irecv(&leader_ranks[3], 3, MPI_INT_STD, remote_leader, tag, peer_comm, &req_r);
-      MPI_Wait(&req_s, &status);
       MPI_Wait(&req_r, &status);
     }
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.cpp	(revision 1196)
@@ -7,4 +7,6 @@
 using namespace std;
 
+std::list< ep_lib::MPI_Request* > ** EP_PendingRequests = 0;
+#pragma omp threadprivate(EP_PendingRequests)
 
 namespace ep_lib
@@ -429,2 +431,3 @@
 
 
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_lib.hpp	(revision 1196)
@@ -62,4 +62,5 @@
 
   int Message_Check(MPI_Comm comm);
+  int Request_Check();
 
   int MPI_Recv  (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status);
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_message.cpp	(revision 1196)
@@ -11,4 +11,7 @@
 
 using namespace std;
+
+extern std::list< ep_lib::MPI_Request* > * EP_PendingRequests;
+#pragma omp threadprivate(EP_PendingRequests)
 
 namespace ep_lib
@@ -248,5 +251,5 @@
           #pragma omp flush
           ptr_comm_target->ep_comm_ptr->message_queue->push_back(*msg_block);
-          printf("probed one message, ep_src = %d, tag = %d, mpi_status = %p (%p), message = %d\n", msg_block->ep_src, msg_block->ep_tag, msg_block->mpi_status, &status, msg_block->mpi_message);
+          //printf("probed one message, ep_src = %d, tag = %d, mpi_status = %p (%p), message = %d\n", msg_block->ep_src, msg_block->ep_tag, msg_block->mpi_status, &status, msg_block->mpi_message);
           #pragma omp flush
         }
@@ -261,3 +264,34 @@
   }
 
+  int Request_Check()
+  {
+    MPI_Status status;
+    MPI_Message message;
+    int probed = false;
+    int recv_count = 0;
+    std::list<MPI_Request* >::iterator it;
+    
+    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
+    { 
+      Message_Check((*it)->comm);
+    }
+
+
+    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); )
+    {
+      MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &message, &status);
+      if(probed)
+      {
+        MPI_Get_count(&status, (*it)->ep_datatype, &recv_count);
+        MPI_Imrecv((*it)->buf, recv_count, (*it)->ep_datatype, &message, *it);
+        (*it)->type = 3;
+        //printf("request add = %p, mpi_request=%d\n", *it, (*it)->mpi_request);
+        EP_PendingRequests->erase(it);
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      it++;
+    }
+  }
+
 }
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_recv.cpp	(revision 1196)
@@ -13,4 +13,6 @@
 using namespace std;
 
+extern std::list< ep_lib::MPI_Request* > * EP_PendingRequests;
+#pragma omp threadprivate(EP_PendingRequests)
 
 namespace ep_lib 
@@ -54,4 +56,6 @@
     int dest_rank;
     MPI_Comm_rank(comm, &dest_rank);
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD_STD, &world_rank);
 
     if(!comm.is_ep)
@@ -83,51 +87,48 @@
     if(EP_PendingRequests == 0 ) 
     {
-      EP_PendingRequests = new std::list< MPI_Request* >; 
-      printf("proc %d : EP_PendingRequests allocated, add = %p\n", dest_rank, EP_PendingRequests);  
+      EP_PendingRequests = new std::list< MPI_Request* >;
+      //printf("proc %d(%d) : EP_PendingRequests allocated, add = %p\n", dest_rank, world_rank, EP_PendingRequests);  
     }
 
-    request->pending_ptr = EP_PendingRequests;
-    printf("proc %d : &EP_PendingRequests add = %p, ptr = %p\n", dest_rank, EP_PendingRequests, request->pending_ptr);  
+
+    EP_PendingRequests->push_back(request);
+
+    Request_Check();
+    //printf("proc %d(%d) : EP_PendingRequests insert one request, src = %d(%d), tag = %d(%d), size = %d; request add = %p\n", 
+    //        dest_rank, world_rank, EP_PendingRequests->back()->ep_src, request->ep_src, 
+    //        EP_PendingRequests->back()->ep_tag, request->ep_tag, 
+    //        EP_PendingRequests->size(), request);
     
-    EP_PendingRequests->push_back(request);
-    //printf("proc %d : EP_PendingRequests insert one request, src = %d, tag = %d, size = %d\n", dest_rank, request->ep_src, request->ep_tag, EP_PendingRequests->size());
+    // check all EP_PendingRequests      
+    //for(std::list<MPI_Request* >::iterator it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); )
+    //{
+    //if((*it)->type == 3) 
+    //{
+    //    EP_PendingRequests->erase(it);
+   //     it = EP_PendingRequests->begin();
+    //    continue;
+     // }
+        
+      //int probed = false;
+      //MPI_Message pending_message;
+      //MPI_Status pending_status;
     
-    // check all EP_PendingRequests
+      //MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &pending_message, &pending_status);
     
-    //printf("proc %d have %d pending irecv request\n", dest_rank, EP_PendingRequests->size());
-      
-    for(std::list<MPI_Request* >::iterator it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); )
-    {
-      if((*it)->type == 3) 
-      {
-        //printf("proc %d : pending request type = %d, src= %d, tag = %d    skip\n", dest_rank, (*it)->type, (*it)->ep_src, (*it)->ep_tag);
-        EP_PendingRequests->erase(it);
-        it = EP_PendingRequests->begin();
-        //printf("proc %d : pending request processed, size = %d\n", dest_rank, EP_PendingRequests->size());
-        continue;
-      }
+      //if(probed) 
+      //{ 
+        //int count;
+        //MPI_Get_count(&pending_status, (*it)->ep_datatype, &count);
+        //MPI_Imrecv((*it)->buf, count, (*it)->ep_datatype, &pending_message, *it);
+
+        //EP_PendingRequests->erase(it);
+        //if(EP_PendingRequests->empty()) return 0;
         
-      //printf("proc %d : pending irecv request src = %d, tag = %d, type = %d\n", dest_rank, (*it)->ep_src, (*it)->ep_tag, (*it)->type);
-      int probed = false;
-      MPI_Message pending_message;
-      MPI_Status pending_status;
-    
-      MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &pending_message, &pending_status);
-      //printf("proc %d : pending irecv request probed to be %d, src = %d, tag = %d\n",dest_rank, probed, (*it)->ep_src, (*it)->ep_tag);
-    
-      if(probed) 
-      { 
-        int count;
-        MPI_Get_count(&pending_status, (*it)->ep_datatype, &count);
-        MPI_Imrecv((*it)->buf, count, (*it)->ep_datatype, &pending_message, *it);
-        //printf("proc %d : pending request is imrecving src = %d, tag = %d, count = %d\n", dest_rank, (*it)->ep_src, (*it)->ep_tag, count);
-        EP_PendingRequests->erase(it);
-        it = EP_PendingRequests->begin();
-        //printf("proc %d : pending request processed, size = %d\n", dest_rank, EP_PendingRequests->size());
-        continue;
-      }
+        //it = EP_PendingRequests->begin();
+        //continue;
+     // }
 
-      it++;
-    }
+      //it++;
+   // }
     
     return 0;
@@ -149,5 +150,4 @@
     request->ep_tag = message->ep_tag;
     request->ep_src = message->ep_src;
-    //request->buf = buf;
 
     return 0;
@@ -177,2 +177,3 @@
 
 
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_send.cpp	(revision 1196)
@@ -50,4 +50,24 @@
     return 0;
   }
+  
+  int MPI_Bsend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)
+  {
+    if(!comm.is_ep)
+    {
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Bsend(buf, count, static_cast< ::MPI_Datatype>(datatype), dest, tag, mpi_comm);
+      return 0;
+    }
+
+    MPI_Request request;
+    MPI_Status status;
+    //MPI_Ibsend(buf, count, datatype, dest, tag, comm, &request);
+    MPI_Wait(&request, &status);
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm);
+
+    return 0;
+  }
+ 
 
 
@@ -103,5 +123,5 @@
     request->buf = const_cast<void*>(buf);
 
-    Message_Check(comm);
+    //Message_Check(comm);
 
     return 0;
@@ -166,4 +186,59 @@
     return 0;
   }
+  
+  int MPI_Ibsend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    Debug("\nMPI_Isend with EP\n");
+    int src_rank;
+    MPI_Comm_rank(comm, &src_rank);
+
+  
+
+    if(!comm.is_ep)
+    {
+      ::MPI_Request mpi_request;
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Ibsend(buf, count, static_cast< ::MPI_Datatype> (datatype), dest, tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+
+      request->ep_src = src_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+      request->type = 1;
+      request->comm = comm;
+
+      return 0;
+    }
+
+    if(comm.is_intercomm) return 0;//MPI_Ibsend_intercomm(buf, count, datatype, dest, tag, comm, request);
+
+   // EP intracomm
+
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int ep_src_loc = comm.ep_comm_ptr->size_rank_info[1].first;
+    int ep_dest_loc = comm.ep_comm_ptr->comm_list->rank_map->at(dest).first;
+    int mpi_tag 		= tag_combine(tag, ep_src_loc, ep_dest_loc);
+    int mpi_dest 	= comm.ep_comm_ptr->comm_list->rank_map->at(dest).second;
+
+    request->ep_src  = src_rank;
+    request->ep_tag  = tag;
+    request->ep_datatype = datatype;
+
+    ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm> (comm.mpi_comm);
+    ::MPI_Request mpi_request;
+
+    ::MPI_Ibsend(buf, count, static_cast< ::MPI_Datatype>(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+    request->mpi_request = mpi_request;
+    request->type = 1; 		// used in wait
+    request->comm = comm;
+    request->buf = const_cast<void*>(buf);
+
+    //Message_Check(comm);
+
+    return 0;  
+  }
 
 
@@ -190,5 +265,5 @@
     }
 
-    Message_Check(comm);
+    //Message_Check(comm);
 
 
@@ -263,5 +338,5 @@
     }
 
-    Message_Check(comm);
+    //Message_Check(comm);
 
 
@@ -313,4 +388,76 @@
 
   }
+  
+  int MPI_Ibsend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)
+  {
+    printf("MPI_Ibsend with intercomm not yet implemented\n");
+    MPI_Abort(comm, 0);
+    //check_sum_send(buf, count, datatype, dest, tag, comm, 1);
+
+    int dest_remote_ep_rank    = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).first;
+    int dest_remote_comm_label = comm.ep_comm_ptr->intercomm->remote_rank_map->at(dest).second;
+
+    int src_ep_rank    = comm.ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int src_comm_label;
+
+    for(int i=0; i<comm.ep_comm_ptr->intercomm->local_rank_map->size(); i++)
+    {
+      if(comm.ep_comm_ptr->intercomm->local_rank_map->at(i).first == src_ep_rank)
+      {
+        src_comm_label = comm.ep_comm_ptr->intercomm->local_rank_map->at(i).second;
+        break;
+      }
+    }
+
+    //Message_Check(comm);
+
+
+    if(dest_remote_comm_label == src_comm_label)       // mpi_dest differs
+    {
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm > (comm.mpi_comm);
+      ::MPI_Request mpi_request;
+  
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    else   // dest_remote_comm_label != src_comm_label
+    { 
+      int inter_src = comm.ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first;
+      int ep_src_loc = comm.rank_map->at(inter_src).first;
+      int ep_dest_loc = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).first;
+      int mpi_dest    = comm.ep_comm_ptr->intercomm->intercomm_rank_map->at(dest_remote_ep_rank).second;
+      int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);
+
+      ::MPI_Comm mpi_comm = static_cast< ::MPI_Comm >(comm.ep_comm_ptr->intercomm->mpi_inter_comm);
+      ::MPI_Request mpi_request;
+
+      ::MPI_Isend(buf, count, static_cast< ::MPI_Datatype >(datatype), mpi_dest, mpi_tag, mpi_comm, &mpi_request);
+
+      request->mpi_request = mpi_request;
+      request->type = 1; 		// used in wait
+      request->comm = comm;
+   
+      request->ep_src = src_ep_rank;
+      request->ep_tag = tag;
+      request->ep_datatype = datatype;
+    }
+
+    return 0;
+
+  }
 }
 
@@ -319,2 +466,3 @@
 
 
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_type.hpp	(revision 1196)
@@ -454,5 +454,15 @@
       #endif
 
-      std::list< MPI_Request* > * pending_ptr;
+
+    bool operator == (MPI_Request right)
+    {
+      //bool a = mpi_request == right.mpi_request;
+      bool b = type == right.type;
+      bool c = buf == right.buf;
+      bool d = ep_src == right.ep_src;
+      bool e = ep_tag == right.ep_tag;
+      bool f = ep_datatype == right.ep_datatype;
+      return b&&c&&d&&e&&f;
+    }
   };
 
@@ -492,6 +502,5 @@
             //    <MPI_Fint,thread_num>   EP_Comm
 
-  static std::list< MPI_Request* > * EP_PendingRequests = 0;
-  #pragma omp threadprivate(EP_PendingRequests)
+  //static std::list<MPI_Request * > *EP_PendingRequests = 0; 
 }
 
@@ -499,2 +508,3 @@
 
 #endif // EP_TYPE_HPP_INCLUDED
+
Index: /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp	(revision 1196)
@@ -12,120 +12,34 @@
 using namespace std;
 
+extern std::list< ep_lib::MPI_Request* > * EP_PendingRequests;
+#pragma omp threadprivate(EP_PendingRequests)
+
 
 
 namespace ep_lib 
 {	
-
+  
   int MPI_Wait(MPI_Request *request, MPI_Status *status)
   {
-
-    if(request->type == 1)  //=>isend
+    if(request->type !=1 && request->type !=2 && request->type !=3) 
     {
-      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
-      ::MPI_Status mpi_status;
-      ::MPI_Errhandler_set(MPI_COMM_WORLD_STD, MPI_ERRORS_RETURN);
-      int error_code = ::MPI_Wait(&mpi_request, &mpi_status);
-      if (error_code != MPI_SUCCESS) {
-      
-         char error_string[BUFSIZ];
-         int length_of_error_string, error_class;
-      
-         ::MPI_Error_class(error_code, &error_class);
-         ::MPI_Error_string(error_class, error_string, &length_of_error_string);
-         printf("%s\n", error_string);
-      }
-
-      status->mpi_status = &mpi_status;
-      status->ep_src = request->ep_src;
-      status->ep_tag = request->ep_tag;
-      status->ep_datatype = request->ep_datatype;
-
-      return 0;
+      printf("Error in request type\n");
+ 
+      exit(1);
     }
 
-    if(request->type == 3) //=>imrecv
-    {
-      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
-      ::MPI_Status mpi_status;
-      ::MPI_Errhandler_set(MPI_COMM_WORLD_STD, MPI_ERRORS_RETURN);
-      int error_code = ::MPI_Wait(&mpi_request, &mpi_status);
-      if (error_code != MPI_SUCCESS) {
+    while(request->type == 2) Request_Check();
+
+
+    ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+    ::MPI_Status mpi_status;
+    ::MPI_Wait(&mpi_request, &mpi_status);
       
-         char error_string[BUFSIZ];
-         int length_of_error_string, error_class;
-      
-         ::MPI_Error_class(error_code, &error_class);
-         ::MPI_Error_string(error_class, error_string, &length_of_error_string);
-         printf("%s\n", error_string);
-      }
-      
+    request->mpi_request = mpi_request;
 
-      status->mpi_status = new ::MPI_Status(mpi_status);
-      status->ep_src = request->ep_src;
-      status->ep_tag = request->ep_tag;
-      status->ep_datatype = request->ep_datatype;
-
-      //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
-      return 0;
-
-    }
-
-    if(request->type == 2) //=>irecv not probed
-    {
-      
-      while(true)
-      {
-        Message_Check(request->comm);
-        // parcours pending list
-        for(std::list<MPI_Request* >::iterator it = (request->pending_ptr)->begin(); it!=(request->pending_ptr)->end(); )
-        {
-          if(*it == request)
-          { 
-            int probed = false;
-            MPI_Message message;
-
-            MPI_Improbe(request->ep_src, request->ep_tag, request->comm, &probed, &message, status);
-                  
-            if(probed)
-            {
-              int recv_count;
-              MPI_Get_count(status, request->ep_datatype, &recv_count);
-              MPI_Mrecv(request->buf, recv_count, request->ep_datatype, &message, status);
-              (request->pending_ptr)->erase(it);
-              //printf("wait  : pending request processed, size = %d\n", (request->pending_ptr)->size());
-              request->type = 3;
-            
-              return 0;
-            }
-
-            it++;
-          }
-          else 
-          {
-            int probed = false;
-            MPI_Message message;
-            MPI_Status status;
-
-            MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &message, &status);
-                  
-            if(probed)
-            {
-              int recv_count;
-              MPI_Get_count(&status, (*it)->ep_datatype, &recv_count);
-              MPI_Imrecv((*it)->buf, recv_count, (*it)->ep_datatype, &message, *it);
-                  
-              (request->pending_ptr)->erase(it);
-                  
-              it = (request->pending_ptr)->begin();
-              //printf("wait  : pending request processed, size = %d\n", (request->pending_ptr)->size());
-            }
-            else it++;
-          }
-        }
-
-      }
-      
-
-    }
+    status->mpi_status = &mpi_status;
+    status->ep_src = request->ep_src;
+    status->ep_tag = request->ep_tag;
+    status->ep_datatype = request->ep_datatype;
 
     return MPI_SUCCESS;
@@ -140,103 +54,44 @@
   int MPI_Waitall(int count, MPI_Request *array_of_requests, MPI_Status *array_of_statuses)
   {
-    //int dest_rank;
-    //MPI_Comm_rank(MPI_COMM_WORLD, &dest_rank);
-    //printf("proc %d enters waitall\n", dest_rank);
+    std::vector<int> finished(count, 0);
 
-    int finished = 0;
-    int finished_index[count];
+    ::MPI_Request* mpi_request = new ::MPI_Request[count];
+    ::MPI_Status* mpi_status = new ::MPI_Status[count];
 
-    for(int i=0; i<count; i++)
-      printf("pending add = %p\n",  array_of_requests[i].pending_ptr);
+    //if(EP_PendingRequests != 0) printf("pending size = %d, add = %p\n", EP_PendingRequests->size(), EP_PendingRequests);
 
-    //if(EP_PendingRequests == 0) EP_PendingRequests = new std::list< MPI_Request* >;  
-    //printf("pending size = %d, add = %p\n", EP_PendingRequests->size(), EP_PendingRequests);
-
-    for(int i=0; i<count; i++)
-    {
-      finished_index[i] = false;
-    }
-
-    while(finished < count)
+    while(std::accumulate(finished.begin(), finished.end(), 0) < count)
     {
       
       for(int i=0; i<count; i++)
       {
-        if(finished_index[i] == false) // this request has not been tested.
+        if(array_of_requests[i].type !=1 && array_of_requests[i].type !=2 && array_of_requests[i].type !=3) 
         {
-          if(array_of_requests[i].type == 1 || array_of_requests[i].type == 3) // isend or imrecv
-          {      
-            //MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
-            int tested;
-            MPI_Test(&array_of_requests[i], &tested, &array_of_statuses[i]);
-            if(!tested) MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
-            finished++;
-            finished_index[i] = true;
-          }
-          else // irecv
-          {
-            
-            Message_Check(array_of_requests[i].comm);
-            // parcours pending list
-            for(std::list<MPI_Request* >::iterator it = (array_of_requests[i].pending_ptr)->begin(); it!=(array_of_requests[i].pending_ptr)->end(); )
-            {
-              bool matched = false;
-              for(int j=0; j<count; j++)
-              {
-                if(*it == &array_of_requests[j])
-                { 
-                  int probed = false;
-                  MPI_Message message;
-
-                  MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &message, &array_of_statuses[j]);
-                  
-                  if(probed)
-                  {
-                    int recv_count;
-                    MPI_Get_count(&array_of_statuses[j], array_of_requests[j].ep_datatype, &recv_count);
-                    MPI_Mrecv(array_of_requests[j].buf, recv_count, array_of_requests[j].ep_datatype, &message, &array_of_statuses[j]);
-                    //check_sum_recv(array_of_requests[i].buf, recv_count, array_of_requests[i].ep_datatype, array_of_requests[i].ep_src, array_of_requests[i].ep_tag, array_of_requests[i].comm, 2);
-                    (array_of_requests[i].pending_ptr)->erase(it);
-                    array_of_requests[j].type = 3;
-                    finished++;
-                    finished_index[j] = true;
-                    matched = true;
-                    it = (array_of_requests[i].pending_ptr)->begin();
-                    j=count;
-                    //printf("waitall  : pending request processed, size = %d\n", (*(array_of_requests[i].pending_ptr))->size());
-                    //printf("matched = %d, j=%d, src = %d, tag = %d, probed = %d\n", matched, j, (*it)->ep_src, (*it)->ep_tag, probed);
-                  }
-                }
-
-              }
-
-              if(!matched)
-              {
-                int probed = false;
-                MPI_Message message;
-                MPI_Status status;
-
-                MPI_Improbe((*it)->ep_src, (*it)->ep_tag, (*it)->comm, &probed, &message, &status);
-                  
-                if(probed)
-                {
-                  int recv_count;
-                  MPI_Get_count(&status, (*it)->ep_datatype, &recv_count);
-                  MPI_Imrecv((*it)->buf, recv_count, (*it)->ep_datatype, &message, *it);
-                  
-                  (array_of_requests[i].pending_ptr)->erase(it);
-                  
-                  it = (array_of_requests[i].pending_ptr)->begin();
-                  //printf("waitall  : pending request processed, size = %d\n", (*(array_of_requests[i].pending_ptr))->size());
-                }
-                else it++;
-              }
-            }
-
-          }
+          printf("Error in request type\n");
+ 
+          exit(1);
+        }
+        
+        if(array_of_requests[i].type == 2) Request_Check(); 
+        if(array_of_requests[i].type != 2 && finished.at(i) == 0) 
+        {
+          finished.at(i) = 1;
+          mpi_request[i] = static_cast< ::MPI_Request >(array_of_requests[i].mpi_request);
         }
       }    
     }
-    //printf("proc %d exits waitall\n", dest_rank);
+
+    ::MPI_Waitall(count, mpi_request, mpi_status);
+
+    for(int i=0; i<count; i++)
+    {
+      array_of_statuses[i].mpi_status = &mpi_status;
+      array_of_statuses[i].ep_src = array_of_requests[i].ep_src;
+      array_of_statuses[i].ep_tag = array_of_requests[i].ep_tag;
+      array_of_statuses[i].ep_datatype = array_of_requests[i].ep_datatype;
+    }
+
+    delete[] mpi_request;
+    delete[] mpi_status;
     return MPI_SUCCESS;
   }  /* end of mpi_waitall*/
@@ -244,2 +99,3 @@
 
 }
+
Index: /XIOS/dev/branch_yushan_merged/src/client.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/src/client.cpp	(revision 1196)
@@ -115,5 +115,5 @@
 
             
-            test_sendrecv(CXios::globalComm);
+            //test_sendrecv(CXios::globalComm);
             MPI_Intercomm_create(intraComm,0,CXios::globalComm,serverLeader,0,&interComm) ;
 
Index: /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp	(revision 1196)
@@ -175,24 +175,47 @@
     recvIndexBuff = new unsigned long[recvNbIndexCount];
 
-  std::vector<ep_lib::MPI_Request> request;
+  int request_size = 0;
+
+  int currentIndex = 0;
+  int nbRecvClient = recvRankClient.size();
+
+  int position = 0;
+
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+    {
+      request_size++;
+    }
+  }
+
+  request_size += client2ClientIndex.size();
+
+
+  std::vector<ep_lib::MPI_Request> request(request_size);
+
   std::vector<int>::iterator itbRecvIndex = recvRankClient.begin(), itRecvIndex,
                              iteRecvIndex = recvRankClient.end(),
                            itbRecvNbIndex = recvNbIndexClientCount.begin(),
                            itRecvNbIndex;
-  int currentIndex = 0;
-  int nbRecvClient = recvRankClient.size();
+  
   
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
                                                 iteIndex = client2ClientIndex.end();
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
-    sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
-
-
-
+  {
+    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG, itIndex->first, MPI_DHT_INDEX, commLevel, &request[position]);
+    position++;
+    //sendIndexToClients(itIndex->first, (itIndex->second), sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+  }
+    
   for (int idx = 0; idx < nbRecvClient; ++idx)
   {
     if (0 != recvNbIndexClientCount[idx])
     {
-      recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
+            recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[position]);
+      position++;
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
     }
     currentIndex += recvNbIndexClientCount[idx];
@@ -200,7 +223,7 @@
 
   
-  std::vector<ep_lib::MPI_Status> status(request.size());
+  std::vector<ep_lib::MPI_Status> status(request_size);
   MPI_Waitall(request.size(), &request[0], &status[0]);
-  
+
 
   CArray<size_t,1>* tmpGlobalIndex;
@@ -256,15 +279,40 @@
   }
 
-  std::vector<ep_lib::MPI_Request> requestOnReturn;
+  request_size = 0;
+  for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
+  {
+    if (0 != recvNbIndexOnReturn[idx])
+    {
+      request_size += 2;
+    }
+  }
+
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != sendNbIndexOnReturn[idx])
+    {
+      request_size += 2;
+    }
+  }
+
+  std::vector<ep_lib::MPI_Request> requestOnReturn(request_size);
   currentIndex = 0;
+  position = 0;
   for (int idx = 0; idx < recvRankOnReturn.size(); ++idx)
   {
     if (0 != recvNbIndexOnReturn[idx])
     {
-      recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      recvInfoFromClients(recvRankOnReturn[idx],
-                          recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-                          recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
-                          commLevel, requestOnReturn);
+      //recvIndexFromClients(recvRankOnReturn[idx], recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], commLevel, requestOnReturn);
+      MPI_Irecv(recvIndexBuffOnReturn+currentIndex, recvNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
+            recvRankOnReturn[idx], MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
+      position++;
+      //recvInfoFromClients(recvRankOnReturn[idx],
+      //                    recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+      //                    recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(),
+      //                    commLevel, requestOnReturn);
+      MPI_Irecv(recvInfoBuffOnReturn+currentIndex*ProcessDHTElement<InfoType>::typeSize(), 
+                recvNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+                recvRankOnReturn[idx], MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
     }
     currentIndex += recvNbIndexOnReturn[idx];
@@ -299,8 +347,14 @@
       }
 
-      sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
-                         sendNbIndexOnReturn[idx], commLevel, requestOnReturn);
-      sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
-                        sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, requestOnReturn);
+      //sendIndexToClients(rank, client2ClientIndexOnReturn[rank],
+      //                   sendNbIndexOnReturn[idx], commLevel, requestOnReturn);
+      MPI_Isend(client2ClientIndexOnReturn[rank], sendNbIndexOnReturn[idx], MPI_UNSIGNED_LONG,
+            rank, MPI_DHT_INDEX, commLevel, &requestOnReturn[position]);
+      position++;
+      //sendInfoToClients(rank, client2ClientInfoOnReturn[rank],
+      //                  sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), commLevel, requestOnReturn);
+      MPI_Isend(client2ClientInfoOnReturn[rank], sendNbIndexOnReturn[idx]*ProcessDHTElement<InfoType>::typeSize(), MPI_CHAR,
+            rank, MPI_DHT_INFO, commLevel, &requestOnReturn[position]);
+      position++;
     }
     currentIndex += recvNbIndexClientCount[idx];
@@ -440,5 +494,7 @@
   int recvNbIndexCount = 0;
   for (int idx = 0; idx < recvNbIndexClientCount.size(); ++idx)
+  { 
     recvNbIndexCount += recvNbIndexClientCount[idx];
+  }
 
   unsigned long* recvIndexBuff;
@@ -453,19 +509,31 @@
   // it will send a message to the correct clients.
   // Contents of the message are index and its corresponding informatioin
-  std::vector<ep_lib::MPI_Request> request;
+  int request_size = 0;  
   int currentIndex = 0;
   int nbRecvClient = recvRankClient.size();
+  int current_pos = 0; 
+
   for (int idx = 0; idx < nbRecvClient; ++idx)
   {
     if (0 != recvNbIndexClientCount[idx])
     {
-      recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
-      recvInfoFromClients(recvRankClient[idx],
-                          recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
-                          recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
-                          commLevel, request);
-    }
-    currentIndex += recvNbIndexClientCount[idx];
-  }
+      request_size += 2;
+    }
+    //currentIndex += recvNbIndexClientCount[idx];
+  }
+
+  request_size += client2ClientIndex.size();
+  request_size += client2ClientInfo.size();
+
+
+
+  std::vector<ep_lib::MPI_Request> request(request_size);
+  
+  //unsigned long* tmp_send_buf_long[client2ClientIndex.size()];
+  //unsigned char* tmp_send_buf_char[client2ClientInfo.size()];
+  
+  int info_position = 0;
+  int index_position = 0;
+
 
   boost::unordered_map<int, size_t* >::iterator itbIndex = client2ClientIndex.begin(), itIndex,
@@ -473,5 +541,17 @@
   for (itIndex = itbIndex; itIndex != iteIndex; ++itIndex)
   {
-    sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+    //sendIndexToClients(itIndex->first, itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], commLevel, request);
+
+    //tmp_send_buf_long[index_position] = new unsigned long[sendNbIndexBuff[itIndex->first-groupRankBegin]];
+    //for(int i=0; i<sendNbIndexBuff[itIndex->first-groupRankBegin]; i++)
+    //{
+    //  tmp_send_buf_long[index_position][i] = (static_cast<unsigned long * >(itIndex->second))[i];
+    //}
+    //MPI_Isend(tmp_send_buf_long[current_pos], sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
+    MPI_Isend(itIndex->second, sendNbIndexBuff[itIndex->first-groupRankBegin], MPI_UNSIGNED_LONG,
+              itIndex->first, MPI_DHT_INDEX, commLevel, &request[current_pos]);
+    current_pos++; 
+    index_position++;
+
   }
 
@@ -480,11 +560,58 @@
   for (itInfo = itbInfo; itInfo != iteInfo; ++itInfo)
   {
-    sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
-
+    //sendInfoToClients(itInfo->first, itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], commLevel, request);
+
+    //tmp_send_buf_char[info_position] = new unsigned char[sendNbInfo[itInfo->first-groupRankBegin]];
+    //for(int i=0; i<sendNbInfo[itInfo->first-groupRankBegin]; i++)
+    //{
+    //  tmp_send_buf_char[info_position][i] = (static_cast<unsigned char * >(itInfo->second))[i];
+    //}
+
+    MPI_Isend(itInfo->second, sendNbInfo[itInfo->first-groupRankBegin], MPI_CHAR,
+              itInfo->first, MPI_DHT_INFO, commLevel, &request[current_pos]);
+
+    current_pos++;
+    info_position++;
+  }
+  
+  for (int idx = 0; idx < nbRecvClient; ++idx)
+  {
+    if (0 != recvNbIndexClientCount[idx])
+    {
+      //recvIndexFromClients(recvRankClient[idx], recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], commLevel, request);
+      MPI_Irecv(recvIndexBuff+currentIndex, recvNbIndexClientCount[idx], MPI_UNSIGNED_LONG,
+                recvRankClient[idx], MPI_DHT_INDEX, commLevel, &request[current_pos]);
+      current_pos++;
+      
+      
+      MPI_Irecv(recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(), 
+                recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(), 
+                MPI_CHAR, recvRankClient[idx], MPI_DHT_INFO, commLevel, &request[current_pos]);
+      
+      current_pos++;
+      
+
+
+      // recvInfoFromClients(recvRankClient[idx],
+      //                     recvInfoBuff+currentIndex*ProcessDHTElement<InfoType>::typeSize(),
+      //                     recvNbIndexClientCount[idx]*ProcessDHTElement<InfoType>::typeSize(),
+      //                     commLevel, request);
+    }
+    currentIndex += recvNbIndexClientCount[idx];
   }
 
   std::vector<ep_lib::MPI_Status> status(request.size());
-
+  
   MPI_Waitall(request.size(), &request[0], &status[0]);
+  
+ 
+  //for(int i=0; i<client2ClientInfo.size(); i++)
+  //  delete[] tmp_send_buf_char[i];
+
+  
+
+  //for(int i=0; i<client2ClientIndex.size(); i++)
+  //  delete[] tmp_send_buf_long[i];
+
 
   Index2VectorInfoTypeMap indexToInfoMapping;
@@ -527,4 +654,5 @@
   else
     index2InfoMapping_.swap(indexToInfoMapping);
+  
 }
 
@@ -720,9 +848,17 @@
   std::vector<ep_lib::MPI_Request> request(sendBuffSize+recvBuffSize);
   std::vector<ep_lib::MPI_Status> requestStatus(sendBuffSize+recvBuffSize);
-
+  //ep_lib::MPI_Request request[sendBuffSize+recvBuffSize];
+  //ep_lib::MPI_Status requestStatus[sendBuffSize+recvBuffSize];
+  
   int my_rank;
   MPI_Comm_rank(this->internalComm_, &my_rank);
   
   int nRequest = 0;
+  for (int idx = 0; idx < recvBuffSize; ++idx)
+  {
+    MPI_Irecv(&recvBuff[2*idx], 2, MPI_INT,
+              recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
+    ++nRequest;
+  }
   
 
@@ -743,15 +879,12 @@
   }
   
-  for (int idx = 0; idx < recvBuffSize; ++idx)
-  {
-    MPI_Irecv(&recvBuff[0]+2*idx, 2, MPI_INT,
-              recvRank[idx], MPI_DHT_INDEX_0, this->internalComm_, &request[nRequest]);
-    ++nRequest;
-  }
+  
 
   //MPI_Barrier(this->internalComm_);
 
   MPI_Waitall(sendBuffSize+recvBuffSize, &request[0], &requestStatus[0]);
-
+  //MPI_Waitall(sendBuffSize+recvBuffSize, request, requestStatus);
+
+  
   int nbRecvRank = 0, nbRecvElements = 0;
   recvNbRank.clear();
@@ -765,5 +898,10 @@
     }
   }
-}
-
-}
+
+
+  
+  
+}
+
+}
+
Index: /XIOS/dev/branch_yushan_merged/src/server.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/server.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/src/server.cpp	(revision 1196)
@@ -93,5 +93,5 @@
                      <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< clientLeader<<endl ;
 
-             test_sendrecv(CXios::globalComm);
+            // test_sendrecv(CXios::globalComm);
              MPI_Intercomm_create(intraComm,0,CXios::globalComm,clientLeader,0,&newComm) ;
              interComm.push_back(newComm) ;
Index: /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/src/test/test_omp.f90	(revision 1196)
@@ -47,5 +47,5 @@
     if(rank < size-2) then
 
-    !$omp parallel default(private)
+    !$omp parallel default(firstprivate)
   
     CALL xios_initialize(id,return_comm=comm)
Index: /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp
===================================================================
--- /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp	(revision 1195)
+++ /XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp	(revision 1196)
@@ -474,4 +474,5 @@
   // Sending global index of grid source to corresponding process as well as the corresponding mask
   std::vector<ep_lib::MPI_Request> requests;
+  requests.reserve(2*recvRankSizeMap.size()+2*globaIndexWeightFromSrcToDst.size());
   std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned char* > recvMaskDst;
@@ -534,4 +535,5 @@
   std::vector<ep_lib::MPI_Request>().swap(requests);
   std::vector<ep_lib::MPI_Status>().swap(status);
+  requests.reserve(sendRankSizeMap.size()+recvRankSizeMap.size());
   // Okie, on destination side, we will wait for information of masked index of source
   for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend)
