Changeset 1518 for XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

Timestamp:

06/01/18 19:36:12 (6 years ago)

Author:

yushan

Message:

optimisation partially finished. To do : special case for intercomm_create and intercomm_merge

File:

• XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

@@ -124 +124 @@
   int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
   {
+    
 
     assert(inter_comm->is_intercomm);
 
-    //if(inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99)
-    //{
-    //    return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm);
-    //}
+    // determine if only one MPI proc
+
+        // to be completed ......
+
+    // multiple MPI proc and high differs
+
+    int newcomm_ep_rank = inter_comm->ep_comm_ptr->intercomm->size_rank_info[0].first;
+    int newcomm_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].first;
+    int newcomm_num_ep = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].second;
+
+    int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first;
+    int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first;
+    int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second;
+
+    if(newcomm_ep_rank_loc == 0)
+    {
+      ::MPI_Comm *mpi_intracomm = new ::MPI_Comm;
+      ::MPI_Intercomm_merge(to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm), high, mpi_intracomm);
+      
+      MPI_Info info;
+      MPI_Comm *ep_comm;
+      MPI_Comm_create_endpoints(mpi_intracomm, newcomm_num_ep, info, ep_comm);
+
+      inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm;
+    }
+
+    MPI_Barrier_local(inter_comm);
+
+    int remote_num_ep = newcomm_num_ep - num_ep;
+
+    *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[high? remote_num_ep+ep_rank_loc : ep_rank_loc];
+
+    int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second;
+    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->intercomm_rank_map->size();
+
+    //printf("ep_size = %d, remote_ep_size = %d\n", ep_size, remote_ep_size);
+
+    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = high? remote_ep_size+ep_rank : ep_rank;
+
+    int my_triple[3];
+    my_triple[0] = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first;
+    my_triple[1] = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first;
+    my_triple[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first;
+
+    int *my_triple_list = new int[3 * (*newintracomm)->ep_comm_ptr->size_rank_info[0].second];
+
+
+    MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm);
+
+    if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first == 0)
+    {
+      (*newintracomm)->ep_rank_map->clear();
+      for(int i=0; i<(*newintracomm)->ep_comm_ptr->size_rank_info[0].second; i++)
+      {
+        (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_list[3*i], my_triple_list[3*i+1], my_triple_list[3*i+2]));
+      }
+    }
+
+#ifdef _showinfo
+    MPI_Barrier_local(inter_comm);
+    if((*newintracomm)->ep_comm_ptr->size_rank_info[0].first == 15)
+    {
+      for(std::map<int, std::pair<int, int> >::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++)
+      {
+        printf("(%d  %d  %d)\n", it->first, it->second.first, it->second.second);
+      }
+    }
+#endif
+    
+    delete my_triple_list;
+ 
+
+    return 0;
+
 
 
     Debug("intercomm_merge kernel\n");
 
-    int ep_rank_loc;
-    int num_ep;
+    //int ep_rank_loc;
+    //int num_ep;
 
     ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first;
@@ -143 +214 @@
 
 
-    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size();
+    //int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size();