Changeset 1525 for XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

Timestamp:

06/07/18 19:39:07 (6 years ago)

Author:

yushan

Message:

save dev : intercomm_create via intracomm. To do : modif related intercomm communication routines, barrier, free, rank, size...

File:

• XIOS/dev/branch_openmp/extern/ep_dev/main.cpp (modified) (12 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -717 +717 @@
       MPI_Barrier(comm);
       
-      if(rank == 0) printf("            \t tMPI_Comm_split \t OK\n");
+      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
       
       MPI_Barrier(comm);
@@ -746 +746 @@
       MPI_Barrier(split_comm2);
       MPI_Comm_free(&split_comm2);
+
       
       
@@ -825 +826 @@
       MPI_Barrier(comm);
       MPI_Barrier(comm);
+
 
       if(color==2 && split_rank==0)
@@ -878 +880 @@
 
 
-      //printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);
+      printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);
 
       MPI_Comm inter_comm_dup_merged;
@@ -892 +894 @@
       //printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
       
-      // TEST OF TESTALL
-      
-        int merged_size, merged_rank;
-        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
-        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
-        
-        int left = merged_rank>0? merged_rank-1 : merged_size-1;
-        int right = (merged_rank+1) % merged_size;
-        
-        //printf("rank = %d, left = %d, right = %d\n", merged_rank, left, right);
-        
-        int N=10000;
-        int *left_buf = new int[N];
-        int *right_buf = new int[N];
-                
-        for(int i=0; i<N; i++)
-        {
-          right_buf[i] = merged_rank*100000+i;
-        }
-        
-        MPI_Request request[2];
-        MPI_Status status[2];
-        
-        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
-        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
-        
-        
-        int flag;
-        MPI_Testall(2, request, &flag, status);
-      
-        printf("rank = %d, flag = %d\n", merged_rank, flag);
-        
-        while(!flag) MPI_Testall(2, request, &flag, status);
-        //if(!flag)  MPI_Waitall(2, request, status);
-        
-        int recv_count;
-        MPI_Get_count(&status[0], MPI_INT, &recv_count);
-        
-        MPI_Barrier(inter_comm_dup_merged);
-        
-        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
-                        
-        delete[] left_buf;
-        delete[] right_buf;
-      
-      
+      
+      MPI_Barrier(inter_comm_dup_merged);
 
       // TEST OF GATHER
       {
+        
+
         int gather_root = 99;
   
@@ -976 +936 @@
       }
 
+      MPI_Barrier(inter_comm_dup_merged);
+
       // TEST OF ALLREDUCE
       {
@@ -1009 +971 @@
       }
 
+      MPI_Barrier(inter_comm_dup_merged);
+
       // TEST OF EXSCAN
-      /*{
+      {
  
         std::vector<int>sendbuf(2, inter_comm_dup_merged_rank);
@@ -1029 +993 @@
           exscan_test = 0; 
         }
-
-        //printf(" ID=%d : sendbuf = (%d, %d), recvbuf = (%d, %d), exscan_test = %d \n", inter_comm_dup_merged_rank, sendbuf[0], sendbuf[1], recvbuf[0], recvbuf[1], exscan_test); 
         
         int exscan_result;
@@ -1037 +999 @@
         if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n");
         if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result);
-      }*/
+      }
+
+      MPI_Barrier(inter_comm_dup_merged);
 
       // TEST OF SCATTERV
@@ -1067 +1031 @@
         MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, inter_comm_dup_merged);
 
-        //printf("ID = %d : %d %d\n", inter_comm_dup_merged_rank, recvbuf[0], recvbuf[1]);
-
         int scatterv_test = 1;
 
@@ -1085 +1047 @@
       }
 
-
-
+      MPI_Barrier(inter_comm_dup_merged);
+
+      // TEST OF TESTALL
+      {
+        int merged_size, merged_rank;
+        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
+        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
+        
+        int left = merged_rank>0? merged_rank-1 : merged_size-1;
+        int right = (merged_rank+1) % merged_size;
+                
+        int N=10000;
+        int *left_buf = new int[N];
+        int *right_buf = new int[N];
+                
+        for(int i=0; i<N; i++)
+        {
+          right_buf[i] = merged_rank*100000+i;
+        }
+        
+        MPI_Request request[2];
+        MPI_Status status[2];
+        
+        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
+        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
+        
+        
+        int flag;
+        MPI_Testall(2, request, &flag, status);
+      
+        printf("rank = %d, flag = %d\n", merged_rank, flag);
+        
+        while(!flag) MPI_Testall(2, request, &flag, status);
+        
+        int recv_count;
+        MPI_Get_count(&status[0], MPI_INT, &recv_count);
+        
+        MPI_Barrier(inter_comm_dup_merged);
+        
+        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
+                        
+        delete[] left_buf;
+        delete[] right_buf;
+      }
+      
+      MPI_Barrier(inter_comm_dup_merged);
+
+      // TEST WAITALL
+      {
+        int merged_rank;
+        int merged_size;
+
+        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
+        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
+
+        int left = merged_rank>0? merged_rank-1 : merged_size-1;
+        int right = (merged_rank+1)%merged_size;
+
+        printf("merged_rank = %d, left = %d, right = %d\n", merged_rank, left, right);
+
+        int NN=10000;
+
+        double *left_buf = new double[NN];
+        double *right_buf = new double[NN];
+
+        for(int i=0; i<NN; i++)
+        {
+          right_buf[i] = merged_rank*1000000 + i;
+        }
+
+        MPI_Request request[2];
+        MPI_Status  status[2];
+
+        MPI_Irecv(left_buf,  NN, MPI_DOUBLE, left,  0, inter_comm_dup_merged, &request[0]);
+        MPI_Isend(right_buf, NN, MPI_DOUBLE, right, 0, inter_comm_dup_merged, &request[1]);
+        
+
+        MPI_Waitall(2, request, status);
+
+        printf("merged_rank = %d, left_buf[0-4] = %lf\t%lf\t%lf\t%lf\t%lf\n", merged_rank, left_buf[0], left_buf[1], left_buf[2], left_buf[3], left_buf[4]);
+
+
+        delete left_buf;
+        delete right_buf;
+      }
 
       MPI_Barrier(comm);
@@ -1108 +1153 @@
       MPI_Comm_free(&inter_comm);
       
-
-      /*
-      MPI_Barrier(comm);
-      if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
-      MPI_Barrier(comm);
-
-    
-
-      int high=color;
-      MPI_Comm intra_comm;
-      MPI_Intercomm_merge(inter_comm, high, &intra_comm);
-    
-      int intra_rank, intra_size;
-      MPI_Comm_rank(intra_comm, &intra_rank);
-      MPI_Comm_size(intra_comm, &intra_size);
-
-      MPI_Barrier(comm);
-      if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
-      MPI_Barrier(comm);
-    
-    
-
-      //check_test_gatherv(comm);
-      
-      MPI_Barrier(comm);
-      MPI_Comm_free(&inter_comm);
-      */
+    }
+
+    // TESE OF INTERCOMM_CREATE
+    {
+      MPI_Barrier(comm);
+      
+      int rank, size;
+      MPI_Comm_rank(comm, &rank);
+      MPI_Comm_size(comm, &size);
+
+      int config = 1;
+      if(config=1)
+      {
+        assert(size == 16 && omp_get_num_threads()==4);
+      }
+      
+      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1
+      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1
+
+      int color = tab_color[rank];
+      int key = tab_key[rank];
+      
+  
+      MPI_Comm split_comm;
+      MPI_Comm_split(comm, color, key, &split_comm);
+
+    
+      int split_rank, split_size;
+      MPI_Comm_rank(split_comm, &split_rank);
+      MPI_Comm_size(split_comm, &split_size);
+
+
+      
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
+      
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+            
+      int local_leader = 5;
+      int remote_leader = color==2? 5: 2; // used for config 1
+
+      MPI_Comm peer_comm = comm;
+
+      MPI_Comm inter_comm;
+      MPI_Intercomm_create2(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);
+
+      int inter_rank;
+      int inter_size;
+      MPI_Comm_rank(inter_comm, &inter_rank);
+      MPI_Comm_size(inter_comm, &inter_size);
+
+      printf("split_rank = %d, inter_rank = %d / %d\n", split_rank, inter_rank, inter_size);
+
+      MPI_Barrier(split_comm);
+      MPI_Comm_free(&split_comm);
+
+      MPI_Barrier2(inter_comm);
+      MPI_Comm_free2(&inter_comm);
+      
     }