Changeset 1640

Timestamp:

01/22/19 17:02:30 (6 years ago)

Author:

oabramkina

Message:

On spatial transformations, avoiding send/receive of data from a rank to itself. Double (send/recv) memory allocation is also eliminated in these cases.

File:

• XIOS/trunk/src/filter/spatial_transform_filter.cpp (modified) (7 diffs)

XIOS/trunk/src/filter/spatial_transform_filter.cpp

@@ -188 +188 @@
     
     CContextClient* client = CContext::getCurrent()->client;
+    int rank;
+    MPI_Comm_rank (client->intraComm, &rank);
 
     // Get default value for output data
@@ -217 +219 @@
       int idxSendBuff = 0;
       std::vector<double*> sendBuff(localIndexToSend.size());
+      double* sendBuffRank;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
       {
+        int destRank = itSend->first;
         if (0 != itSend->second.numElements())
-          sendBuff[idxSendBuff] = new double[itSend->second.numElements()];
+        {
+          if (rank != itSend->first)
+            sendBuff[idxSendBuff] = new double[itSend->second.numElements()];
+          else
+            sendBuffRank = new double[itSend->second.numElements()];
+        }
       }
 
@@ -230 +239 @@
         const CArray<int,1>& localIndex_p = itSend->second;
         int countSize = localIndex_p.numElements();
-        for (int idx = 0; idx < countSize; ++idx)
-        {
-          sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
-        }
-        sendRecvRequest.push_back(MPI_Request());
-        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest.back());
+        if (destRank != rank)
+        {
+          for (int idx = 0; idx < countSize; ++idx)
+          {
+            sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
+            sendRecvRequest.push_back(MPI_Request());
+            MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest.back());
+          }
+        }
+        else
+        {
+          for (int idx = 0; idx < countSize; ++idx)
+          {
+            sendBuffRank[idx] = dataCurrentSrc(localIndex_p(idx));
+          }
+        }
       }
 
@@ -243 +262 @@
                                                                        iteRecv = localIndexToReceive.end();
       int recvBuffSize = 0;
-      for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv) recvBuffSize += itRecv->second.size(); //(recvBuffSize < itRecv->second.size())
-                                                                       //? itRecv->second.size() : recvBuffSize;
+      for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv)
+      {
+        if (itRecv->first != rank )
+          recvBuffSize += itRecv->second.size();
+      }
+      //(recvBuffSize < itRecv->second.size()) ? itRecv->second.size() : recvBuffSize;
       double* recvBuff;
+
       if (0 != recvBuffSize) recvBuff = new double[recvBuffSize];
       int currentBuff = 0;
@@ -251 +275 @@
       {
         int srcRank = itRecv->first;
-        int countSize = itRecv->second.size();
-        sendRecvRequest.push_back(MPI_Request());
-        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest.back());
-        currentBuff += countSize;
+        if (srcRank != rank)
+        {
+          int countSize = itRecv->second.size();
+          sendRecvRequest.push_back(MPI_Request());
+          MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest.back());
+          currentBuff += countSize;
+        }
       }
       std::vector<MPI_Status> status(sendRecvRequest.size());
@@ -267 +294 @@
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv)
       {
-        int countSize = itRecv->second.size();
         const std::vector<std::pair<int,double> >& localIndex_p = itRecv->second;
-        (*itAlgo)->apply(localIndex_p,
-                         recvBuff+currentBuff,
-                         dataCurrentDest,
-                         localInitFlag,
-                         ignoreMissingValue,firstPass);
-
-        currentBuff += countSize;
+        int srcRank = itRecv->first;
+        if (srcRank != rank)
+        {
+          int countSize = itRecv->second.size();
+          (*itAlgo)->apply(localIndex_p,
+                           recvBuff+currentBuff,
+                           dataCurrentDest,
+                           localInitFlag,
+                           ignoreMissingValue,firstPass);
+          currentBuff += countSize;
+        }
+        else
+        {
+          (*itAlgo)->apply(localIndex_p,
+                           sendBuffRank,
+                           dataCurrentDest,
+                           localInitFlag,
+                           ignoreMissingValue,firstPass);
+        }
+
         firstPass=false ;
       }
@@ -285 +324 @@
       {
         if (0 != itSend->second.numElements())
-          delete [] sendBuff[idxSendBuff];
+        {
+          if (rank != itSend->first)
+            delete [] sendBuff[idxSendBuff];
+          else
+            delete [] sendBuffRank;
+        }
       }
       if (0 != recvBuffSize) delete [] recvBuff;