Changeset 1249 for XIOS/dev/XIOS_DEV_CMIP6/src/node/axis.cpp

Timestamp:

08/30/17 15:33:51 (7 years ago)

Author:

mhnguyen

Message:

Various bug fixes on mask and zoom

+) Rearrange local index on the receiving side to be coherent global index
+) Include masking information in compress data (data_index) on the receiving side
+) Correct zoom to work in case there are several (not all) processes participating to write data

Test
+) On Curie
+) Simple test

File:

• XIOS/dev/XIOS_DEV_CMIP6/src/node/axis.cpp (modified) (9 diffs)

XIOS/dev/XIOS_DEV_CMIP6/src/node/axis.cpp

@@ -23 +23 @@
       , isClientAfterTransformationChecked(false)
       , hasBounds(false), isCompressible_(false)
-      , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , numberWrittenIndexes_(), totalNumberWrittenIndexes_(), offsetWrittenIndexes_()
       , transformationMap_(), hasValue(false), hasLabel(false)
       , computedWrittenIndex_(false)
@@ -34 +34 @@
       , isClientAfterTransformationChecked(false)
       , hasBounds(false), isCompressible_(false)
-      , numberWrittenIndexes_(0), totalNumberWrittenIndexes_(0), offsetWrittenIndexes_(0)
+      , numberWrittenIndexes_(), totalNumberWrittenIndexes_(), offsetWrittenIndexes_()
       , transformationMap_(), hasValue(false), hasLabel(false)
       , computedWrittenIndex_(false)
@@ -107 +107 @@
      \return the number of indexes written by each server
    */
-   int CAxis::getNumberWrittenIndexes() const
-   {
-     return numberWrittenIndexes_;
+   int CAxis::getNumberWrittenIndexes(MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     MPI_Comm_size(writtenCom, &writtenSize);
+     return numberWrittenIndexes_[writtenSize];
    }
 
@@ -116 +118 @@
      \return the total number of indexes written by the servers
    */
-   int CAxis::getTotalNumberWrittenIndexes() const
-   {
-     return totalNumberWrittenIndexes_;
+   int CAxis::getTotalNumberWrittenIndexes(MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     MPI_Comm_size(writtenCom, &writtenSize);
+     return totalNumberWrittenIndexes_[writtenSize];
    }
 
@@ -125 +129 @@
      \return the offset of indexes written by each server
    */
-   int CAxis::getOffsetWrittenIndexes() const
-   {
-     return offsetWrittenIndexes_;
-   }
-
+   int CAxis::getOffsetWrittenIndexes(MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     MPI_Comm_size(writtenCom, &writtenSize);
+     return offsetWrittenIndexes_[writtenSize];
+   }
+
+   CArray<int, 1>& CAxis::getCompressedIndexToWriteOnServer(MPI_Comm writtenCom)
+   {
+     int writtenSize;
+     MPI_Comm_size(writtenCom, &writtenSize);
+     return compressedIndexToWriteOnServer[writtenSize];
+   }
    //----------------------------------------------------------------
 
@@ -692 +704 @@
     }
 
+    // if (isCompressible())
+    // {
+    //   nbWritten = 0;
+    //   boost::unordered_map<size_t,size_t> localGlobalIndexMap;
+    //   for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
+    //   {
+    //     indGlo = *itSrv;
+    //     if (ite != globalLocalIndexMap_.find(indGlo))
+    //     {
+    //       localGlobalIndexMap[localIndexToWriteOnServer(nbWritten)] = indGlo;
+    //       ++nbWritten;
+    //     }                 
+    //   }
+
+    //   nbWritten = 0;
+    //   for (int idx = 0; idx < data_index.numElements(); ++idx)
+    //   {
+    //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+    //     {
+    //       ++nbWritten;
+    //     }
+    //   }
+
+    //   compressedIndexToWriteOnServer.resize(nbWritten);
+    //   nbWritten = 0;
+    //   for (int idx = 0; idx < data_index.numElements(); ++idx)
+    //   {
+    //     if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
+    //     {
+    //       compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_index(idx)];
+    //       ++nbWritten;
+    //     }
+    //   }
+
+    //   numberWrittenIndexes_ = nbWritten;
+    //   if (isDistributed())
+    //   {
+             
+    //     MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+    //     MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
+    //     offsetWrittenIndexes_ -= numberWrittenIndexes_;
+    //   }
+    //   else
+    //     totalNumberWrittenIndexes_ = numberWrittenIndexes_;
+    // }
+  }
+
+  void CAxis::computeWrittenCompressedIndex(MPI_Comm writtenComm)
+  {
+    int writtenCommSize;
+    MPI_Comm_size(writtenComm, &writtenCommSize);
+    if (compressedIndexToWriteOnServer.find(writtenCommSize) != compressedIndexToWriteOnServer.end())
+      return;
+
     if (isCompressible())
     {
-      nbWritten = 0;
+      size_t nbWritten = 0, indGlo;
+      CContext* context=CContext::getCurrent();      
+      CContextServer* server = context->server; 
+
+      // We describe the distribution of client (server) on which data are written
+      std::vector<int> nBegin(1), nSize(1), nBeginGlobal(1), nGlob(1);
+      nBegin[0]       = zoom_begin;
+      nSize[0]        = zoom_n;   
+      nBeginGlobal[0] = 0; 
+      nGlob[0]        = n_glo;
+      CDistributionServer srvDist(server->intraCommSize, nBegin, nSize, nBeginGlobal, nGlob); 
+      const CArray<size_t,1>& writtenGlobalIndex  = srvDist.getGlobalIndex();
+      boost::unordered_map<size_t,size_t>::const_iterator itb = globalLocalIndexMap_.begin(),
+                                                          ite = globalLocalIndexMap_.end(), it;   
+
+      CArray<size_t,1>::const_iterator itSrvb = writtenGlobalIndex.begin(),
+                                       itSrve = writtenGlobalIndex.end(), itSrv;
       boost::unordered_map<size_t,size_t> localGlobalIndexMap;
       for (itSrv = itSrvb; itSrv != itSrve; ++itSrv)
@@ -715 +797 @@
       }
 
-      compressedIndexToWriteOnServer.resize(nbWritten);
+      compressedIndexToWriteOnServer[writtenCommSize].resize(nbWritten);
       nbWritten = 0;
       for (int idx = 0; idx < data_index.numElements(); ++idx)
@@ -721 +803 @@
         if (localGlobalIndexMap.end() != localGlobalIndexMap.find(data_index(idx)))
         {
-          compressedIndexToWriteOnServer(nbWritten) = localGlobalIndexMap[data_index(idx)];
+          compressedIndexToWriteOnServer[writtenCommSize](nbWritten) = localGlobalIndexMap[data_index(idx)];
           ++nbWritten;
         }
       }
 
-      numberWrittenIndexes_ = nbWritten;
+      numberWrittenIndexes_[writtenCommSize] = nbWritten;
       if (isDistributed())
       {
              
-        MPI_Allreduce(&numberWrittenIndexes_, &totalNumberWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-        MPI_Scan(&numberWrittenIndexes_, &offsetWrittenIndexes_, 1, MPI_INT, MPI_SUM, server->intraComm);
-        offsetWrittenIndexes_ -= numberWrittenIndexes_;
+        MPI_Allreduce(&numberWrittenIndexes_[writtenCommSize], &totalNumberWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        MPI_Scan(&numberWrittenIndexes_[writtenCommSize], &offsetWrittenIndexes_[writtenCommSize], 1, MPI_INT, MPI_SUM, writtenComm);
+        offsetWrittenIndexes_[writtenCommSize] -= numberWrittenIndexes_[writtenCommSize];
       }
       else
-        totalNumberWrittenIndexes_ = numberWrittenIndexes_;
-    }
-  }
-
+        totalNumberWrittenIndexes_[writtenCommSize] = numberWrittenIndexes_[writtenCommSize];
+    }
+  }
 
   /*!
@@ -853 +934 @@
     if (zoom_n<=0)
     {
-      zoom_begin = 0; zoom_n = 0;
+      zoom_n = 0; zoom_begin=global_zoom_begin; //0; zoom_begin = 0; 
     }