Changeset 1232 for XIOS/dev/XIOS_DEV_CMIP6/src/node/context.cpp

Timestamp:

07/31/17 17:59:25 (7 years ago)

Author:

mhnguyen

Message:

Fixing the blocking problem where there are more servers than the number of grid band distribution

+) Correct this problem not only for writing but also for reading
+) Allow "zero-size" domain, axis (i.e: domain, axis with ni = 0, and/or nj=0)

Test
+) On Curie
+) Work in both cases: Read and Write data

File:

• XIOS/dev/XIOS_DEV_CMIP6/src/node/context.cpp (modified) (19 diffs)

XIOS/dev/XIOS_DEV_CMIP6/src/node/context.cpp

@@ -477 +477 @@
          finalized = true;
 
+         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
          if (hasServer && !hasClient)
-         {
-           closeAllFile();
+         {           
            registryOut->hierarchicalGatherRegistry() ;
            if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
@@ -525 +525 @@
 
      // Check grid and calculate its distribution
-     checkGridEnabledFields();    
-
+     checkGridEnabledFields();
+ 
      // Distribute files between secondary servers according to the data size
      distributeFiles();
@@ -548 +548 @@
       // We have enough information to send to server
       // First of all, send all enabled files
-      sendEnabledFiles();
-
-      // Then, send all enabled fields
-      sendEnabledFields();
+      sendEnabledFiles(this->enabledWriteModeFiles);
+      // We only use server-level 1 (for now) to read data
+      if (!hasServer)
+        sendEnabledFiles(this->enabledReadModeFiles);
+
+      // Then, send all enabled fields      
+      sendEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendEnabledFieldsInFiles(this->enabledReadModeFiles);
 
       // At last, we have all info of domain and axis, then send them
-       sendRefDomainsAxis();
+       sendRefDomainsAxisScalars(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendRefDomainsAxisScalars(this->enabledReadModeFiles);        
 
        // After that, send all grid (if any)
-       sendRefGrid();
+      sendRefGrid(this->enabledWriteModeFiles);
+      if (!hasServer)
+        sendRefGrid(this->enabledReadModeFiles);
 
        // We have a xml tree on the server side and now, it should be also processed
        sendPostProcessing();
-
-       sendGridEnabledFields();       
+       
+       sendGridEnabledFieldsInFiles(this->enabledWriteModeFiles);       
+       if (!hasServer)
+        sendGridEnabledFieldsInFiles(this->enabledReadModeFiles);       
      }
      allProcessed = true;
@@ -630 +641 @@
     }
     
-    checkGridEnabledFields();    
+    checkGridEnabledFields();   
 
     if (hasClient) this->sendProcessingGridOfEnabledFields();
@@ -646 +657 @@
    }
 
-   void CContext::findAllEnabledFields(void)
-   {
-     for (unsigned int i = 0; i < this->enabledFiles.size(); i++)
-     (void)this->enabledFiles[i]->getEnabledFields();
-   }
-
-   void CContext::findAllEnabledFieldsInReadModeFiles(void)
-   {
-     for (unsigned int i = 0; i < this->enabledReadModeFiles.size(); ++i)
-     (void)this->enabledReadModeFiles[i]->getEnabledFields();
+   void CContext::findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     for (unsigned int i = 0; i < activeFiles.size(); i++)
+     (void)activeFiles[i]->getEnabledFields();
    }
 
@@ -664 +669 @@
    }
 
-   void CContext::sendGridEnabledFields()
+   /*!
+      Send active (enabled) fields in file from a client to others
+      \param [in] activeFiles files contains enabled fields to send
+   */
+   void CContext::sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {       
+       activeFiles[i]->sendGridOfEnabledFields();
+     }
+   }
+
+   void CContext::checkGridEnabledFields()
+   {
+     int size = enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       enabledFiles[i]->checkGridOfEnabledFields();       
+     }
+   }
+
+   /*!
+      Check grid of active (enabled) fields in file 
+      \param [in] activeFiles files contains enabled fields whose grid needs checking
+   */
+   void CContext::checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       activeFiles[i]->checkGridOfEnabledFields();       
+     }
+   }
+
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields
+      This can be done in a client then all computed information will be sent from this client to others
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
+   void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
    {
      int size = this->enabledFiles.size();
      for (int i = 0; i < size; ++i)
-     {       
-       this->enabledFiles[i]->sendGridOfEnabledFields();
-     }
-   }
-
-   void CContext::checkGridEnabledFields()
-   {
-     int size = this->enabledFiles.size();
+     {
+       this->enabledFiles[i]->solveOnlyRefOfEnabledFields(sendToServer);
+     }
+
      for (int i = 0; i < size; ++i)
      {
-       this->enabledFiles[i]->checkGridOfEnabledFields();       
-     }
-   }
-
-   void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
-   {
-     int size = this->enabledFiles.size();
-     for (int i = 0; i < size; ++i)
-     {
-       this->enabledFiles[i]->solveOnlyRefOfEnabledFields(sendToServer);
-     }
-
-     for (int i = 0; i < size; ++i)
-     {
        this->enabledFiles[i]->generateNewTransformationGridDest();
      }
    }
 
+    /*!
+      Go up the hierachical tree via field_ref and do check of attributes of fields.
+      The transformation can be done in this step.
+      All computed information will be sent from this client to others.
+      \param [in] sendToServer Flag to indicate whether calculated information will be sent
+   */
    void CContext::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
    {
@@ -831 +864 @@
 
        // (1) Find all enabled files in write mode
-       for (int i = 0; i < this->enabledFiles.size(); ++i)
-       {
-         if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
-          enabledWriteModeFiles.push_back(enabledFiles[i]);
-       }
+       // for (int i = 0; i < this->enabledFiles.size(); ++i)
+       // {
+       //   if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+       //    enabledWriteModeFiles.push_back(enabledFiles[i]);
+       // }
 
        // (2) Estimate the data volume for each file
@@ -882 +915 @@
  
        for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
-         enabledReadModeFiles[i]->setContextClient(client);
+       {
+         enabledReadModeFiles[i]->setContextClient(client);          
+       }
      }
      else
@@ -891 +926 @@
    }
 
+   /*!
+      Find all files in write mode
+   */
+   void CContext::findEnabledWriteModeFiles(void)
+   {
+     int size = this->enabledFiles.size();
+     for (int i = 0; i < size; ++i)
+     {
+       if (enabledFiles[i]->mode.isEmpty() || 
+          (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
+        enabledWriteModeFiles.push_back(enabledFiles[i]);
+     }
+   }
+
+   /*!
+      Find all files in read mode
+   */
    void CContext::findEnabledReadModeFiles(void)
    {
@@ -1217 +1269 @@
 
       //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers à sortir.
-      this->findEnabledFiles();
-      
+      findEnabledFiles();
+      findEnabledWriteModeFiles();
+      findEnabledReadModeFiles();
+
       // For now, only read files with client and only one level server
-      if (hasClient && !hasServer) this->findEnabledReadModeFiles();
-
-      // Find all enabled fields of each file
-      this->findAllEnabledFields();
+      // if (hasClient && !hasServer) findEnabledReadModeFiles();      
+
+      // Find all enabled fields of each file      
+      findAllEnabledFieldsInFiles(this->enabledWriteModeFiles);
+      findAllEnabledFieldsInFiles(this->enabledReadModeFiles);
+
       // For now, only read files with client and only one level server
-      if (hasClient && !hasServer) this->findAllEnabledFieldsInReadModeFiles();
-
+      // if (hasClient && !hasServer) 
+      //   findAllEnabledFieldsInFiles(this->enabledReadModeFiles);      
 
       if (hasClient && !hasServer)
       {
-       // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
-       this->readAttributesOfEnabledFieldsInReadModeFiles();
+        initReadFiles();
+        // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
+        this->readAttributesOfEnabledFieldsInReadModeFiles();
       }
 
@@ -1259 +1316 @@
      if (hasClient)
      {
-       size_t numEnabledFiles = this->enabledFiles.size();
+       size_t numEnabledFiles = this->enabledWriteModeFiles.size();
        for (size_t i = 0; i < numEnabledFiles; ++i)
        {
-         CFile* file = this->enabledFiles[i];
+         CFile* file = this->enabledWriteModeFiles[i];
 //         if (file->getContextClient() == contextClient)
          {
@@ -1280 +1337 @@
                if (maxEventSize[it->first] < it->second)
                  maxEventSize[it->first] = it->second;
+             }
+           }
+         }
+       }
+
+      // Not a good approach here, duplicate code
+       if (!hasServer)
+       {
+         size_t numEnabledFiles = this->enabledReadModeFiles.size();
+         for (size_t i = 0; i < numEnabledFiles; ++i)
+         {
+           CFile* file = this->enabledReadModeFiles[i];  
+           {
+             std::vector<CField*> enabledFields = file->getEnabledFields();
+             size_t numEnabledFields = enabledFields.size();
+             for (size_t j = 0; j < numEnabledFields; ++j)
+             {
+               const std::map<int, StdSize> mapSize = enabledFields[j]->getGridAttributesBufferSize();
+               std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
+               for (; it != itE; ++it)
+               {
+                 // If attributesSize[it->first] does not exist, it will be zero-initialized
+                 // so we can use it safely without checking for its existance
+                 if (attributesSize[it->first] < it->second)
+                   attributesSize[it->first] = it->second;
+
+                 if (maxEventSize[it->first] < it->second)
+                   maxEventSize[it->first] = it->second;
+               }
              }
            }
@@ -1341 +1427 @@
 
    //! Client side: Send infomation of active files (files are enabled to write out)
-   void CContext::sendEnabledFiles()
-   {
-     int size = this->enabledFiles.size();
+   void CContext::sendEnabledFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
 
      // In a context, each type has a root definition, e.g: axis, domain, field.
@@ -1353 +1439 @@
      for (int i = 0; i < size; ++i)
      {
-       cfgrpPtr->sendCreateChild(this->enabledFiles[i]->getId(),enabledFiles[i]->getContextClient());
-       this->enabledFiles[i]->sendAllAttributesToServer(enabledFiles[i]->getContextClient());
-       this->enabledFiles[i]->sendAddAllVariables(enabledFiles[i]->getContextClient());
+       CFile* f = activeFiles[i];
+       cfgrpPtr->sendCreateChild(f->getId(),f->getContextClient());
+       f->sendAllAttributesToServer(f->getContextClient());
+       f->sendAddAllVariables(f->getContextClient());
      }
    }
 
    //! Client side: Send information of active fields (ones are written onto files)
-   void CContext::sendEnabledFields()
-   {
-     int size = this->enabledFiles.size();
+   void CContext::sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
+   {
+     int size = activeFiles.size();
      for (int i = 0; i < size; ++i)
      {
-       this->enabledFiles[i]->sendEnabledFields(enabledFiles[i]->getContextClient());
+       activeFiles[i]->sendEnabledFields(activeFiles[i]->getContextClient());
      }
    }
@@ -1488 +1575 @@
 
    //! Client side: Send information of reference grid of active fields
-   void CContext::sendRefGrid()
+   void CContext::sendRefGrid(const std::vector<CFile*>& activeFiles)
    {
      std::set<StdString> gridIds;
-     int sizeFile = this->enabledFiles.size();
+     int sizeFile = activeFiles.size();
      CFile* filePtr(NULL);
 
@@ -1497 +1584 @@
      for (int i = 0; i < sizeFile; ++i)
      {
-       filePtr = this->enabledFiles[i];
+       filePtr = activeFiles[i];
        std::vector<CField*> enabledFields = filePtr->getEnabledFields();
        int sizeField = enabledFields.size();
@@ -1521 +1608 @@
    }
 
-
-   //! Client side: Send information of reference domain and axis of active fields
-   void CContext::sendRefDomainsAxis()
+   //! Client side: Send information of reference domain, axis and scalar of active fields
+   void CContext::sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles)
    {
      std::set<StdString> domainIds, axisIds, scalarIds;
 
      // Find all reference domain and axis of all active fields
-     int numEnabledFiles = this->enabledFiles.size();
+     int numEnabledFiles = activeFiles.size();
      for (int i = 0; i < numEnabledFiles; ++i)
      {
-       std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
+       std::vector<CField*> enabledFields = activeFiles[i]->getEnabledFields();
        int numEnabledFields = enabledFields.size();
        for (int j = 0; j < numEnabledFields; ++j)
@@ -1592 +1678 @@
       info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
 #endif
-      if (hasClient)
+      //if (hasClient) 
+      if (hasClient && !hasServer) // For now we only use server level 1 to read data
       {
         checkPrefetchingOfEnabledReadModeFiles();
@@ -1599 +1686 @@
    }
 
+   void CContext::initReadFiles(void)
+   {
+      vector<CFile*>::const_iterator it;
+
+      for (it=enabledReadModeFiles.begin(); it != enabledReadModeFiles.end(); it++)
+      {
+         (*it)->initRead();
+      }
+   }
+
    //! Server side: Create header of netcdf file
-   void CContext::createFileHeader(void )
+   void CContext::createFileHeader(void)
    {
       vector<CFile*>::const_iterator it;
 
       for (it=enabledFiles.begin(); it != enabledFiles.end(); it++)
+      // for (it=enabledWriteModeFiles.begin(); it != enabledWriteModeFiles.end(); it++)
       {
-         (*it)->initFile();
+         (*it)->initWrite();
       }
    }