source: XIOS/trunk/src/node/context.cpp @ 2253

Last change on this file since 2253 was 2253, checked in by aclsce, 3 years ago

Enable the use of record_offset in case of reading and server level 2.

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 72.6 KB
RevLine 
[219]1#include "context.hpp"
[352]2#include "attribute_template.hpp"
3#include "object_template.hpp"
4#include "group_template.hpp"
[219]5
6#include "calendar_type.hpp"
[278]7#include "duration.hpp"
[219]8
[300]9#include "context_client.hpp"
10#include "context_server.hpp"
11#include "nc4_data_output.hpp"
[346]12#include "node_type.hpp"
[352]13#include "message.hpp"
14#include "type.hpp"
[591]15#include "xios_spl.hpp"
[1158]16#include "timer.hpp"
17#include "memtrack.hpp"
[1215]18#include <limits>
[1349]19#include <fstream>
[992]20#include "server.hpp"
[1349]21#include "distribute_file_server2.hpp"
[697]22
[335]23namespace xios {
[509]24
[1542]25  std::shared_ptr<CContextGroup> CContext::root;
[509]26
[219]27   /// ////////////////////// Définitions ////////////////////// ///
28
29   CContext::CContext(void)
30      : CObjectTemplate<CContext>(), CContextAttributes()
[983]31      , calendar(), hasClient(false), hasServer(false)
[1139]32      , isPostProcessed(false), finalized(false)
[1009]33      , idServer_(), client(0), server(0)
[1130]34      , allProcessed(false), countChildCtx_(0)
[1622]35
[219]36   { /* Ne rien faire de plus */ }
37
38   CContext::CContext(const StdString & id)
39      : CObjectTemplate<CContext>(id), CContextAttributes()
[983]40      , calendar(), hasClient(false), hasServer(false)
[1139]41      , isPostProcessed(false), finalized(false)
[1009]42      , idServer_(), client(0), server(0)
[1130]43      , allProcessed(false), countChildCtx_(0)
[219]44   { /* Ne rien faire de plus */ }
45
46   CContext::~CContext(void)
[509]47   {
[597]48     delete client;
49     delete server;
[1071]50     for (std::vector<CContextClient*>::iterator it = clientPrimServer.begin(); it != clientPrimServer.end(); it++)  delete *it;
51     for (std::vector<CContextServer*>::iterator it = serverPrimServer.begin(); it != serverPrimServer.end(); it++)  delete *it;
[1139]52
[300]53   }
[219]54
55   //----------------------------------------------------------------
[509]56   //! Get name of context
[219]57   StdString CContext::GetName(void)   { return (StdString("context")); }
58   StdString CContext::GetDefName(void){ return (CContext::GetName()); }
59   ENodeType CContext::GetType(void)   { return (eContext); }
60
61   //----------------------------------------------------------------
[640]62
[509]63   /*!
64   \brief Get context group (context root)
65   \return Context root
66   */
[347]67   CContextGroup* CContext::getRoot(void)
[1622]68   TRY
[509]69   {
[1542]70      if (root.get()==NULL) root=std::shared_ptr<CContextGroup>(new CContextGroup(xml::CXMLNode::GetRootName()));
[509]71      return root.get();
[219]72   }
[1622]73   CATCH
[219]74
[640]75   //----------------------------------------------------------------
[509]76
77   /*!
78   \brief Get calendar of a context
79   \return Calendar
80   */
[1542]81   std::shared_ptr<CCalendar> CContext::getCalendar(void) const
[1622]82   TRY
[219]83   {
84      return (this->calendar);
85   }
[1622]86   CATCH
[509]87
[219]88   //----------------------------------------------------------------
[640]89
[509]90   /*!
91   \brief Set a context with a calendar
92   \param[in] newCalendar new calendar
93   */
[1542]94   void CContext::setCalendar(std::shared_ptr<CCalendar> newCalendar)
[1622]95   TRY
[219]96   {
97      this->calendar = newCalendar;
98   }
[1622]99   CATCH_DUMP_ATTR
[509]100
[219]101   //----------------------------------------------------------------
[509]102   /*!
103   \brief Parse xml file and write information into context object
104   \param [in] node xmld node corresponding in xml file
105   */
[219]106   void CContext::parse(xml::CXMLNode & node)
[1622]107   TRY
[219]108   {
109      CContext::SuperClass::parse(node);
110
111      // PARSING POUR GESTION DES ENFANTS
[278]112      xml::THashAttributes attributes = node.getAttributes();
[219]113
[278]114      if (attributes.end() != attributes.find("src"))
115      {
116         StdIFStream ifs ( attributes["src"].c_str() , StdIFStream::in );
[462]117         if ( (ifs.rdstate() & std::ifstream::failbit ) != 0 )
118            ERROR("void CContext::parse(xml::CXMLNode & node)",
119                  <<endl<< "Can not open <"<<attributes["src"].c_str()<<"> file" );
[278]120         if (!ifs.good())
121            ERROR("CContext::parse(xml::CXMLNode & node)",
122                  << "[ filename = " << attributes["src"] << " ] Bad xml stream !");
[462]123         xml::CXMLParser::ParseInclude(ifs, attributes["src"], *this);
[278]124      }
125
[219]126      if (node.getElementName().compare(CContext::GetName()))
[421]127         DEBUG("Le noeud is wrong defined but will be considered as a context !");
[219]128
129      if (!(node.goToChildElement()))
130      {
131         DEBUG("Le context ne contient pas d'enfant !");
132      }
133      else
134      {
135         do { // Parcours des contextes pour traitement.
136
137            StdString name = node.getElementName();
138            attributes.clear();
139            attributes = node.getAttributes();
140
141            if (attributes.end() != attributes.find("id"))
[509]142            {
143              DEBUG(<< "Definition node has an id,"
144                    << "it will not be taking account !");
145            }
[219]146
147#define DECLARE_NODE(Name_, name_)    \
148   if (name.compare(C##Name_##Definition::GetDefName()) == 0) \
[549]149   { C##Name_##Definition::create(C##Name_##Definition::GetDefName()) -> parse(node); continue; }
[219]150#define DECLARE_NODE_PAR(Name_, name_)
151#include "node_type.conf"
[286]152
[421]153            DEBUG(<< "The element \'"     << name
154                  << "\' in the context \'" << CContext::getCurrent()->getId()
155                  << "\' is not a definition !");
[219]156
157         } while (node.goToNextElement());
158
159         node.goToParentElement(); // Retour au parent
160      }
161   }
[1622]162   CATCH_DUMP_ATTR
[219]163
164   //----------------------------------------------------------------
[509]165   //! Show tree structure of context
[219]166   void CContext::ShowTree(StdOStream & out)
[1622]167   TRY
[219]168   {
[549]169      StdString currentContextId = CContext::getCurrent() -> getId();
[347]170      std::vector<CContext*> def_vector =
[346]171         CContext::getRoot()->getChildList();
[347]172      std::vector<CContext*>::iterator
[219]173         it = def_vector.begin(), end = def_vector.end();
174
175      out << "<? xml version=\"1.0\" ?>" << std::endl;
176      out << "<"  << xml::CXMLNode::GetRootName() << " >" << std::endl;
[509]177
[219]178      for (; it != end; it++)
179      {
[509]180         CContext* context = *it;
181         CContext::setCurrent(context->getId());
[219]182         out << *context << std::endl;
183      }
[509]184
[219]185      out << "</" << xml::CXMLNode::GetRootName() << " >" << std::endl;
[509]186      CContext::setCurrent(currentContextId);
[219]187   }
[1622]188   CATCH
[346]189
[219]190   //----------------------------------------------------------------
191
[509]192   //! Convert context object into string (to print)
[219]193   StdString CContext::toString(void) const
[1622]194   TRY
[219]195   {
196      StdOStringStream oss;
197      oss << "<" << CContext::GetName()
198          << " id=\"" << this->getId() << "\" "
199          << SuperClassAttribute::toString() << ">" << std::endl;
200      if (!this->hasChild())
201      {
202         //oss << "<!-- No definition -->" << std::endl; // fait planter l'incrémentation
203      }
204      else
205      {
206
207#define DECLARE_NODE(Name_, name_)    \
[346]208   if (C##Name_##Definition::has(C##Name_##Definition::GetDefName())) \
209   oss << * C##Name_##Definition::get(C##Name_##Definition::GetDefName()) << std::endl;
[219]210#define DECLARE_NODE_PAR(Name_, name_)
211#include "node_type.conf"
212
213      }
214      oss << "</" << CContext::GetName() << " >";
215      return (oss.str());
216   }
[1622]217   CATCH
[219]218
219   //----------------------------------------------------------------
220
[509]221   /*!
222   \brief Find all inheritace among objects in a context.
223   \param [in] apply (true) write attributes of parent into ones of child if they are empty
224                     (false) write attributes of parent into a new container of child
225   \param [in] parent unused
226   */
[445]227   void CContext::solveDescInheritance(bool apply, const CAttributeMap * const UNUSED(parent))
[1622]228   TRY
[219]229   {
230#define DECLARE_NODE(Name_, name_)    \
[354]231   if (C##Name_##Definition::has(C##Name_##Definition::GetDefName())) \
[445]232     C##Name_##Definition::get(C##Name_##Definition::GetDefName())->solveDescInheritance(apply);
[219]233#define DECLARE_NODE_PAR(Name_, name_)
234#include "node_type.conf"
235   }
[1622]236   CATCH_DUMP_ATTR
[219]237
238   //----------------------------------------------------------------
239
[509]240   //! Verify if all root definition in the context have child.
[219]241   bool CContext::hasChild(void) const
[1622]242   TRY
[219]243   {
244      return (
245#define DECLARE_NODE(Name_, name_)    \
[346]246   C##Name_##Definition::has(C##Name_##Definition::GetDefName())   ||
[219]247#define DECLARE_NODE_PAR(Name_, name_)
248#include "node_type.conf"
249      false);
250}
[1622]251   CATCH
[219]252
253   //----------------------------------------------------------------
254
255   void CContext::CleanTree(void)
[1622]256   TRY
[219]257   {
[549]258#define DECLARE_NODE(Name_, name_) C##Name_##Definition::ClearAllAttributes();
[219]259#define DECLARE_NODE_PAR(Name_, name_)
260#include "node_type.conf"
261   }
[1622]262   CATCH
263
[219]264   ///---------------------------------------------------------------
[509]265
266   //! Initialize client side
[1639]267   void CContext::initClient(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtServer /*= 0*/)
[1622]268   TRY
[300]269   {
[983]270
271     hasClient = true;
[1639]272     MPI_Comm intraCommServer, interCommServer;
[1316]273     
[1054]274
[1021]275     if (CServer::serverLevel != 1)
[1071]276      // initClient is called by client
[983]277     {
278       client = new CContextClient(this, intraComm, interComm, cxtServer);
[1054]279       if (cxtServer) // Attached mode
280       {
281         intraCommServer = intraComm;
282         interCommServer = interComm;
283       }
284       else
285       {
[1639]286         MPI_Comm_dup(intraComm, &intraCommServer);
[1071]287         comms.push_back(intraCommServer);
[1639]288         MPI_Comm_dup(interComm, &interCommServer);
[1071]289         comms.push_back(interCommServer);
[1054]290       }
[1316]291/* for registry take the id of client context */
292/* for servers, supress the _server_ from id  */
293       string contextRegistryId=getId() ;
294       size_t pos=contextRegistryId.find("_server_") ;
295       if (pos!=std::string::npos)  contextRegistryId=contextRegistryId.substr(0,pos) ;
296
[1071]297       registryIn=new CRegistry(intraComm);
[1316]298       registryIn->setPath(contextRegistryId) ;
[1071]299       if (client->clientRank==0) registryIn->fromFile("xios_registry.bin") ;
300       registryIn->bcastRegistry() ;
301       registryOut=new CRegistry(intraComm) ;
[1316]302       
303       registryOut->setPath(contextRegistryId) ;
[1071]304
305       server = new CContextServer(this, intraCommServer, interCommServer);
[983]306     }
[1021]307     else
[1054]308     // initClient is called by primary server
[983]309     {
[1009]310       clientPrimServer.push_back(new CContextClient(this, intraComm, interComm));
[1639]311       MPI_Comm_dup(intraComm, &intraCommServer);
[1071]312       comms.push_back(intraCommServer);
[1639]313       MPI_Comm_dup(interComm, &interCommServer);
[1071]314       comms.push_back(interCommServer);
315       serverPrimServer.push_back(new CContextServer(this, intraCommServer, interCommServer));
[983]316     }
[509]317   }
[1622]318   CATCH_DUMP_ATTR
[219]319
[1330]320   /*!
321    Sets client buffers.
322    \param [in] contextClient
323    \param [in] bufferForWriting True if buffers are used for sending data for writing
324    This flag is only true for client and server-1 for communication with server-2
325  */
326   void CContext::setClientServerBuffer(CContextClient* contextClient, bool bufferForWriting)
[1622]327   TRY
[509]328   {
[1201]329      // Estimated minimum event size for small events (10 is an arbitrary constant just for safety)
[1372]330     const size_t minEventSize = CEventClient::headerSize + getIdServer().size() + 10 * sizeof(int);
[1201]331
332      // Ensure there is at least some room for 20 of such events in the buffers
333      size_t minBufferSize = std::max(CXios::minBufferSize, 20 * minEventSize);
334
[583]335#define DECLARE_NODE(Name_, name_)    \
[731]336     if (minBufferSize < sizeof(C##Name_##Definition)) minBufferSize = sizeof(C##Name_##Definition);
[583]337#define DECLARE_NODE_PAR(Name_, name_)
338#include "node_type.conf"
[719]339#undef DECLARE_NODE
340#undef DECLARE_NODE_PAR
341
[1201]342     // Compute the buffer sizes needed to send the attributes and data corresponding to fields
[917]343     std::map<int, StdSize> maxEventSize;
[1330]344     std::map<int, StdSize> bufferSize = getAttributesBufferSize(maxEventSize, contextClient, bufferForWriting);
345     std::map<int, StdSize> dataBufferSize = getDataBufferSize(maxEventSize, contextClient, bufferForWriting);
[729]346
[909]347     std::map<int, StdSize>::iterator it, ite = dataBufferSize.end();
[731]348     for (it = dataBufferSize.begin(); it != ite; ++it)
349       if (it->second > bufferSize[it->first]) bufferSize[it->first] = it->second;
350
[1227]351     // Apply the buffer size factor, check that we are above the minimum buffer size and below the maximum size
[909]352     ite = bufferSize.end();
353     for (it = bufferSize.begin(); it != ite; ++it)
354     {
355       it->second *= CXios::bufferSizeFactor;
356       if (it->second < minBufferSize) it->second = minBufferSize;
[1227]357       if (it->second > CXios::maxBufferSize) it->second = CXios::maxBufferSize;
[909]358     }
359
[1201]360     // Leaders will have to send some control events so ensure there is some room for those in the buffers
[1184]361     if (contextClient->isServerLeader())
[598]362     {
[1184]363       const std::list<int>& ranks = contextClient->getRanksServerLeader();
[729]364       for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
[1201]365       {
366         if (!bufferSize.count(*itRank))
367         {
368           bufferSize[*itRank] = minBufferSize;
369           maxEventSize[*itRank] = minEventSize;
370         }
371       }
[598]372     }
[1184]373     contextClient->setBufferSize(bufferSize, maxEventSize);
[509]374   }
[1622]375   CATCH_DUMP_ATTR
[509]376
377   //! Verify whether a context is initialized
[461]378   bool CContext::isInitialized(void)
[1622]379   TRY
[461]380   {
[549]381     return hasClient;
[461]382   }
[1622]383   CATCH_DUMP_ATTR
[509]384
[1639]385   void CContext::initServer(MPI_Comm intraComm, MPI_Comm interComm, CContext* cxtClient /*= 0*/)
[1622]386   TRY
[300]387   {
[549]388     hasServer=true;
389     server = new CContextServer(this,intraComm,interComm);
[697]390
[1316]391/* for registry take the id of client context */
392/* for servers, supress the _server_ from id  */
393     string contextRegistryId=getId() ;
394     size_t pos=contextRegistryId.find("_server_") ;
395     if (pos!=std::string::npos)  contextRegistryId=contextRegistryId.substr(0,pos) ;
396       
[1071]397     registryIn=new CRegistry(intraComm);
[1316]398     registryIn->setPath(contextRegistryId) ;
[1071]399     if (server->intraCommRank==0) registryIn->fromFile("xios_registry.bin") ;
400     registryIn->bcastRegistry() ;
401     registryOut=new CRegistry(intraComm) ;
[1316]402     registryOut->setPath(contextRegistryId) ;
[775]403
[1639]404     MPI_Comm intraCommClient, interCommClient;
[597]405     if (cxtClient) // Attached mode
406     {
407       intraCommClient = intraComm;
408       interCommClient = interComm;
409     }
410     else
411     {
[1639]412       MPI_Comm_dup(intraComm, &intraCommClient);
[1071]413       comms.push_back(intraCommClient);
[1639]414       MPI_Comm_dup(interComm, &interCommClient);
[1071]415       comms.push_back(interCommClient);
[597]416     }
[1158]417     client = new CContextClient(this,intraCommClient,interCommClient, cxtClient);
[509]418   }
[1622]419   CATCH_DUMP_ATTR
[300]420
[597]421   //! Try to send the buffers and receive possible answers
[1378]422  bool CContext::checkBuffersAndListen(bool enableEventsProcessing /*= true*/)
[1622]423  TRY
[1130]424  {
425    bool clientReady, serverFinished;
[1054]426
[1130]427    // Only classical servers are non-blocking
428    if (CServer::serverLevel == 0)
429    {
430      client->checkBuffers();
431      bool hasTmpBufferedEvent = client->hasTemporarilyBufferedEvent();
432      if (hasTmpBufferedEvent)
433        hasTmpBufferedEvent = !client->sendTemporarilyBufferedEvent();
434      // Don't process events if there is a temporarily buffered event
[1378]435      return server->eventLoop(!hasTmpBufferedEvent || !enableEventsProcessing);
[1130]436    }
437    else if (CServer::serverLevel == 1)
438    {
[1139]439      if (!finalized)
440        client->checkBuffers();
441      bool serverFinished = true;
442      if (!finalized)
[1380]443        serverFinished = server->eventLoop(enableEventsProcessing);
[1130]444      bool serverPrimFinished = true;
445      for (int i = 0; i < clientPrimServer.size(); ++i)
446      {
[1139]447        if (!finalized)
448          clientPrimServer[i]->checkBuffers();
449        if (!finalized)
[1378]450          serverPrimFinished *= serverPrimServer[i]->eventLoop(enableEventsProcessing);
[1130]451      }
452      return ( serverFinished && serverPrimFinished);
453    }
[1054]454
[1130]455    else if (CServer::serverLevel == 2)
456    {
457      client->checkBuffers();
[1378]458      return server->eventLoop(enableEventsProcessing);
[1130]459    }
460  }
[1622]461   CATCH_DUMP_ATTR
[1054]462
[509]463   //! Terminate a context
[300]464   void CContext::finalize(void)
[1622]465   TRY
[300]466   {
[1418]467      if (hasClient && !hasServer) // For now we only use server level 1 to read data
468      {
469        doPreTimestepOperationsForEnabledReadModeFiles();
470      }
[1130]471     // Send registry upon calling the function the first time
472     if (countChildCtx_ == 0)
473       if (hasClient) sendRegistry() ;
474
475     // Client:
476     // (1) blocking send context finalize to its server
477     // (2) blocking receive context finalize from its server
[1194]478     // (3) some memory deallocations
[1130]479     if (CXios::isClient)
[1009]480     {
[1130]481       // Make sure that client (model) enters the loop only once
482       if (countChildCtx_ < 1)
483       {
484         ++countChildCtx_;
[983]485
[1130]486         client->finalize();
487         while (client->havePendingRequests())
488            client->checkBuffers();
[1054]489
[1130]490         while (!server->hasFinished())
491           server->eventLoop();
[1139]492
[1194]493         if (hasServer) // Mode attache
494         {
495           closeAllFile();
496           registryOut->hierarchicalGatherRegistry() ;
497           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
498         }
[1139]499
[1194]500         //! Deallocate client buffers
501         client->releaseBuffers();
502
503         //! Free internally allocated communicators
[1639]504         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
[2029]505           /* MPI_Comm_free(&(*it)) */ ; // WARNING remove freeing communicator !! --> deadlock raised, to be checked
[1194]506         comms.clear();
507
508         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
[1130]509       }
510     }
511     else if (CXios::isServer)
512     {
[1194]513       // First context finalize message received from a model
514       // Send context finalize to its child contexts (if any)
[1130]515       if (countChildCtx_ == 0)
[1009]516         for (int i = 0; i < clientPrimServer.size(); ++i)
517           clientPrimServer[i]->finalize();
[983]518
[1139]519       // (Last) context finalized message received
[1130]520       if (countChildCtx_ == clientPrimServer.size())
[1194]521       {
522         // Blocking send of context finalize message to its client (e.g. primary server or model)
[1378]523         info(100)<<"DEBUG: context "<<getId()<<" Send client finalize<<"<<endl ;
[1130]524         client->finalize();
[1194]525         bool bufferReleased;
526         do
527         {
528           client->checkBuffers();
529           bufferReleased = !client->havePendingRequests();
530         } while (!bufferReleased);
531         finalized = true;
[983]532
[1232]533         closeAllFile(); // Just move to here to make sure that server-level 1 can close files
[1194]534         if (hasServer && !hasClient)
[1232]535         {           
[1194]536           registryOut->hierarchicalGatherRegistry() ;
537           if (server->intraCommRank==0) CXios::globalRegistry->mergeRegistry(*registryOut) ;
538         }
[1077]539
[1194]540         //! Deallocate client buffers
541         client->releaseBuffers();
542         for (int i = 0; i < clientPrimServer.size(); ++i)
543           clientPrimServer[i]->releaseBuffers();
[1139]544
[1194]545         //! Free internally allocated communicators
[1639]546         for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
[2029]547           /* MPI_Comm_free(&(*it)) */;  // WARNING remove freeing communicator !! --> deadlock raised, to be checked
[1194]548         comms.clear();
549
550         info(20)<<"CContext: Context <"<<getId()<<"> is finalized."<<endl;
[1139]551       }
552
[1194]553       ++countChildCtx_;
[1130]554     }
555   }
[1622]556   CATCH_DUMP_ATTR
[1130]557
[1071]558   //! Free internally allocated communicators
559   void CContext::freeComms(void)
[1622]560   TRY
[1071]561   {
[1639]562     for (std::list<MPI_Comm>::iterator it = comms.begin(); it != comms.end(); ++it)
[2029]563       /* MPI_Comm_free(&(*it)) */ ; // WARNING remove freeing communicator !! --> deadlock raised, to be checked
[1071]564     comms.clear();
565   }
[1622]566   CATCH_DUMP_ATTR
[509]567
[1130]568   //! Deallocate buffers allocated by clientContexts
569   void CContext::releaseClientBuffers(void)
[1622]570   TRY
[1130]571   {
572     client->releaseBuffers();
573     for (int i = 0; i < clientPrimServer.size(); ++i)
574       clientPrimServer[i]->releaseBuffers();
575   }
[1622]576   CATCH_DUMP_ATTR
[1130]577
[1025]578   void CContext::postProcessingGlobalAttributes()
[1622]579   TRY
[300]580   {
[1144]581     if (allProcessed) return; 
[1025]582     
[1144]583     // After xml is parsed, there are some more works with post processing
584     postProcessing();
[983]585
[1144]586     // Check grid and calculate its distribution
[1232]587     checkGridEnabledFields();
588 
[1208]589     // Distribute files between secondary servers according to the data size
590     distributeFiles();
591
[1330]592     setClientServerBuffer(client, (hasClient && !hasServer));
[1255]593     for (int i = 0; i < clientPrimServer.size(); ++i)
[1330]594         setClientServerBuffer(clientPrimServer[i], true);
[300]595
[983]596     if (hasClient)
[510]597     {
[509]598      // Send all attributes of current context to server
599      this->sendAllAttributesToServer();
[459]600
[549]601      // Send all attributes of current calendar
602      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->sendAllAttributesToServer();
603
[509]604      // We have enough information to send to server
[1021]605      // First of all, send all enabled files
[1232]606      sendEnabledFiles(this->enabledWriteModeFiles);
607      // We only use server-level 1 (for now) to read data
608      if (!hasServer)
609        sendEnabledFiles(this->enabledReadModeFiles);
[509]610
[1232]611      // Then, send all enabled fields     
612      sendEnabledFieldsInFiles(this->enabledWriteModeFiles);
613      if (!hasServer)
614        sendEnabledFieldsInFiles(this->enabledReadModeFiles);
[1021]615
[1239]616      // Then, check whether we have domain_ref, axis_ref or scalar_ref attached to the enabled fields
617      // If any, so send them to server
[1232]618       sendRefDomainsAxisScalars(this->enabledWriteModeFiles);
619      if (!hasServer)
620        sendRefDomainsAxisScalars(this->enabledReadModeFiles);       
[1021]621
[1239]622       // Check whether enabled fields have grid_ref, if any, send this info to server
[1237]623      sendRefGrid(this->enabledFiles);
624      // This code may be useful in the future when we want to seperate completely read and write
625      // sendRefGrid(this->enabledWriteModeFiles);
626      // if (!hasServer)
627      //   sendRefGrid(this->enabledReadModeFiles);
[1239]628     
629      // A grid of enabled fields composed of several components which must be checked then their
630      // checked attributes should be sent to server
631      sendGridComponentEnabledFieldsInFiles(this->enabledFiles); // This code can be seperated in two (one for reading, another for writing)
[1021]632
[983]633       // We have a xml tree on the server side and now, it should be also processed
[1239]634      sendPostProcessing();
[1232]635       
[1239]636      // Finally, we send information of grid itself to server
637      sendGridEnabledFieldsInFiles(this->enabledWriteModeFiles);       
638      if (!hasServer)
[1232]639        sendGridEnabledFieldsInFiles(this->enabledReadModeFiles);       
[983]640     }
[1025]641     allProcessed = true;
642   }
[1622]643   CATCH_DUMP_ATTR
[983]644
[1025]645   void CContext::sendPostProcessingGlobalAttributes()
[1622]646   TRY
[1025]647   {
[1027]648      // Use correct context client to send message
[1030]649     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
650    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1027]651     for (int i = 0; i < nbSrvPools; ++i)
652     {
653       CContextClient* contextClientTmp = (0 != clientPrimServer.size()) ? clientPrimServer[i] : client;
654       CEventClient event(getType(),EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES);
[987]655
[1027]656       if (contextClientTmp->isServerLeader())
657       {
658         CMessage msg;
[1099]659         if (hasServer)
660           msg<<this->getIdServer(i);
661         else
662           msg<<this->getIdServer();
[1027]663         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
664         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
665           event.push(*itRank,1,msg);
666         contextClientTmp->sendEvent(event);
667       }
668       else contextClientTmp->sendEvent(event);
[1025]669     }
670   }
[1622]671   CATCH_DUMP_ATTR
[1025]672
673   void CContext::recvPostProcessingGlobalAttributes(CEventServer& event)
[1622]674   TRY
[1025]675   {
676      CBufferIn* buffer=event.subEvents.begin()->buffer;
677      string id;
678      *buffer>>id;
[1099]679      get(id)->recvPostProcessingGlobalAttributes(*buffer);
[1025]680   }
[1622]681   CATCH
[1025]682
683   void CContext::recvPostProcessingGlobalAttributes(CBufferIn& buffer)
[1622]684   TRY
[1144]685   {     
[1025]686      postProcessingGlobalAttributes();
687   }
[1622]688   CATCH_DUMP_ATTR
[1025]689
690   /*!
691   \brief Close all the context defintion and do processing data
692      After everything is well defined on client side, they will be processed and sent to server
693   From the version 2.0, sever and client work no more on the same database. Moreover, client(s) will send
694   all necessary information to server, from which each server can build its own database.
695   Because the role of server is to write out field data on a specific netcdf file,
696   the only information that it needs is the enabled files
697   and the active fields (fields will be written onto active files)
698   */
[1622]699
[1025]700   void CContext::closeDefinition(void)
[1622]701   TRY
[1025]702   {
[1158]703    CTimer::get("Context : close definition").resume() ;
[1025]704    postProcessingGlobalAttributes();
[983]705
[1130]706    if (hasClient) sendPostProcessingGlobalAttributes();
707
708    // There are some processings that should be done after all of above. For example: check mask or index
[1025]709    this->buildFilterGraphOfEnabledFields();
710   
[1130]711     if (hasClient && !hasServer)
[509]712    {
[1130]713      buildFilterGraphOfFieldsWithReadAccess();
[1358]714      postProcessFilterGraph();
[509]715    }
[1144]716   
[1232]717    checkGridEnabledFields();   
[509]718
[1025]719    if (hasClient) this->sendProcessingGridOfEnabledFields();
[1077]720    if (hasClient) this->sendCloseDefinition();
[510]721
[584]722    // Nettoyage de l'arborescence
[983]723    if (hasClient) CleanTree(); // Only on client side??
[510]724
[598]725    if (hasClient)
726    {
727      sendCreateFileHeader();
[983]728      if (!hasServer) startPrefetchingOfEnabledReadModeFiles();
[598]729    }
[1158]730    CTimer::get("Context : close definition").suspend() ;
[300]731   }
[1622]732   CATCH_DUMP_ATTR
[509]733
[1232]734   void CContext::findAllEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
[1622]735   TRY
[300]736   {
[1232]737     for (unsigned int i = 0; i < activeFiles.size(); i++)
738     (void)activeFiles[i]->getEnabledFields();
[300]739   }
[1622]740   CATCH_DUMP_ATTR
[300]741
[775]742   void CContext::readAttributesOfEnabledFieldsInReadModeFiles()
[1622]743   TRY
[775]744   {
745      for (unsigned int i = 0; i < this->enabledReadModeFiles.size(); ++i)
746        (void)this->enabledReadModeFiles[i]->readAttributesOfEnabledFieldsInReadMode();
747   }
[1622]748   CATCH_DUMP_ATTR
[775]749
[1239]750   void CContext::sendGridComponentEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
[1622]751   TRY
[1239]752   {
753     int size = activeFiles.size();
754     for (int i = 0; i < size; ++i)
755     {       
756       activeFiles[i]->sendGridComponentOfEnabledFields();
757     }
758   }
[1622]759   CATCH_DUMP_ATTR
[1239]760
[1232]761   /*!
762      Send active (enabled) fields in file from a client to others
763      \param [in] activeFiles files contains enabled fields to send
764   */
765   void CContext::sendGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
[1622]766   TRY
[1025]767   {
[1232]768     int size = activeFiles.size();
[1025]769     for (int i = 0; i < size; ++i)
[1099]770     {       
[1232]771       activeFiles[i]->sendGridOfEnabledFields();
[1025]772     }
773   }
[1622]774   CATCH_DUMP_ATTR
[1025]775
[1099]776   void CContext::checkGridEnabledFields()
[1622]777   TRY
[1099]778   {
[1232]779     int size = enabledFiles.size();
[1099]780     for (int i = 0; i < size; ++i)
781     {
[1232]782       enabledFiles[i]->checkGridOfEnabledFields();       
[1099]783     }
784   }
[1622]785   CATCH_DUMP_ATTR
[1025]786
[1232]787   /*!
788      Check grid of active (enabled) fields in file
789      \param [in] activeFiles files contains enabled fields whose grid needs checking
790   */
791   void CContext::checkGridEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
[1622]792   TRY
[1232]793   {
794     int size = activeFiles.size();
795     for (int i = 0; i < size; ++i)
796     {
797       activeFiles[i]->checkGridOfEnabledFields();       
798     }
799   }
[1622]800   CATCH_DUMP_ATTR
[1232]801
802    /*!
803      Go up the hierachical tree via field_ref and do check of attributes of fields
804      This can be done in a client then all computed information will be sent from this client to others
805      \param [in] sendToServer Flag to indicate whether calculated information will be sent
806   */
[823]807   void CContext::solveOnlyRefOfEnabledFields(bool sendToServer)
[1622]808   TRY
[823]809   {
810     int size = this->enabledFiles.size();
811     for (int i = 0; i < size; ++i)
812     {
813       this->enabledFiles[i]->solveOnlyRefOfEnabledFields(sendToServer);
814     }
815
816     for (int i = 0; i < size; ++i)
817     {
818       this->enabledFiles[i]->generateNewTransformationGridDest();
819     }
820   }
[1622]821   CATCH_DUMP_ATTR
[823]822
[1232]823    /*!
824      Go up the hierachical tree via field_ref and do check of attributes of fields.
825      The transformation can be done in this step.
826      All computed information will be sent from this client to others.
827      \param [in] sendToServer Flag to indicate whether calculated information will be sent
828   */
[1129]829   void CContext::solveAllRefOfEnabledFieldsAndTransform(bool sendToServer)
[1622]830   TRY
[300]831   {
[509]832     int size = this->enabledFiles.size();
833     for (int i = 0; i < size; ++i)
834     {
[1129]835       this->enabledFiles[i]->solveAllRefOfEnabledFieldsAndTransform(sendToServer);
[509]836     }
[300]837   }
[1622]838   CATCH_DUMP_ATTR
[300]839
[640]840   void CContext::buildFilterGraphOfEnabledFields()
[1622]841   TRY
[640]842   {
843     int size = this->enabledFiles.size();
844     for (int i = 0; i < size; ++i)
845     {
846       this->enabledFiles[i]->buildFilterGraphOfEnabledFields(garbageCollector);
847     }
848   }
[1622]849   CATCH_DUMP_ATTR
[640]850
[1358]851   void CContext::postProcessFilterGraph()
[1622]852   TRY
[1358]853   {
854     int size = enabledFiles.size();
855     for (int i = 0; i < size; ++i)
856     {
857        enabledFiles[i]->postProcessFilterGraph();
858     }
859   }
[1622]860   CATCH_DUMP_ATTR
[1358]861
[598]862   void CContext::startPrefetchingOfEnabledReadModeFiles()
[1622]863   TRY
[598]864   {
865     int size = enabledReadModeFiles.size();
866     for (int i = 0; i < size; ++i)
867     {
868        enabledReadModeFiles[i]->prefetchEnabledReadModeFields();
869     }
870   }
[1622]871   CATCH_DUMP_ATTR
[598]872
[1358]873   void CContext::doPreTimestepOperationsForEnabledReadModeFiles()
[1622]874   TRY
[1358]875   {
876     int size = enabledReadModeFiles.size();
877     for (int i = 0; i < size; ++i)
878     {
879        enabledReadModeFiles[i]->doPreTimestepOperationsForEnabledReadModeFields();
880     }
881   }
[1622]882   CATCH_DUMP_ATTR
[1358]883
[1318]884   void CContext::doPostTimestepOperationsForEnabledReadModeFiles()
[1622]885   TRY
[598]886   {
887     int size = enabledReadModeFiles.size();
888     for (int i = 0; i < size; ++i)
889     {
[1318]890        enabledReadModeFiles[i]->doPostTimestepOperationsForEnabledReadModeFields();
[598]891     }
892   }
[1622]893   CATCH_DUMP_ATTR
[598]894
[593]895  void CContext::findFieldsWithReadAccess(void)
[1622]896  TRY
[593]897  {
898    fieldsWithReadAccess.clear();
899    const vector<CField*> allFields = CField::getAll();
900    for (size_t i = 0; i < allFields.size(); ++i)
901    {
[740]902      CField* field = allFields[i];
903
904      if (field->file && !field->file->mode.isEmpty() && field->file->mode == CFile::mode_attr::read)
905        field->read_access = true;
906      else if (!field->read_access.isEmpty() && field->read_access && (field->enabled.isEmpty() || field->enabled))
907        fieldsWithReadAccess.push_back(field);
[593]908    }
909  }
[1622]910  CATCH_DUMP_ATTR
[593]911
912  void CContext::solveAllRefOfFieldsWithReadAccess()
[1622]913  TRY
[593]914  {
915    for (size_t i = 0; i < fieldsWithReadAccess.size(); ++i)
916      fieldsWithReadAccess[i]->solveAllReferenceEnabledField(false);
917  }
[1622]918  CATCH_DUMP_ATTR
[593]919
[640]920  void CContext::buildFilterGraphOfFieldsWithReadAccess()
[1622]921  TRY
[640]922  {
923    for (size_t i = 0; i < fieldsWithReadAccess.size(); ++i)
924      fieldsWithReadAccess[i]->buildFilterGraph(garbageCollector, true);
925  }
[1622]926  CATCH_DUMP_ATTR
[640]927
[445]928   void CContext::solveAllInheritance(bool apply)
[1622]929   TRY
[300]930   {
931     // Résolution des héritages descendants (càd des héritages de groupes)
932     // pour chacun des contextes.
[445]933      solveDescInheritance(apply);
[300]934
935     // Résolution des héritages par référence au niveau des fichiers.
[549]936      const vector<CFile*> allFiles=CFile::getAll();
[540]937      const vector<CGrid*> allGrids= CGrid::getAll();
[300]938
[983]939      if (hasClient && !hasServer)
940      //if (hasClient)
[540]941      {
[510]942        for (unsigned int i = 0; i < allFiles.size(); i++)
943          allFiles[i]->solveFieldRefInheritance(apply);
[540]944      }
945
946      unsigned int vecSize = allGrids.size();
947      unsigned int i = 0;
948      for (i = 0; i < vecSize; ++i)
949        allGrids[i]->solveDomainAxisRefInheritance(apply);
950
[300]951   }
[1622]952  CATCH_DUMP_ATTR
[300]953
954   void CContext::findEnabledFiles(void)
[1622]955   TRY
[300]956   {
[347]957      const std::vector<CFile*> allFiles = CFile::getAll();
[1158]958      const CDate& initDate = calendar->getInitDate();
[300]959
960      for (unsigned int i = 0; i < allFiles.size(); i++)
961         if (!allFiles[i]->enabled.isEmpty()) // Si l'attribut 'enabled' est défini.
[430]962         {
[300]963            if (allFiles[i]->enabled.getValue()) // Si l'attribut 'enabled' est fixé à vrai.
[1158]964            {
[1489]965              if (allFiles[i]->output_freq.isEmpty())
[1488]966              {
967                 ERROR("CContext::findEnabledFiles()",
968                     << "Mandatory attribute output_freq must be defined for file \""<<allFiles[i]->getFileOutputName()
969                     <<" \".")
970              }
[1158]971              if ((initDate + allFiles[i]->output_freq.getValue()) < (initDate + this->getCalendar()->getTimeStep()))
972              {
973                error(0)<<"WARNING: void CContext::findEnabledFiles()"<<endl
974                    << "Output frequency in file \""<<allFiles[i]->getFileOutputName()
975                    <<"\" is less than the time step. File will not be written."<<endl;
976              }
977              else
[300]978               enabledFiles.push_back(allFiles[i]);
[1158]979            }
[430]980         }
[1158]981         else
982         {
[1489]983           if (allFiles[i]->output_freq.isEmpty())
[1488]984           {
985              ERROR("CContext::findEnabledFiles()",
986                  << "Mandatory attribute output_freq must be defined for file \""<<allFiles[i]->getFileOutputName()
987                  <<" \".")
988           }
[1158]989           if ( (initDate + allFiles[i]->output_freq.getValue()) < (initDate + this->getCalendar()->getTimeStep()))
990           {
991             error(0)<<"WARNING: void CContext::findEnabledFiles()"<<endl
992                 << "Output frequency in file \""<<allFiles[i]->getFileOutputName()
993                 <<"\" is less than the time step. File will not be written."<<endl;
994           }
995           else
996             enabledFiles.push_back(allFiles[i]); // otherwise true by default
997         }
[300]998
999      if (enabledFiles.size() == 0)
1000         DEBUG(<<"Aucun fichier ne va être sorti dans le contexte nommé \""
1001               << getId() << "\" !");
[1021]1002
[300]1003   }
[1622]1004   CATCH_DUMP_ATTR
[300]1005
[1208]1006   void CContext::distributeFiles(void)
[1622]1007   TRY
[1208]1008   {
[1349]1009     bool distFileMemory=false ;
1010     distFileMemory=CXios::getin<bool>("server2_dist_file_memory", distFileMemory);
1011
1012     if (distFileMemory) distributeFileOverMemoryBandwith() ;
1013     else distributeFileOverBandwith() ;
1014   }
[1622]1015   CATCH_DUMP_ATTR
[1349]1016
1017   void CContext::distributeFileOverBandwith(void)
[1622]1018   TRY
[1349]1019   {
[1215]1020     double eps=std::numeric_limits<double>::epsilon()*10 ;
1021     
[1208]1022     // If primary server
1023     if (hasServer && hasClient)
1024     {
[1349]1025       std::ofstream ofs(("distribute_file_"+getId()+".dat").c_str(), std::ofstream::out);
[1212]1026       int nbPools = clientPrimServer.size();
1027
[1208]1028       // (1) Find all enabled files in write mode
[1232]1029       // for (int i = 0; i < this->enabledFiles.size(); ++i)
1030       // {
1031       //   if (enabledFiles[i]->mode.isEmpty() || (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
1032       //    enabledWriteModeFiles.push_back(enabledFiles[i]);
1033       // }
[1208]1034
1035       // (2) Estimate the data volume for each file
1036       int size = this->enabledWriteModeFiles.size();
[1215]1037       std::vector<std::pair<double, CFile*> > dataSizeMap;
1038       double dataPerPool = 0;
1039       int nfield=0 ;
[1349]1040       ofs<<size<<endl ;
[1208]1041       for (size_t i = 0; i < size; ++i)
1042       {
1043         CFile* file = this->enabledWriteModeFiles[i];
[1349]1044         ofs<<file->getId()<<endl ;
[1215]1045         StdSize dataSize=0;
[1208]1046         std::vector<CField*> enabledFields = file->getEnabledFields();
1047         size_t numEnabledFields = enabledFields.size();
[1349]1048         ofs<<numEnabledFields<<endl ;
1049         for (size_t j = 0; j < numEnabledFields; ++j)
1050         {
1051           dataSize += enabledFields[j]->getGlobalWrittenSize() ;
1052           ofs<<enabledFields[j]->grid->getId()<<endl ;
1053           ofs<<enabledFields[j]->getGlobalWrittenSize()<<endl ;
1054         }
[1215]1055         double outFreqSec = (Time)(calendar->getCurrentDate()+file->output_freq)-(Time)(calendar->getCurrentDate()) ;
1056         double dataSizeSec= dataSize/ outFreqSec;
[1349]1057         ofs<<dataSizeSec<<endl ;
[1215]1058         nfield++ ;
1059// add epsilon*nField to dataSizeSec in order to  preserve reproductive ordering when sorting
1060         dataSizeMap.push_back(make_pair(dataSizeSec + dataSizeSec * eps * nfield , file));
1061         dataPerPool += dataSizeSec;
[1208]1062       }
[1212]1063       dataPerPool /= nbPools;
[1208]1064       std::sort(dataSizeMap.begin(), dataSizeMap.end());
1065
[1212]1066       // (3) Assign contextClient to each enabled file
[1215]1067
1068       std::multimap<double,int> poolDataSize ;
1069// multimap is not garanty to preserve stable sorting in c++98 but it seems it does for c++11
1070
1071       int j;
1072       double dataSize ;
1073       for (j = 0 ; j < nbPools ; ++j) poolDataSize.insert(std::pair<double,int>(0.,j)) ; 
1074             
[1212]1075       for (int i = dataSizeMap.size()-1; i >= 0; --i)
[1208]1076       {
[1215]1077         dataSize=(*poolDataSize.begin()).first ;
1078         j=(*poolDataSize.begin()).second ;
1079         dataSizeMap[i].second->setContextClient(clientPrimServer[j]);
1080         dataSize+=dataSizeMap[i].first;
1081         poolDataSize.erase(poolDataSize.begin()) ;
1082         poolDataSize.insert(std::pair<double,int>(dataSize,j)) ; 
[1208]1083       }
[1212]1084
[1215]1085       for (std::multimap<double,int>:: iterator it=poolDataSize.begin() ; it!=poolDataSize.end(); ++it) info(30)<<"Load Balancing for servers (perfect=1) : "<<it->second<<" :  ratio "<<it->first*1./dataPerPool<<endl ;
1086 
[1212]1087       for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
[1232]1088       {
1089         enabledReadModeFiles[i]->setContextClient(client);         
1090       }
[1208]1091     }
1092     else
1093     {
1094       for (int i = 0; i < this->enabledFiles.size(); ++i)
1095         enabledFiles[i]->setContextClient(client);
1096     }
1097   }
[1622]1098   CATCH_DUMP_ATTR
[1208]1099
[1349]1100   void CContext::distributeFileOverMemoryBandwith(void)
[1622]1101   TRY
[1349]1102   {
1103     // If primary server
1104     if (hasServer && hasClient)
1105     {
1106       int nbPools = clientPrimServer.size();
1107       double ratio=0.5 ;
1108       ratio=CXios::getin<double>("server2_dist_file_memory_ratio", ratio);
1109
1110       int nFiles = this->enabledWriteModeFiles.size();
1111       vector<SDistFile> files(nFiles);
1112       vector<SDistGrid> grids;
1113       map<string,int> gridMap ;
1114       string gridId; 
1115       int gridIndex=0 ;
1116
1117       for (size_t i = 0; i < nFiles; ++i)
1118       {
1119         StdSize dataSize=0;
1120         CFile* file = this->enabledWriteModeFiles[i];
1121         std::vector<CField*> enabledFields = file->getEnabledFields();
1122         size_t numEnabledFields = enabledFields.size();
1123
1124         files[i].id_=file->getId() ;
1125         files[i].nbGrids_=numEnabledFields;
1126         files[i].assignedGrid_ = new int[files[i].nbGrids_] ;
1127         
1128         for (size_t j = 0; j < numEnabledFields; ++j)
1129         {
1130           gridId=enabledFields[j]->grid->getId() ;
1131           if (gridMap.find(gridId)==gridMap.end())
1132           {
1133              gridMap[gridId]=gridIndex  ;
1134              SDistGrid newGrid; 
1135              grids.push_back(newGrid) ;
1136              gridIndex++ ;
1137           }
1138           files[i].assignedGrid_[j]=gridMap[gridId] ;
1139           grids[files[i].assignedGrid_[j]].size_=enabledFields[j]->getGlobalWrittenSize() ;
1140           dataSize += enabledFields[j]->getGlobalWrittenSize() ; // usefull
1141         }
1142         double outFreqSec = (Time)(calendar->getCurrentDate()+file->output_freq)-(Time)(calendar->getCurrentDate()) ;
1143         files[i].bandwith_= dataSize/ outFreqSec ;
1144       }
1145
1146       double bandwith=0 ;
1147       double memory=0 ;
1148   
1149       for(int i=0; i<nFiles; i++)  bandwith+=files[i].bandwith_ ;
1150       for(int i=0; i<nFiles; i++)  files[i].bandwith_ = files[i].bandwith_/bandwith * ratio ;
1151
1152       for(int i=0; i<grids.size(); i++)  memory+=grids[i].size_ ;
1153       for(int i=0; i<grids.size(); i++)  grids[i].size_ = grids[i].size_ / memory * (1.0-ratio) ;
1154       
1155       distributeFileOverServer2(nbPools, grids.size(), &grids[0], nFiles, &files[0]) ;
1156
1157       vector<double> memorySize(nbPools,0.) ;
1158       vector< set<int> > serverGrids(nbPools) ;
1159       vector<double> bandwithSize(nbPools,0.) ;
1160       
1161       for (size_t i = 0; i < nFiles; ++i)
1162       {
1163         bandwithSize[files[i].assignedServer_] += files[i].bandwith_* bandwith /ratio ;
1164         for(int j=0 ; j<files[i].nbGrids_;j++)
1165         {
1166           if (serverGrids[files[i].assignedServer_].find(files[i].assignedGrid_[j]) == serverGrids[files[i].assignedServer_].end())
1167           {
1168             memorySize[files[i].assignedServer_]+= grids[files[i].assignedGrid_[j]].size_ * memory / (1.0-ratio);
1169             serverGrids[files[i].assignedServer_].insert(files[i].assignedGrid_[j]) ;
1170           }
1171         }
1172         enabledWriteModeFiles[i]->setContextClient(clientPrimServer[files[i].assignedServer_]) ;
1173         delete [] files[i].assignedGrid_ ;
1174       }
1175
1176       for (int i = 0; i < nbPools; ++i) info(100)<<"Pool server level2 "<<i<<"   assigned file bandwith "<<bandwithSize[i]*86400.*4./1024/1024.<<" Mb / days"<<endl ;
1177       for (int i = 0; i < nbPools; ++i) info(100)<<"Pool server level2 "<<i<<"   assigned grid memory "<<memorySize[i]*100/1024./1024.<<" Mb"<<endl ;
1178
1179
1180       for (int i = 0; i < this->enabledReadModeFiles.size(); ++i)
1181       {
1182         enabledReadModeFiles[i]->setContextClient(client);         
1183       }
1184
1185   }
1186   else
1187   {
1188     for (int i = 0; i < this->enabledFiles.size(); ++i)
1189        enabledFiles[i]->setContextClient(client);
1190   }
1191}
[1622]1192   CATCH_DUMP_ATTR
[1349]1193
[1232]1194   /*!
1195      Find all files in write mode
1196   */
1197   void CContext::findEnabledWriteModeFiles(void)
[1622]1198   TRY
[1232]1199   {
1200     int size = this->enabledFiles.size();
1201     for (int i = 0; i < size; ++i)
1202     {
1203       if (enabledFiles[i]->mode.isEmpty() || 
1204          (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::write ))
1205        enabledWriteModeFiles.push_back(enabledFiles[i]);
1206     }
1207   }
[1622]1208   CATCH_DUMP_ATTR
[1232]1209
1210   /*!
1211      Find all files in read mode
1212   */
[598]1213   void CContext::findEnabledReadModeFiles(void)
[1622]1214   TRY
[598]1215   {
1216     int size = this->enabledFiles.size();
1217     for (int i = 0; i < size; ++i)
1218     {
1219       if (!enabledFiles[i]->mode.isEmpty() && enabledFiles[i]->mode.getValue() == CFile::mode_attr::read)
1220        enabledReadModeFiles.push_back(enabledFiles[i]);
1221     }
1222   }
[1622]1223   CATCH_DUMP_ATTR
[598]1224
[300]1225   void CContext::closeAllFile(void)
[1622]1226   TRY
[300]1227   {
[347]1228     std::vector<CFile*>::const_iterator
[300]1229            it = this->enabledFiles.begin(), end = this->enabledFiles.end();
[509]1230
[300]1231     for (; it != end; it++)
1232     {
1233       info(30)<<"Closing File : "<<(*it)->getId()<<endl;
1234       (*it)->close();
1235     }
1236   }
[1622]1237   CATCH_DUMP_ATTR
[509]1238
1239   /*!
1240   \brief Dispatch event received from client
1241      Whenever a message is received in buffer of server, it will be processed depending on
1242   its event type. A new event type should be added in the switch list to make sure
1243   it processed on server side.
1244   \param [in] event: Received message
1245   */
[300]1246   bool CContext::dispatchEvent(CEventServer& event)
[1622]1247   TRY
[300]1248   {
[509]1249
[549]1250      if (SuperClass::dispatchEvent(event)) return true;
[300]1251      else
1252      {
1253        switch(event.type)
1254        {
1255           case EVENT_ID_CLOSE_DEFINITION :
[549]1256             recvCloseDefinition(event);
1257             return true;
1258             break;
[584]1259           case EVENT_ID_UPDATE_CALENDAR:
[549]1260             recvUpdateCalendar(event);
1261             return true;
1262             break;
[300]1263           case EVENT_ID_CREATE_FILE_HEADER :
[549]1264             recvCreateFileHeader(event);
1265             return true;
1266             break;
[509]1267           case EVENT_ID_POST_PROCESS:
[549]1268             recvPostProcessing(event);
1269             return true;
[697]1270            case EVENT_ID_SEND_REGISTRY:
1271             recvRegistry(event);
1272             return true;
[1025]1273             break;
1274            case EVENT_ID_POST_PROCESS_GLOBAL_ATTRIBUTES:
1275             recvPostProcessingGlobalAttributes(event);
1276             return true;
1277             break;
1278            case EVENT_ID_PROCESS_GRID_ENABLED_FIELDS:
1279             recvProcessingGridOfEnabledFields(event);
1280             return true;
1281             break;
[300]1282           default :
1283             ERROR("bool CContext::dispatchEvent(CEventServer& event)",
[549]1284                    <<"Unknown Event");
1285           return false;
[300]1286         }
1287      }
1288   }
[1622]1289   CATCH
[509]1290
1291   //! Client side: Send a message to server to make it close
[300]1292   void CContext::sendCloseDefinition(void)
[1622]1293   TRY
[300]1294   {
[987]1295     // Use correct context client to send message
[1030]1296     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1009]1297     for (int i = 0; i < nbSrvPools; ++i)
[300]1298     {
[1009]1299       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
1300       CEventClient event(getType(),EVENT_ID_CLOSE_DEFINITION);
1301       if (contextClientTmp->isServerLeader())
1302       {
1303         CMessage msg;
1304         if (hasServer)
1305           msg<<this->getIdServer(i);
1306         else
1307           msg<<this->getIdServer();
1308         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
1309         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
1310           event.push(*itRank,1,msg);
1311         contextClientTmp->sendEvent(event);
1312       }
1313       else contextClientTmp->sendEvent(event);
[300]1314     }
1315   }
[1622]1316   CATCH_DUMP_ATTR
[509]1317
1318   //! Server side: Receive a message of client announcing a context close
[300]1319   void CContext::recvCloseDefinition(CEventServer& event)
[1622]1320   TRY
[300]1321   {
1322      CBufferIn* buffer=event.subEvents.begin()->buffer;
1323      string id;
[549]1324      *buffer>>id;
[509]1325      get(id)->closeDefinition();
[300]1326   }
[1622]1327   CATCH
[509]1328
1329   //! Client side: Send a message to update calendar in each time step
[300]1330   void CContext::sendUpdateCalendar(int step)
[1622]1331   TRY
[300]1332   {
[987]1333     // Use correct context client to send message
[1030]1334    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1009]1335     for (int i = 0; i < nbSrvPools; ++i)
1336     {
1337       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
1338       CEventClient event(getType(),EVENT_ID_UPDATE_CALENDAR);
[987]1339
[1009]1340         if (contextClientTmp->isServerLeader())
1341         {
1342           CMessage msg;
1343           if (hasServer)
1344             msg<<this->getIdServer(i)<<step;
1345           else
1346             msg<<this->getIdServer()<<step;
1347           const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
1348           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
1349             event.push(*itRank,1,msg);
1350           contextClientTmp->sendEvent(event);
1351         }
1352         else contextClientTmp->sendEvent(event);
1353     }
[300]1354   }
[1622]1355   CATCH_DUMP_ATTR
[509]1356
1357   //! Server side: Receive a message of client annoucing calendar update
[300]1358   void CContext::recvUpdateCalendar(CEventServer& event)
[1622]1359   TRY
[300]1360   {
1361      CBufferIn* buffer=event.subEvents.begin()->buffer;
1362      string id;
[549]1363      *buffer>>id;
1364      get(id)->recvUpdateCalendar(*buffer);
[300]1365   }
[1622]1366   CATCH
[509]1367
1368   //! Server side: Receive a message of client annoucing calendar update
[300]1369   void CContext::recvUpdateCalendar(CBufferIn& buffer)
[1622]1370   TRY
[300]1371   {
[549]1372      int step;
1373      buffer>>step;
1374      updateCalendar(step);
[987]1375      if (hasClient && hasServer)
1376      {       
1377        sendUpdateCalendar(step);
1378      }
[300]1379   }
[1622]1380   CATCH_DUMP_ATTR
[509]1381
1382   //! Client side: Send a message to create header part of netcdf file
[300]1383   void CContext::sendCreateFileHeader(void)
[1622]1384   TRY
[300]1385   {
[987]1386     // Use correct context client to send message
[1030]1387     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
1388     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1009]1389     for (int i = 0; i < nbSrvPools; ++i)
1390     {
1391       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
1392       CEventClient event(getType(),EVENT_ID_CREATE_FILE_HEADER);
[983]1393
[1009]1394       if (contextClientTmp->isServerLeader())
1395       {
1396         CMessage msg;
1397         if (hasServer)
1398           msg<<this->getIdServer(i);
1399         else
1400           msg<<this->getIdServer();
1401         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
1402         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
1403           event.push(*itRank,1,msg) ;
1404         contextClientTmp->sendEvent(event);
1405       }
1406       else contextClientTmp->sendEvent(event);
[300]1407     }
1408   }
[1622]1409   CATCH_DUMP_ATTR
[509]1410
1411   //! Server side: Receive a message of client annoucing the creation of header part of netcdf file
[300]1412   void CContext::recvCreateFileHeader(CEventServer& event)
[1622]1413   TRY
[300]1414   {
1415      CBufferIn* buffer=event.subEvents.begin()->buffer;
1416      string id;
[549]1417      *buffer>>id;
1418      get(id)->recvCreateFileHeader(*buffer);
[300]1419   }
[1622]1420   CATCH
[509]1421
1422   //! Server side: Receive a message of client annoucing the creation of header part of netcdf file
[300]1423   void CContext::recvCreateFileHeader(CBufferIn& buffer)
[1622]1424   TRY
[300]1425   {
[2253]1426      createFileHeader();
[300]1427   }
[1622]1428   CATCH_DUMP_ATTR
[509]1429
1430   //! Client side: Send a message to do some post processing on server
[1025]1431   void CContext::sendProcessingGridOfEnabledFields()
[1622]1432   TRY
[1025]1433   {
1434      // Use correct context client to send message
[1030]1435     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1027]1436     for (int i = 0; i < nbSrvPools; ++i)
1437     {
1438       CContextClient* contextClientTmp = (0 != clientPrimServer.size()) ? clientPrimServer[i] : client;
1439       CEventClient event(getType(),EVENT_ID_PROCESS_GRID_ENABLED_FIELDS);
[1025]1440
[1027]1441       if (contextClientTmp->isServerLeader())
1442       {
1443         CMessage msg;
[1099]1444         if (hasServer)
1445           msg<<this->getIdServer(i);
1446         else
1447           msg<<this->getIdServer();
[1027]1448         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
1449         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
1450           event.push(*itRank,1,msg);
1451         contextClientTmp->sendEvent(event);
1452       }
1453       else contextClientTmp->sendEvent(event);
[1025]1454     }
1455   }
[1622]1456   CATCH_DUMP_ATTR
[1025]1457
1458   //! Server side: Receive a message to do some post processing
1459   void CContext::recvProcessingGridOfEnabledFields(CEventServer& event)
[1622]1460   TRY
[1025]1461   {
1462      CBufferIn* buffer=event.subEvents.begin()->buffer;
1463      string id;
[1144]1464      *buffer>>id;     
[1025]1465   }
[1622]1466   CATCH
[1025]1467
1468   //! Client side: Send a message to do some post processing on server
[509]1469   void CContext::sendPostProcessing()
[1622]1470   TRY
[509]1471   {
[987]1472      // Use correct context client to send message
[1030]1473     // int nbSrvPools = (hasServer) ? clientPrimServer.size() : 1;
1474     int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1009]1475     for (int i = 0; i < nbSrvPools; ++i)
[509]1476     {
[1009]1477       CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
1478       CEventClient event(getType(),EVENT_ID_POST_PROCESS);
1479       if (contextClientTmp->isServerLeader())
1480       {
1481         CMessage msg;
1482         if (hasServer)
1483           msg<<this->getIdServer(i);
1484         else
1485           msg<<this->getIdServer();
1486         const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
1487         for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
[987]1488         event.push(*itRank,1,msg);
[1009]1489         contextClientTmp->sendEvent(event);
1490       }
1491       else contextClientTmp->sendEvent(event);
[509]1492     }
1493   }
[1622]1494   CATCH_DUMP_ATTR
[509]1495
1496   //! Server side: Receive a message to do some post processing
1497   void CContext::recvPostProcessing(CEventServer& event)
[1622]1498   TRY
[509]1499   {
1500      CBufferIn* buffer=event.subEvents.begin()->buffer;
1501      string id;
1502      *buffer>>id;
1503      get(id)->recvPostProcessing(*buffer);
1504   }
[1622]1505   CATCH
[509]1506
1507   //! Server side: Receive a message to do some post processing
1508   void CContext::recvPostProcessing(CBufferIn& buffer)
[1622]1509   TRY
[509]1510   {
[549]1511      CCalendarWrapper::get(CCalendarWrapper::GetDefName())->createCalendar();
[509]1512      postProcessing();
1513   }
[1622]1514   CATCH_DUMP_ATTR
[509]1515
[511]1516   const StdString& CContext::getIdServer()
[1622]1517   TRY
[511]1518   {
1519      if (hasClient)
1520      {
1521        idServer_ = this->getId();
1522        idServer_ += "_server";
1523        return idServer_;
1524      }
1525      if (hasServer) return (this->getId());
1526   }
[1622]1527   CATCH_DUMP_ATTR
[511]1528
[1009]1529   const StdString& CContext::getIdServer(const int i)
[1622]1530   TRY
[1009]1531   {
1532     idServer_ = this->getId();
1533     idServer_ += "_server_";
[1542]1534     idServer_ += std::to_string(static_cast<unsigned long long>(i));
[1009]1535     return idServer_;
1536   }
[1622]1537   CATCH_DUMP_ATTR
[1009]1538
[509]1539   /*!
1540   \brief Do some simple post processings after parsing xml file
1541      After the xml file (iodef.xml) is parsed, it is necessary to build all relations among
1542   created object, e.g: inhertance among fields, domain, axis. After that, all fiels as well as their parents (reference fields),
1543   which will be written out into netcdf files, are processed
1544   */
1545   void CContext::postProcessing()
[1622]1546   TRY
[509]1547   {
1548     if (isPostProcessed) return;
1549
[549]1550      // Make sure the calendar was correctly created
1551      if (!calendar)
1552        ERROR("CContext::postProcessing()", << "A calendar must be defined for the context \"" << getId() << "!\"")
1553      else if (calendar->getTimeStep() == NoneDu)
1554        ERROR("CContext::postProcessing()", << "A timestep must be defined for the context \"" << getId() << "!\"")
[550]1555      // Calendar first update to set the current date equals to the start date
1556      calendar->update(0);
[509]1557
1558      // Find all inheritance in xml structure
1559      this->solveAllInheritance();
1560
[992]1561//      ShowTree(info(10));
[983]1562
[676]1563      // Check if some axis, domains or grids are eligible to for compressed indexed output.
1564      // Warning: This must be done after solving the inheritance and before the rest of post-processing
[1025]1565      checkAxisDomainsGridsEligibilityForCompressedOutput();     
[676]1566
[711]1567      // Check if some automatic time series should be generated
[1144]1568      // Warning: This must be done after solving the inheritance and before the rest of post-processing     
[711]1569
[1025]1570      // The timeseries should only be prepared in client
1571      if (hasClient && !hasServer) prepareTimeseries();
1572
[509]1573      //Initialisation du vecteur 'enabledFiles' contenant la liste des fichiers à sortir.
[1232]1574      findEnabledFiles();
1575      findEnabledWriteModeFiles();
1576      findEnabledReadModeFiles();
1577
[1025]1578      // For now, only read files with client and only one level server
[1232]1579      // if (hasClient && !hasServer) findEnabledReadModeFiles();     
[509]1580
[1232]1581      // Find all enabled fields of each file     
1582      findAllEnabledFieldsInFiles(this->enabledWriteModeFiles);
1583      findAllEnabledFieldsInFiles(this->enabledReadModeFiles);
1584
[1025]1585      // For now, only read files with client and only one level server
[1232]1586      // if (hasClient && !hasServer)
1587      //   findAllEnabledFieldsInFiles(this->enabledReadModeFiles);     
[509]1588
[1144]1589      if (hasClient && !hasServer)
1590      {
[1232]1591        initReadFiles();
1592        // Try to read attributes of fields in file then fill in corresponding grid (or domain, axis)
1593        this->readAttributesOfEnabledFieldsInReadModeFiles();
[1144]1594      }
1595
[823]1596      // Only search and rebuild all reference objects of enable fields, don't transform
[1099]1597      this->solveOnlyRefOfEnabledFields(false);
[823]1598
[1099]1599      // Search and rebuild all reference object of enabled fields, and transform
[1129]1600      this->solveAllRefOfEnabledFieldsAndTransform(false);
[1099]1601
[593]1602      // Find all fields with read access from the public API
[1025]1603      if (hasClient && !hasServer) findFieldsWithReadAccess();
[593]1604      // and solve the all reference for them
[1025]1605      if (hasClient && !hasServer) solveAllRefOfFieldsWithReadAccess();
[593]1606
[509]1607      isPostProcessed = true;
1608   }
[1622]1609   CATCH_DUMP_ATTR
[509]1610
[917]1611   /*!
1612    * Compute the required buffer size to send the attributes (mostly those grid related).
1613    * \param maxEventSize [in/out] the size of the bigger event for each connected server
[1330]1614    * \param [in] contextClient
1615    * \param [in] bufferForWriting True if buffers are used for sending data for writing
1616      This flag is only true for client and server-1 for communication with server-2
[917]1617    */
[1330]1618   std::map<int, StdSize> CContext::getAttributesBufferSize(std::map<int, StdSize>& maxEventSize,
1619                                                           CContextClient* contextClient, bool bufferForWriting /*= "false"*/)
[1622]1620   TRY
[509]1621   {
[1372]1622         // As calendar attributes are sent even if there are no active files or fields, maps are initialized according the size of calendar attributes
1623     std::map<int, StdSize> attributesSize = CCalendarWrapper::get(CCalendarWrapper::GetDefName())->getMinimumBufferSizeForAttributes(contextClient);
1624     maxEventSize = CCalendarWrapper::get(CCalendarWrapper::GetDefName())->getMinimumBufferSizeForAttributes(contextClient);
[731]1625
[1370]1626     std::vector<CFile*>& fileList = this->enabledFiles;
1627     size_t numEnabledFiles = fileList.size();
1628     for (size_t i = 0; i < numEnabledFiles; ++i)
[731]1629     {
[1370]1630//         CFile* file = this->enabledWriteModeFiles[i];
1631        CFile* file = fileList[i];
1632        std::vector<CField*> enabledFields = file->getEnabledFields();
1633        size_t numEnabledFields = enabledFields.size();
1634        for (size_t j = 0; j < numEnabledFields; ++j)
1635        {
1636          const std::map<int, StdSize> mapSize = enabledFields[j]->getGridAttributesBufferSize(contextClient, bufferForWriting);
1637          std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
1638          for (; it != itE; ++it)
1639          {
1640                     // If attributesSize[it->first] does not exist, it will be zero-initialized
1641                     // so we can use it safely without checking for its existence
1642             if (attributesSize[it->first] < it->second)
1643                           attributesSize[it->first] = it->second;
[917]1644
[1370]1645                     if (maxEventSize[it->first] < it->second)
1646                           maxEventSize[it->first] = it->second;
1647          }
1648        }
[731]1649     }
1650     return attributesSize;
1651   }
[1622]1652   CATCH_DUMP_ATTR
[731]1653
[917]1654   /*!
1655    * Compute the required buffer size to send the fields data.
1656    * \param maxEventSize [in/out] the size of the bigger event for each connected server
[1330]1657    * \param [in] contextClient
1658    * \param [in] bufferForWriting True if buffers are used for sending data for writing
1659      This flag is only true for client and server-1 for communication with server-2
[917]1660    */
[1330]1661   std::map<int, StdSize> CContext::getDataBufferSize(std::map<int, StdSize>& maxEventSize,
1662                                                      CContextClient* contextClient, bool bufferForWriting /*= "false"*/)
[1622]1663   TRY
[731]1664   {
[730]1665     std::map<int, StdSize> dataSize;
[509]1666
1667     // Find all reference domain and axis of all active fields
[1330]1668     std::vector<CFile*>& fileList = bufferForWriting ? this->enabledWriteModeFiles : this->enabledReadModeFiles;
1669     size_t numEnabledFiles = fileList.size();
[730]1670     for (size_t i = 0; i < numEnabledFiles; ++i)
[509]1671     {
[1330]1672       CFile* file = fileList[i];
[1193]1673       if (file->getContextClient() == contextClient)
[1184]1674       {
[1370]1675         std::vector<CField*> enabledFields = file->getEnabledFields();
1676         size_t numEnabledFields = enabledFields.size();
1677         for (size_t j = 0; j < numEnabledFields; ++j)
[509]1678         {
[1370]1679           // const std::vector<std::map<int, StdSize> > mapSize = enabledFields[j]->getGridDataBufferSize(contextClient);
1680           const std::map<int, StdSize> mapSize = enabledFields[j]->getGridDataBufferSize(contextClient,bufferForWriting);
1681           std::map<int, StdSize>::const_iterator it = mapSize.begin(), itE = mapSize.end();
1682           for (; it != itE; ++it)
[509]1683           {
[1370]1684             // If dataSize[it->first] does not exist, it will be zero-initialized
1685             // so we can use it safely without checking for its existance
1686                 if (CXios::isOptPerformance)
1687               dataSize[it->first] += it->second;
1688             else if (dataSize[it->first] < it->second)
1689               dataSize[it->first] = it->second;
[917]1690
[1370]1691                 if (maxEventSize[it->first] < it->second)
1692               maxEventSize[it->first] = it->second;
[598]1693           }
[509]1694         }
1695       }
1696     }
[730]1697     return dataSize;
[509]1698   }
[1622]1699   CATCH_DUMP_ATTR
[509]1700
1701   //! Client side: Send infomation of active files (files are enabled to write out)
[1232]1702   void CContext::sendEnabledFiles(const std::vector<CFile*>& activeFiles)
[1622]1703   TRY
[509]1704   {
[1232]1705     int size = activeFiles.size();
[509]1706
1707     // In a context, each type has a root definition, e.g: axis, domain, field.
1708     // Every object must be a child of one of these root definition. In this case
1709     // all new file objects created on server must be children of the root "file_definition"
1710     StdString fileDefRoot("file_definition");
1711     CFileGroup* cfgrpPtr = CFileGroup::get(fileDefRoot);
[1158]1712
[509]1713     for (int i = 0; i < size; ++i)
1714     {
[1232]1715       CFile* f = activeFiles[i];
1716       cfgrpPtr->sendCreateChild(f->getId(),f->getContextClient());
1717       f->sendAllAttributesToServer(f->getContextClient());
1718       f->sendAddAllVariables(f->getContextClient());
[509]1719     }
1720   }
[1622]1721   CATCH_DUMP_ATTR
[509]1722
1723   //! Client side: Send information of active fields (ones are written onto files)
[1232]1724   void CContext::sendEnabledFieldsInFiles(const std::vector<CFile*>& activeFiles)
[1622]1725   TRY
[509]1726   {
[1232]1727     int size = activeFiles.size();
[509]1728     for (int i = 0; i < size; ++i)
1729     {
[1232]1730       activeFiles[i]->sendEnabledFields(activeFiles[i]->getContextClient());
[509]1731     }
1732   }
[1622]1733   CATCH_DUMP_ATTR
[509]1734
[676]1735   //! Client side: Check if the defined axis, domains and grids are eligible for compressed indexed output
1736   void CContext::checkAxisDomainsGridsEligibilityForCompressedOutput()
[1622]1737   TRY
[676]1738   {
1739     if (!hasClient) return;
1740
1741     const vector<CAxis*> allAxis = CAxis::getAll();
1742     for (vector<CAxis*>::const_iterator it = allAxis.begin(); it != allAxis.end(); it++)
1743       (*it)->checkEligibilityForCompressedOutput();
1744
1745     const vector<CDomain*> allDomains = CDomain::getAll();
1746     for (vector<CDomain*>::const_iterator it = allDomains.begin(); it != allDomains.end(); it++)
1747       (*it)->checkEligibilityForCompressedOutput();
1748
1749     const vector<CGrid*> allGrids = CGrid::getAll();
1750     for (vector<CGrid*>::const_iterator it = allGrids.begin(); it != allGrids.end(); it++)
1751       (*it)->checkEligibilityForCompressedOutput();
1752   }
[1622]1753   CATCH_DUMP_ATTR
[676]1754
[711]1755   //! Client side: Prepare the timeseries by adding the necessary files
1756   void CContext::prepareTimeseries()
[1622]1757   TRY
[711]1758   {
1759     if (!hasClient) return;
1760
1761     const std::vector<CFile*> allFiles = CFile::getAll();
1762     for (size_t i = 0; i < allFiles.size(); i++)
1763     {
1764       CFile* file = allFiles[i];
1765
[1158]1766       std::vector<CVariable*> fileVars, fieldVars, vars = file->getAllVariables();
1767       for (size_t k = 0; k < vars.size(); k++)
1768       {
1769         CVariable* var = vars[k];
1770
1771         if (var->ts_target.isEmpty()
1772              || var->ts_target == CVariable::ts_target_attr::file || var->ts_target == CVariable::ts_target_attr::both)
1773           fileVars.push_back(var);
1774
1775         if (!var->ts_target.isEmpty()
1776              && (var->ts_target == CVariable::ts_target_attr::field || var->ts_target == CVariable::ts_target_attr::both))
1777           fieldVars.push_back(var);
1778       }
1779
[711]1780       if (!file->timeseries.isEmpty() && file->timeseries != CFile::timeseries_attr::none)
1781       {
[1158]1782         StdString fileNameStr("%file_name%") ;
1783         StdString tsPrefix = !file->ts_prefix.isEmpty() ? file->ts_prefix : fileNameStr ;
1784         
1785         StdString fileName=file->getFileOutputName();
1786         size_t pos=tsPrefix.find(fileNameStr) ;
1787         while (pos!=std::string::npos)
1788         {
1789           tsPrefix=tsPrefix.replace(pos,fileNameStr.size(),fileName) ;
1790           pos=tsPrefix.find(fileNameStr) ;
1791         }
1792       
[711]1793         const std::vector<CField*> allFields = file->getAllFields();
1794         for (size_t j = 0; j < allFields.size(); j++)
1795         {
1796           CField* field = allFields[j];
1797
1798           if (!field->ts_enabled.isEmpty() && field->ts_enabled)
1799           {
1800             CFile* tsFile = CFile::create();
1801             tsFile->duplicateAttributes(file);
1802
[1158]1803             // Add variables originating from file and targeted to timeserie file
1804             for (size_t k = 0; k < fileVars.size(); k++)
1805               tsFile->getVirtualVariableGroup()->addChild(fileVars[k]);
1806
1807           
[711]1808             tsFile->name = tsPrefix + "_";
1809             if (!field->name.isEmpty())
1810               tsFile->name.get() += field->name;
1811             else if (field->hasDirectFieldReference()) // We cannot use getBaseFieldReference() just yet
1812               tsFile->name.get() += field->field_ref;
1813             else
1814               tsFile->name.get() += field->getId();
1815
1816             if (!field->ts_split_freq.isEmpty())
1817               tsFile->split_freq = field->ts_split_freq;
1818
1819             CField* tsField = tsFile->addField();
1820             tsField->field_ref = field->getId();
1821
[1158]1822             // Add variables originating from file and targeted to timeserie field
1823             for (size_t k = 0; k < fieldVars.size(); k++)
1824               tsField->getVirtualVariableGroup()->addChild(fieldVars[k]);
1825
1826             vars = field->getAllVariables();
1827             for (size_t k = 0; k < vars.size(); k++)
1828             {
1829               CVariable* var = vars[k];
1830
1831               // Add variables originating from field and targeted to timeserie field
1832               if (var->ts_target.isEmpty()
1833                    || var->ts_target == CVariable::ts_target_attr::field || var->ts_target == CVariable::ts_target_attr::both)
1834                 tsField->getVirtualVariableGroup()->addChild(var);
1835
1836               // Add variables originating from field and targeted to timeserie file
1837               if (!var->ts_target.isEmpty()
1838                    && (var->ts_target == CVariable::ts_target_attr::file || var->ts_target == CVariable::ts_target_attr::both))
1839                 tsFile->getVirtualVariableGroup()->addChild(var);
1840             }
1841
[711]1842             tsFile->solveFieldRefInheritance(true);
1843
1844             if (file->timeseries == CFile::timeseries_attr::exclusive)
1845               field->enabled = false;
1846           }
1847         }
1848
1849         // Finally disable the original file is need be
1850         if (file->timeseries == CFile::timeseries_attr::only)
1851          file->enabled = false;
1852       }
1853     }
1854   }
[1622]1855   CATCH_DUMP_ATTR
[711]1856
[509]1857   //! Client side: Send information of reference grid of active fields
[1232]1858   void CContext::sendRefGrid(const std::vector<CFile*>& activeFiles)
[1622]1859   TRY
[509]1860   {
1861     std::set<StdString> gridIds;
[1232]1862     int sizeFile = activeFiles.size();
[509]1863     CFile* filePtr(NULL);
1864
1865     // Firstly, find all reference grids of all active fields
1866     for (int i = 0; i < sizeFile; ++i)
1867     {
[1232]1868       filePtr = activeFiles[i];
[509]1869       std::vector<CField*> enabledFields = filePtr->getEnabledFields();
1870       int sizeField = enabledFields.size();
1871       for (int numField = 0; numField < sizeField; ++numField)
1872       {
1873         if (0 != enabledFields[numField]->getRelGrid())
1874           gridIds.insert(CGrid::get(enabledFields[numField]->getRelGrid())->getId());
1875       }
1876     }
1877
1878     // Create all reference grids on server side
1879     StdString gridDefRoot("grid_definition");
1880     CGridGroup* gridPtr = CGridGroup::get(gridDefRoot);
1881     std::set<StdString>::const_iterator it, itE = gridIds.end();
1882     for (it = gridIds.begin(); it != itE; ++it)
1883     {
1884       gridPtr->sendCreateChild(*it);
[1927]1885       CGrid::get(*it)->sendGrid() ;
[509]1886     }
1887   }
[1622]1888   CATCH_DUMP_ATTR
[509]1889
[1232]1890   //! Client side: Send information of reference domain, axis and scalar of active fields
1891   void CContext::sendRefDomainsAxisScalars(const std::vector<CFile*>& activeFiles)
[1622]1892   TRY
[569]1893   {
[887]1894     std::set<StdString> domainIds, axisIds, scalarIds;
[509]1895
[569]1896     // Find all reference domain and axis of all active fields
[1232]1897     int numEnabledFiles = activeFiles.size();
[569]1898     for (int i = 0; i < numEnabledFiles; ++i)
1899     {
[1232]1900       std::vector<CField*> enabledFields = activeFiles[i]->getEnabledFields();
[569]1901       int numEnabledFields = enabledFields.size();
1902       for (int j = 0; j < numEnabledFields; ++j)
1903       {
[887]1904         const std::vector<StdString>& prDomAxisScalarId = enabledFields[j]->getRefDomainAxisIds();
1905         if ("" != prDomAxisScalarId[0]) domainIds.insert(prDomAxisScalarId[0]);
1906         if ("" != prDomAxisScalarId[1]) axisIds.insert(prDomAxisScalarId[1]);
1907         if ("" != prDomAxisScalarId[2]) scalarIds.insert(prDomAxisScalarId[2]);
[569]1908       }
1909     }
1910
1911     // Create all reference axis on server side
[887]1912     std::set<StdString>::iterator itDom, itAxis, itScalar;
[569]1913     std::set<StdString>::const_iterator itE;
1914
[887]1915     StdString scalarDefRoot("scalar_definition");
1916     CScalarGroup* scalarPtr = CScalarGroup::get(scalarDefRoot);
1917     itE = scalarIds.end();
1918     for (itScalar = scalarIds.begin(); itScalar != itE; ++itScalar)
1919     {
1920       if (!itScalar->empty())
1921       {
1922         scalarPtr->sendCreateChild(*itScalar);
1923         CScalar::get(*itScalar)->sendAllAttributesToServer();
1924       }
1925     }
1926
[569]1927     StdString axiDefRoot("axis_definition");
1928     CAxisGroup* axisPtr = CAxisGroup::get(axiDefRoot);
1929     itE = axisIds.end();
1930     for (itAxis = axisIds.begin(); itAxis != itE; ++itAxis)
1931     {
1932       if (!itAxis->empty())
1933       {
1934         axisPtr->sendCreateChild(*itAxis);
1935         CAxis::get(*itAxis)->sendAllAttributesToServer();
1936       }
1937     }
1938
1939     // Create all reference domains on server side
1940     StdString domDefRoot("domain_definition");
1941     CDomainGroup* domPtr = CDomainGroup::get(domDefRoot);
1942     itE = domainIds.end();
1943     for (itDom = domainIds.begin(); itDom != itE; ++itDom)
1944     {
1945       if (!itDom->empty()) {
1946          domPtr->sendCreateChild(*itDom);
1947          CDomain::get(*itDom)->sendAllAttributesToServer();
1948       }
1949     }
1950   }
[1622]1951   CATCH_DUMP_ATTR
[569]1952
[509]1953   //! Update calendar in each time step
[300]1954   void CContext::updateCalendar(int step)
[1622]1955   TRY
[300]1956   {
[1357]1957      int prevStep = calendar->getStep();
1958
1959      if (prevStep < step)
[639]1960      {
[1358]1961        if (hasClient && !hasServer) // For now we only use server level 1 to read data
1962        {
1963          doPreTimestepOperationsForEnabledReadModeFiles();
1964        }
1965
[1357]1966        info(50) << "updateCalendar : before : " << calendar->getCurrentDate() << endl;
1967        calendar->update(step);
1968        info(50) << "updateCalendar : after : " << calendar->getCurrentDate() << endl;
1969  #ifdef XIOS_MEMTRACK_LIGHT
1970        info(50) << " Current memory used by XIOS : "<<  MemTrack::getCurrentMemorySize()*1.0/(1024*1024)<<" Mbyte, at timestep "<<step<<" of context "<<this->getId()<<endl ;
1971  #endif
1972
1973        if (hasClient && !hasServer) // For now we only use server level 1 to read data
1974        {
1975          doPostTimestepOperationsForEnabledReadModeFiles();
1976          garbageCollector.invalidate(calendar->getCurrentDate());
1977        }
[639]1978      }
[1357]1979      else if (prevStep == step)
1980        info(50) << "updateCalendar: already at step " << step << ", no operation done." << endl;
1981      else // if (prevStep > step)
1982        ERROR("void CContext::updateCalendar(int step)",
1983              << "Illegal calendar update: previous step was " << prevStep << ", new step " << step << "is in the past!")
[300]1984   }
[1622]1985   CATCH_DUMP_ATTR
[509]1986
[1232]1987   void CContext::initReadFiles(void)
[1622]1988   TRY
[1232]1989   {
1990      vector<CFile*>::const_iterator it;
1991
1992      for (it=enabledReadModeFiles.begin(); it != enabledReadModeFiles.end(); it++)
1993      {
1994         (*it)->initRead();
1995      }
1996   }
[1622]1997   CATCH_DUMP_ATTR
[1232]1998
[509]1999   //! Server side: Create header of netcdf file
[1232]2000   void CContext::createFileHeader(void)
[1622]2001   TRY
[300]2002   {
[549]2003      vector<CFile*>::const_iterator it;
[2253]2004      if (!hasClient && hasServer) 
2005        for (it=enabledFiles.begin(); it != enabledFiles.end(); it++) (*it)->initWrite();
2006      else if (hasClient && hasServer)
2007        for (it=enabledReadModeFiles.begin(); it != enabledReadModeFiles.end(); it++)  (*it)->initWrite();
[509]2008   }
[1622]2009   CATCH_DUMP_ATTR
[509]2010
2011   //! Get current context
[347]2012   CContext* CContext::getCurrent(void)
[1622]2013   TRY
[300]2014   {
[549]2015     return CObjectFactory::GetObject<CContext>(CObjectFactory::GetCurrentContextId()).get();
[300]2016   }
[1622]2017   CATCH
[509]2018
2019   /*!
2020   \brief Set context with an id be the current context
2021   \param [in] id identity of context to be set to current
2022   */
[346]2023   void CContext::setCurrent(const string& id)
[1622]2024   TRY
[346]2025   {
2026     CObjectFactory::SetCurrentContextId(id);
2027     CGroupFactory::SetCurrentContextId(id);
2028   }
[1622]2029   CATCH
[509]2030
2031  /*!
2032  \brief Create a context with specific id
2033  \param [in] id identity of new context
2034  \return pointer to the new context or already-existed one with identity id
2035  */
[347]2036  CContext* CContext::create(const StdString& id)
[1622]2037  TRY
[346]2038  {
[549]2039    CContext::setCurrent(id);
[509]2040
[346]2041    bool hasctxt = CContext::has(id);
[347]2042    CContext* context = CObjectFactory::CreateObject<CContext>(id).get();
[549]2043    getRoot();
[347]2044    if (!hasctxt) CGroupFactory::AddChild(root, context->getShared());
[346]2045
2046#define DECLARE_NODE(Name_, name_) \
2047    C##Name_##Definition::create(C##Name_##Definition::GetDefName());
2048#define DECLARE_NODE_PAR(Name_, name_)
2049#include "node_type.conf"
2050
2051    return (context);
2052  }
[1622]2053  CATCH
[697]2054
2055     //! Server side: Receive a message to do some post processing
2056  void CContext::recvRegistry(CEventServer& event)
[1622]2057  TRY
[697]2058  {
2059    CBufferIn* buffer=event.subEvents.begin()->buffer;
2060    string id;
2061    *buffer>>id;
2062    get(id)->recvRegistry(*buffer);
2063  }
[1622]2064  CATCH
[697]2065
2066  void CContext::recvRegistry(CBufferIn& buffer)
[1622]2067  TRY
[697]2068  {
2069    if (server->intraCommRank==0)
2070    {
2071      CRegistry registry(server->intraComm) ;
2072      registry.fromBuffer(buffer) ;
2073      registryOut->mergeRegistry(registry) ;
2074    }
2075  }
[1622]2076  CATCH_DUMP_ATTR
[697]2077
2078  void CContext::sendRegistry(void)
[1622]2079  TRY
[1158]2080  {
[697]2081    registryOut->hierarchicalGatherRegistry() ;
2082
[1009]2083    // Use correct context client to send message
[1030]2084    int nbSrvPools = (this->hasServer) ? (this->hasClient ? this->clientPrimServer.size() : 0) : 1;
[1009]2085    for (int i = 0; i < nbSrvPools; ++i)
2086    {
2087      CContextClient* contextClientTmp = (hasServer) ? clientPrimServer[i] : client;
2088      CEventClient event(CContext::GetType(), CContext::EVENT_ID_SEND_REGISTRY);
[1021]2089        if (contextClientTmp->isServerLeader())
[1009]2090        {
2091           CMessage msg ;
2092           if (hasServer)
2093             msg<<this->getIdServer(i);
2094           else
2095             msg<<this->getIdServer();
2096           if (contextClientTmp->clientRank==0) msg<<*registryOut ;
[1021]2097           const std::list<int>& ranks = contextClientTmp->getRanksServerLeader();
[1009]2098           for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
2099             event.push(*itRank,1,msg);
2100           contextClientTmp->sendEvent(event);
2101         }
2102         else contextClientTmp->sendEvent(event);
2103    }
[697]2104  }
[1622]2105  CATCH_DUMP_ATTR
[697]2106
[1130]2107  /*!
2108  * \fn bool CContext::isFinalized(void)
[1139]2109  * Context is finalized if it received context post finalize event.
[1130]2110  */
[1054]2111  bool CContext::isFinalized(void)
[1622]2112  TRY
[1054]2113  {
[1139]2114    return finalized;
[1054]2115  }
[1622]2116  CATCH_DUMP_ATTR
2117  ///--------------------------------------------------------------
2118  StdString CContext::dumpClassAttributes(void)
2119  {
2120    StdString str;
2121    str.append("enabled files=\"");
2122    int size = this->enabledFiles.size();
2123    for (int i = 0; i < size; ++i)
2124    {
2125      str.append(enabledFiles[i]->getId());
2126      str.append(" ");
2127    }
2128    str.append("\"");
2129    return str;
2130  }
[1054]2131
[335]2132} // namespace xios
Note: See TracBrowser for help on using the repository browser.