source: XIOS/dev/dev_ym/XIOS_COUPLING/src/node/file.cpp @ 2149

Last change on this file since 2149 was 2149, checked in by jderouillat, 3 years ago

Do not remove ranks with empty data from file writing communicators

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
File size: 42.1 KB
Line 
1#include "file.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6#include "object_factory.hpp"
7#include "context.hpp"
8#include "context_server.hpp"
9#include "nc4_data_output.hpp"
10#include "nc4_data_input.hpp"
11#include "calendar_util.hpp"
12#include "date.hpp"
13#include "message.hpp"
14#include "type.hpp"
15#include "xios_spl.hpp"
16#include "context_client.hpp"
17#include "mpi.hpp"
18#include "timer.hpp"
19#include "server.hpp"
20
21namespace xios {
22
23   /// ////////////////////// Dfinitions ////////////////////// ///
24
25   CFile::CFile(void)
26      : CObjectTemplate<CFile>(), CFileAttributes()
27      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
28      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
29   {
30     setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
31     setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
32   }
33
34   CFile::CFile(const StdString & id)
35      : CObjectTemplate<CFile>(id), CFileAttributes()
36      , vFieldGroup(), data_out(), enabledFields(), fileComm(MPI_COMM_NULL)
37      , isOpen(false), read_client(0), checkRead(false), allZoneEmpty(false)
38    {
39      setVirtualFieldGroup(CFieldGroup::create(getId() + "_virtual_field_group"));
40      setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group"));
41    }
42
43   CFile::~CFile(void)
44   { /* Ne rien faire de plus */ }
45
46   ///---------------------------------------------------------------
47  //! Get name of file
48   StdString CFile::GetName(void)   { return (StdString("file")); }
49   StdString CFile::GetDefName(void){ return (CFile::GetName()); }
50   ENodeType CFile::GetType(void)   { return (eFile); }
51
52   //----------------------------------------------------------------
53
54   const StdString CFile::getFileOutputName(void) const
55   TRY
56   {
57     return (name.isEmpty() ? getId() : name) + (name_suffix.isEmpty() ? StdString("") :  name_suffix.getValue());
58   }
59   CATCH
60
61   //----------------------------------------------------------------
62   /*!
63   \brief Get data writer object.
64   Each enabled file in xml represents a physical netcdf file.
65   This function allows to access the data writer object.
66   \return data writer object.
67   */
68   std::shared_ptr<CDataOutput> CFile::getDataOutput(void) const
69   TRY
70   {
71      return data_out;
72   }
73   CATCH
74
75   /*!
76   \brief Get data reader object.
77   Each enabled file in xml represents a physical netcdf file.
78   This function allows to access the data reader object.
79   \return data reader object.
80   */
81   std::shared_ptr<CDataInput> CFile::getDataInput(void) const
82   TRY
83   {
84      return data_in;
85   }
86   CATCH
87
88   /*!
89   \brief Get virtual field group
90      In each file, there always exists a field group which is the ancestor of all
91   fields in the file. This is considered be virtual because it is created automatically during
92   file initialization and it normally doesn't appear on xml file
93   \return Pointer to field group
94   */
95   CFieldGroup* CFile::getVirtualFieldGroup(void) const
96   TRY
97   {
98      return (this->vFieldGroup);
99   }
100   CATCH
101
102   /*!
103   \brief Get virtual variable group
104      In each file, there always exists a variable group which is the ancestor of all
105   variable in the file. This is considered be virtual because it is created automatically during
106   file initialization and it normally doesn't appear on xml file
107   \return Pointer to variable group
108   */
109   CVariableGroup* CFile::getVirtualVariableGroup(void) const
110   TRY
111   {
112      return (this->vVariableGroup);
113   }
114   CATCH
115
116   //! Get all fields of a file
117   std::vector<CField*> CFile::getAllFields(void) const
118   TRY
119   {
120      return (this->vFieldGroup->getAllChildren());
121   }
122   CATCH
123
124   //! Get all variables of a file
125   std::vector<CVariable*> CFile::getAllVariables(void) const
126   TRY
127   {
128      return (this->vVariableGroup->getAllChildren());
129   }
130   CATCH
131
132   //----------------------------------------------------------------
133   /*!
134   \brief Get all enabled fields of file
135      A field is considered to be enabled if it fullfil these conditions: it is enabled, inside a enabled file
136   and its own level is not larger than file output level.
137   \param [in] default_outputlevel default value output level of file
138   \param [in] default_level default value level of field
139   \param [in] default_enabled flag determine by default if field is enabled
140   \return Vector of pointers of enabled fields
141   */
142   std::vector<CField*> CFile::getEnabledFields(int default_outputlevel,
143                                                int default_level,
144                                                bool default_enabled)
145   TRY
146   {
147      if (!this->enabledFields.empty()) return (this->enabledFields);
148
149      const int _outputlevel = (!output_level.isEmpty()) ? output_level.getValue() : default_outputlevel;
150      std::vector<CField*>::iterator it;
151      this->enabledFields = this->getAllFields();
152
153      std::vector<CField*> newEnabledFields;
154
155      for ( it = this->enabledFields.begin(); it != this->enabledFields.end(); it++ )
156      {
157         if (!(*it)->enabled.isEmpty()) // Si l'attribut 'enabled' est dfini ...
158         {
159            if (! (*it)->enabled.getValue()) continue;
160         }
161         else // Si l'attribut 'enabled' n'est pas dfini ...
162         {
163            if (!default_enabled) continue;
164         }
165
166         if (!(*it)->level.isEmpty()) // Si l'attribut 'level' est dfini ...
167         {
168            if ((*it)->level.getValue() > _outputlevel) continue;
169         }
170         else // Si l'attribut 'level' n'est pas dfini ...
171         {
172            if (default_level > _outputlevel) continue;
173         }
174
175         newEnabledFields.push_back(*it);
176      }
177      enabledFields = newEnabledFields;
178
179      return (this->enabledFields);
180   }
181   CATCH_DUMP_ATTR
182
183   //----------------------------------------------------------------
184   //! Change virtual field group to a new one
185   void CFile::setVirtualFieldGroup(CFieldGroup* newVFieldGroup)
186   TRY
187   {
188      this->vFieldGroup = newVFieldGroup;
189   }
190   CATCH_DUMP_ATTR
191
192   //! Change virtual variable group to new one
193   void CFile::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
194   TRY
195   {
196      this->vVariableGroup = newVVariableGroup;
197   }
198   CATCH_DUMP_ATTR
199
200   //----------------------------------------------------------------
201   bool CFile::isSyncTime(void)
202   TRY
203   {
204     CContext* context = CContext::getCurrent();
205     const CDate& currentDate = context->calendar->getCurrentDate();
206     if (!sync_freq.isEmpty())
207     {
208       if (lastSync + sync_freq.getValue() < currentDate)
209       {
210         lastSync = currentDate;
211         return true;
212        }
213      }
214      return false;
215    }
216    CATCH_DUMP_ATTR
217
218   //! Initialize a file in order to write into it
219   void CFile::initWrite(void)
220   TRY
221   {
222      CContext* context = CContext::getCurrent();
223      const CDate& currentDate = context->calendar->getCurrentDate();
224
225      lastSync  = currentDate;
226      lastSplit = currentDate;
227      if (!split_freq.isEmpty())
228      {
229        StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
230        if (context->registryIn->foundKey(keySuffix+"splitStart") && context->registryIn->foundKey(keySuffix+"splitEnd"))
231        {
232          CDate savedSplitStart(*context->getCalendar()), savedSplitEnd(*context->getCalendar());
233          context->registryIn->getKey(keySuffix+"splitStart", savedSplitStart);
234          context->registryIn->getKey(keySuffix+"splitEnd",   savedSplitEnd);
235
236          if (savedSplitStart <= lastSplit && lastSplit <= savedSplitEnd)
237            lastSplit = savedSplitStart;
238        }
239      }
240      isOpen = false;     
241
242      set<StdString> setAxis;
243      set<StdString> setDomains;
244
245      std::vector<CField*>::iterator it, end = this->enabledFields.end();
246      for (it = this->enabledFields.begin(); it != end; it++)
247      {
248         CField* field = *it;         
249         std::vector<CAxis*> vecAxis = field->getGrid()->getAxis();
250         for (size_t i = 0; i < vecAxis.size(); ++i)
251           setAxis.insert(vecAxis[i]->getAxisOutputName());
252         std::vector<CDomain*> vecDomains = field->getGrid()->getDomains();
253         for (size_t i = 0; i < vecDomains.size(); ++i)
254           setDomains.insert(vecDomains[i]->getDomainOutputName());
255      }
256      nbAxis = setAxis.size();
257      nbDomains = setDomains.size();
258
259      // create sub communicator for file
260      createSubComFile();  // NetCDF runs now write of null data
261      MPI_Comm_dup(context->intraComm_, &fileComm) ;
262
263      if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
264    }
265    CATCH_DUMP_ATTR
266
267    //! Initialize a file in order to write into it
268    void CFile::initRead(void)
269    TRY
270    {
271      CContext* context = CContext::getCurrent();
272      if (checkRead) return;
273      //createSubComFile();
274      allZoneEmpty = false; 
275      MPI_Comm_dup(context->intraComm_, &fileComm) ;
276      checkRead = true;
277    }
278    CATCH_DUMP_ATTR
279
280    /*!
281      Create a sub communicator in which processes participate in reading/opening file
282      Seems to be deprecated (allZoneEmpty too, which is always false)
283    */
284    void CFile::createSubComFile()
285    TRY
286    {
287      CContext* context = CContext::getCurrent();
288
289      // create sub communicator for file
290      allZoneEmpty = true;     
291      std::vector<CField*>::iterator it, end = this->enabledFields.end();
292      for (it = this->enabledFields.begin(); it != end; it++)
293      {
294         CField* field = *it;
295         bool nullGrid = (nullptr == field->getGrid());
296         allZoneEmpty &= nullGrid ? false : !field->getGrid()->doGridHaveDataToWrite();
297      }
298
299      int color = allZoneEmpty ? 0 : 1;
300      MPI_Comm_split(context->intraComm_, color, context->intraCommRank_, &fileComm);
301      if (allZoneEmpty) MPI_Comm_free(&fileComm);
302    }
303    CATCH_DUMP_ATTR
304
305    /*
306       Check condition to write into a file
307       For now, we only use the level-2 server to write files (if this mode is activated)
308       or classical server to do this job.
309    */
310    void CFile::checkWriteFile(void)
311    TRY
312    {
313      CContext* context = CContext::getCurrent();
314
315      if (mode.isEmpty() || mode.getValue() == mode_attr::write)
316      {
317        CTimer::get("Files : create headers").resume();
318        if (!isOpen) createHeader();
319        CTimer::get("Files : create headers").suspend();
320        checkSync();
321      }       
322      checkSplit(); 
323    }
324    CATCH_DUMP_ATTR
325
326    /*
327       Check condition to read from a file
328       For now, we only use the level-1 server to write files (if this mode is activated)
329       or classical server to do this job.
330       This function can be used by client for reading metadata
331    */
332    void CFile::checkReadFile(void)
333    TRY
334    {
335      CContext* context = CContext::getCurrent();
336      // Done by classical server or secondary server
337      // TODO: This condition should be changed soon. It only works with maximum number of level as 2
338
339//ym      if (CServer::serverLevel == 0 || CServer::serverLevel == 1)
340// ym client must doing it also
341//      if (context->getServiceType()==CServicesManager::IO_SERVER || context->getServiceType()==CServicesManager::GATHERER)
342//      {
343        if (!mode.isEmpty() && mode.getValue() == mode_attr::read)
344        {
345          CTimer::get("Files : open headers").resume();
346         
347          if (!isOpen) openInReadMode();
348
349          CTimer::get("Files : open headers").suspend();
350        }
351        //checkSplit(); // Really need for reading?
352 //     }
353    }
354    CATCH_DUMP_ATTR
355
356    /*!
357      Verify if a process participates in an opening-file communicator
358      \return true if the process doesn't participate in opening file
359    */
360    bool CFile::isEmptyZone()
361    TRY
362    {
363      return allZoneEmpty;
364    }
365    CATCH_DUMP_ATTR
366
367    /*!
368    \brief Verify if synchronisation should be done
369        If syn option is enabled, syn frequence and current time will be used to
370    calculate the moment to syn file(s)
371    \return True if it is the moment to synchronize file, otherwise false
372    */
373   bool CFile::checkSync(void)
374   TRY
375   {
376     CContext* context = CContext::getCurrent();
377     const CDate& currentDate = context->calendar->getCurrentDate();
378     if (!sync_freq.isEmpty())
379     {
380       if (lastSync + sync_freq.getValue() <= currentDate)
381       {
382         lastSync = currentDate;
383         data_out->syncFile();
384         return true;
385        }
386      }
387      return false;
388    }
389   CATCH_DUMP_ATTR
390
391    /*!
392    \brief Verify if splitting should be done
393        If split option is enabled, split frequence and current time will be used to
394    calculate the moment to split file
395    \return True if it is the moment to split file, otherwise false
396    */
397    bool CFile::checkSplit(void)
398    TRY
399    {
400      CContext* context = CContext::getCurrent();
401      const CDate& currentDate = context->calendar->getCurrentDate();
402      if (!split_freq.isEmpty())
403      {
404        if (currentDate > lastSplit + split_freq.getValue())
405        {
406          lastSplit = lastSplit + split_freq.getValue();
407          std::vector<CField*>::iterator it, end = this->enabledFields.end();
408/*          for (it = this->enabledFields.begin(); it != end; it++)
409          {
410            (*it)->resetNStep();
411            (*it)->resetNStepMax();
412          }*/
413          if (mode.isEmpty() || mode.getValue() == mode_attr::write)
414            createHeader();
415          else
416            openInReadMode();
417          return true;
418        }
419      }
420      return false;
421    }
422    CATCH_DUMP_ATTR
423
424
425   /*!
426   \brief Create header of netcdf file
427   There are some information to fill in header of each netcdf.
428   */
429   void CFile::createHeader(void)
430   TRY
431   {
432      CContext* context = CContext::getCurrent();
433     
434      if (!allZoneEmpty)
435      {
436         StdString filename = getFileOutputName();
437
438// determine splitting format in the file name  : firstPart%start_date%middlePart%end_date%lastPart
439
440         std::string strStartDate="%start_date%" ;
441         std::string strEndDate="%end_date%" ;
442
443         std::string firstPart ;
444         std::string middlePart ;
445         std::string lastPart ;
446         size_t pos1, pos2 ;
447         bool hasStartDate=false ;
448         bool hasEndDate=false ;
449         bool hasSplit = (!split_freq.isEmpty());
450                 
451         pos1=filename.find(strStartDate) ;
452         if (pos1!=std::string::npos)
453         {
454           firstPart=filename.substr(0,pos1) ;
455           pos1+=strStartDate.size() ;
456           hasStartDate=true ;
457         }
458         else pos1=0 ;
459
460         pos2=filename.find(strEndDate,pos1) ;
461         if (pos2!=std::string::npos)
462         {
463           middlePart=filename.substr(pos1,pos2-pos1) ;
464           pos2+=strEndDate.size() ;
465           lastPart=filename.substr(pos2,filename.size()-pos2) ;
466           hasEndDate=true ;
467         }
468         else middlePart=filename.substr(pos1,filename.size()) ;
469
470         if (!hasStartDate && !hasEndDate)
471         {
472           hasStartDate=true ;
473           hasEndDate=true;
474           firstPart=middlePart ;
475           if (hasSplit) firstPart +="_";
476           middlePart="-" ;
477         }
478   
479         StdOStringStream oss;
480
481         if (!split_freq.isEmpty())
482         {
483           CDate split_start ;
484           CDate splitEnd ;
485           if (!split_start_offset.isEmpty()) split_start=lastSplit + split_start_offset ;
486           else split_start=lastSplit ;
487
488           splitEnd = lastSplit + split_freq ;
489           if (!split_last_date.isEmpty())
490           {
491             CDate splitLastDate=CDate::FromString(split_last_date,*CContext::getCurrent()->getCalendar()) ;
492             if( splitLastDate < splitEnd)  splitEnd=splitLastDate ;
493           }
494           
495           if (!split_end_offset.isEmpty()) splitEnd = splitEnd + split_end_offset;
496           else splitEnd = splitEnd - 1 * Second;
497
498           string splitFormat;
499           if (split_freq_format.isEmpty())
500           {
501             CDuration splitFreq = split_freq.getValue();
502             splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
503             if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
504             else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
505             else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
506             else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
507             else if (splitFreq.month != 0) splitFormat = "%y%mo";
508             else splitFormat = "%y";
509           }
510           else splitFormat = split_freq_format;
511
512           oss << firstPart ;
513           if (hasStartDate) oss << split_start.getStr(splitFormat) ;
514           oss << middlePart ;
515           if (hasEndDate) oss << splitEnd.getStr(splitFormat);
516           oss << lastPart ;
517
518           StdString keySuffix("CFile::"+getFileOutputName()+"::") ; 
519           context->registryOut->setKey(keySuffix+"splitStart", lastSplit);
520           context->registryOut->setKey(keySuffix+"splitEnd",   splitEnd);
521         }
522         else oss<<firstPart<<lastPart ;
523
524        bool append = !this->append.isEmpty() && this->append.getValue();
525
526         bool useClassicFormat = !format.isEmpty() && format == format_attr::netcdf4_classic;
527         bool useCFConvention = convention.isEmpty() || convention == convention_attr::CF;
528
529         bool multifile = true;
530         if (!type.isEmpty())
531         {
532           if (type == type_attr::one_file) multifile = false;
533           else if (type == type_attr::multiple_file) multifile = true;
534
535         }
536#ifndef USING_NETCDF_PAR
537         if (!multifile)
538         {
539            info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
540            multifile = true;
541          }
542#endif
543         if (multifile)
544         {
545            int commSize, commRank;
546            MPI_Comm_size(fileComm, &commSize);
547            MPI_Comm_rank(fileComm, &commRank);
548
549            if (context->intraCommSize_ > 1)
550            {
551              oss << "_" ;
552              int width=0; int n = commSize-1;
553              while (n != 0) { n = n / 10; width++;}
554              if (!min_digits.isEmpty())
555                if (width < min_digits) width = min_digits;
556              oss.width(width);
557              oss.fill('0');
558              oss << right << commRank;
559            }
560         }
561         oss << ".nc";
562
563         bool isCollective = par_access.isEmpty() ||  par_access == par_access_attr::collective;
564
565         if (isOpen) data_out->closeFile();
566
567        data_out = std::shared_ptr<CDataOutput>(new CNc4DataOutput(this, oss.str(), append, useClassicFormat, useCFConvention,
568                                                              fileComm, multifile, isCollective, time_counter_name));
569        isOpen = true;
570
571        data_out->writeFile(CFile::get(this));
572
573        if (!useCFConvention) sortEnabledFieldsForUgrid();
574
575        // Do not recreate the file structure if opening an existing file
576        if (!data_out->IsInAppendMode())
577        {
578          std::vector<CField*>::iterator it, end = this->enabledFields.end();
579          for (it = this->enabledFields.begin(); it != end; it++)
580          {
581            CField* field = *it;
582            this->data_out->writeFieldGrid(field);
583          }
584          this->data_out->writeTimeDimension();
585
586          for (it = this->enabledFields.begin(); it != end; it++)
587          {
588            CField* field = *it;
589            this->data_out->writeFieldTimeAxis(field);
590          }
591         
592          for (it = this->enabledFields.begin(); it != end; it++)
593          {
594            CField* field = *it;
595            this->data_out->writeField(field);
596          }
597
598          vector<CVariable*> listVars = getAllVariables();
599          for (vector<CVariable*>::iterator it = listVars.begin(); it != listVars.end(); it++)
600            this->data_out->writeAttribute(*it);
601
602          this->data_out->definition_end();
603        }
604        else
605        {
606          // check time axis even in append mode
607          std::vector<CField*>::iterator it, end = this->enabledFields.end();
608          for (it = this->enabledFields.begin(); it != end; it++)
609          {
610            CField* field = *it;
611            this->data_out->writeFieldTimeAxis(field);
612          }
613        }
614      }
615   }
616   CATCH_DUMP_ATTR
617
618  /*!
619  \brief Open an existing NetCDF file in read-only mode
620  */
621  void CFile::openInReadMode()
622  TRY
623  {
624    CContext* context = CContext::getCurrent();
625    MPI_Comm readComm = this->fileComm;
626
627    if (!allZoneEmpty)
628    {
629      StdString filename = getFileOutputName();
630      StdOStringStream oss;
631      oss << filename;
632
633      if (!split_freq.isEmpty())
634      {
635        string splitFormat;
636        if (split_freq_format.isEmpty())
637        {
638          CDuration splitFreq = split_freq.getValue();
639          splitFreq.solveTimeStep(*CContext::getCurrent()->getCalendar());
640          if (splitFreq.second != 0) splitFormat = "%y%mo%d%h%mi%s";
641          else if (splitFreq.minute != 0) splitFormat = "%y%mo%d%h%mi";
642          else if (splitFreq.hour != 0) splitFormat = "%y%mo%d%h";
643          else if (splitFreq.day != 0) splitFormat = "%y%mo%d";
644          else if (splitFreq.month != 0) splitFormat = "%y%mo";
645          else splitFormat = "%y";
646        }
647        else splitFormat = split_freq_format;
648        oss << "_" << lastSplit.getStr(splitFormat)
649        << "-" << (lastSplit + split_freq.getValue() - 1 * Second).getStr(splitFormat);
650      }
651
652      bool multifile = true;
653      if (!type.isEmpty())
654      {
655        if (type == type_attr::one_file) multifile = false;
656        else if (type == type_attr::multiple_file) multifile = true;
657      }
658  #ifndef USING_NETCDF_PAR
659      if (!multifile)
660      {
661        info(0) << "!!! Warning -> Using non parallel version of netcdf, switching in multiple_file mode for file : " << filename << " ..." << endl;
662        multifile = true;
663      }
664  #endif
665      if (multifile)
666      {
667        int commSize, commRank;
668        MPI_Comm_size(readComm, &commSize);
669        MPI_Comm_rank(readComm, &commRank);
670
671        if (context->intraCommSize_ > 1)
672        {
673          oss << "_";
674          int width = 0, n = commSize - 1;
675          while (n != 0) { n = n / 10; width++; }
676          if (!min_digits.isEmpty() && width < min_digits)
677            width = min_digits;
678          oss.width(width);
679          oss.fill('0');
680          oss << right << commRank;
681        }
682      }
683      oss << ".nc";
684
685      bool isCollective = par_access.isEmpty() || par_access == par_access_attr::collective;
686      bool readMetaDataPar = true;
687      if (context->getServiceType()==CServicesManager::CLIENT) readMetaDataPar = (read_metadata_par.isEmpty()) ? false : read_metadata_par;
688
689      if (isOpen) data_out->closeFile();
690      bool ugridConvention = !convention.isEmpty() ? (convention == convention_attr::UGRID) : false;
691      if (time_counter_name.isEmpty())
692        data_in = std::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention));
693      else
694        data_in = std::shared_ptr<CDataInput>(new CNc4DataInput(oss.str(), readComm, multifile, isCollective, readMetaDataPar, ugridConvention, time_counter_name));
695      isOpen = true;
696    }
697  }
698  CATCH_DUMP_ATTR
699
700   //! Close file
701   void CFile::close(void)
702   TRY
703   {
704     if (!allZoneEmpty)
705       if (isOpen)
706       {
707         if (mode.isEmpty() || mode.getValue() == mode_attr::write)
708          this->data_out->closeFile();
709         else
710          this->data_in->closeFile();
711        isOpen = false;
712       }
713      if (fileComm != MPI_COMM_NULL) MPI_Comm_free(&fileComm);
714   }
715   CATCH_DUMP_ATTR
716
717   //----------------------------------------------------------------
718
719   void CFile::readAttributesOfEnabledFieldsInReadMode()
720   TRY
721   {
722     if (enabledFields.empty()) return;
723
724     // Just check file and try to open it
725     if (time_counter_name.isEmpty()) time_counter_name = "time_counter";
726
727     checkReadFile();
728
729     for (int idx = 0; idx < enabledFields.size(); ++idx)
730     {
731        // First of all, find out which domain and axis associated with this field
732        enabledFields[idx]->solveGridReference();
733
734        // Read attributes of domain and axis from this file
735        this->data_in->readFieldAttributesMetaData(enabledFields[idx]);
736
737        // Now complete domain and axis associated with this field
738       
739       //ym => this a problem in wich order are done the grid generation. Probably metadata are read in file and after the grid is ditributed by the filter
740       // => must be checked in detail. But solveGenerated grid is not existing any more with new transformation framework
741       // enabledFields[idx]->solveGenerateGrid();
742
743        // Read necessary value from file
744        this->data_in->readFieldAttributesValues(enabledFields[idx]);
745
746        // Fill attributes for base reference
747        enabledFields[idx]->solveGridDomainAxisBaseRef();
748     }
749
750     // Now everything is ok, close it
751     close();
752   }
753   CATCH_DUMP_ATTR
754
755   void CFile::readFieldAttributesMetaData(CField* field)
756   {
757     this->data_in->readFieldAttributesMetaData(field);
758   }
759
760   void CFile::readFieldAttributesValues(CField* field)
761   {
762     this->data_in->readFieldAttributesValues(field);
763   }
764   /*!
765   \brief Parse xml file and write information into file object
766   \param [in] node xmld node corresponding in xml file
767   */
768   void CFile::parse(xml::CXMLNode & node)
769   TRY
770   {
771      SuperClass::parse(node);
772
773      if (node.goToChildElement())
774      {
775        do
776        {
777           if (node.getElementName()=="field" || node.getElementName()=="field_group") this->getVirtualFieldGroup()->parseChild(node);
778           else if (node.getElementName()=="variable" || node.getElementName()=="variable_group") this->getVirtualVariableGroup()->parseChild(node);
779        } while (node.goToNextElement());
780        node.goToParentElement();
781      }
782   }
783   CATCH_DUMP_ATTR
784
785   //----------------------------------------------------------------
786
787   /*!
788   \brief Represent a file in form of string with all its info
789   \return String
790   */
791   StdString CFile::toString(void) const
792   TRY
793   {
794      StdOStringStream oss;
795
796      oss << "<" << CFile::GetName() << " ";
797      if (this->hasId())
798         oss << " id=\"" << this->getId() << "\" ";
799      oss << SuperClassAttribute::toString() << ">" << std::endl;
800      if (this->getVirtualFieldGroup() != NULL)
801         oss << *this->getVirtualFieldGroup() << std::endl;
802      oss << "</" << CFile::GetName() << " >";
803      return (oss.str());
804   }
805   CATCH
806
807   //----------------------------------------------------------------
808
809   /*!
810   \brief Find all inheritace among objects in a file.
811   \param [in] apply (true) write attributes of parent into ones of child if they are empty
812                     (false) write attributes of parent into a new container of child
813   \param [in] parent
814   */
815   void CFile::solveDescInheritance(bool apply, const CAttributeMap * const parent)
816   TRY
817   {
818      SuperClassAttribute::setAttributes(parent,apply);
819      this->getVirtualFieldGroup()->solveDescInheritance(apply, NULL);
820      this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
821   }
822   CATCH_DUMP_ATTR
823
824   //----------------------------------------------------------------
825
826   void CFile::checkGridOfEnabledFields()
827   TRY
828   { 
829     int size = this->enabledFields.size();
830     for (int i = 0; i < size; ++i)
831     {
832       this->enabledFields[i]->checkGridOfEnabledFields();
833     }
834   }
835   CATCH_DUMP_ATTR
836
837   void CFile::sendGridComponentOfEnabledFields()
838   TRY
839   { 
840     int size = this->enabledFields.size();
841     for (int i = 0; i < size; ++i)
842     {
843       this->enabledFields[i]->sendGridComponentOfEnabledFields();
844     }
845   }
846   CATCH_DUMP_ATTR
847
848   /*!
849   \brief Sorting domains with the same name (= describing the same mesh) in the decreasing order of nvertex for UGRID files.
850   This insures that the domain with the highest nvertex is written first and thus all known mesh connectivity is generated at once by this domain.
851   */
852   void CFile::sortEnabledFieldsForUgrid()
853   TRY
854   {
855     int size = this->enabledFields.size();
856     std::vector<int> domainNvertices;
857     std::vector<StdString> domainNames;
858
859     for (int i = 0; i < size; ++i)
860     {
861       std::vector<CDomain*> domain = this->enabledFields[i]->getRelGrid()->getDomains();
862       if (domain.size() != 1)
863       {
864         ERROR("void CFile::sortEnabledFieldsForUgrid()",
865               "A domain, and only one, should be defined for grid "<< this->enabledFields[i]->getRelGrid()->getId() << ".");
866       }
867       StdString domainName = domain[0]->getDomainOutputName();
868       int nvertex;
869       if (domain[0]->nvertex.isEmpty())
870       {
871         ERROR("void CFile::sortEnabledFieldsForUgrid()",
872               "Attributes nvertex must be defined for domain "<< domain[0]->getDomainOutputName() << ".");
873       }
874       else
875         nvertex = domain[0]->nvertex;
876
877       for (int j = 0; j < i; ++j)
878       {
879         if (domainName == domainNames[j] && nvertex > domainNvertices[j])
880         {
881           CField* tmpSwap = this->enabledFields[j];
882           this->enabledFields[j] = this->enabledFields[i];
883           this->enabledFields[i] = tmpSwap;
884           domainNames.push_back(domainNames[j]);
885           domainNames[j] = domainName;
886           domainNvertices.push_back(domainNvertices[j]);
887           domainNvertices[j] = nvertex;
888         }
889         else
890         {
891           domainNames.push_back(domainName);
892           domainNvertices.push_back(nvertex);
893         }
894       }
895       if (i==0)
896       {
897         domainNames.push_back(domainName);
898         domainNvertices.push_back(nvertex);
899       }
900     }
901   }
902   CATCH_DUMP_ATTR
903
904   void CFile::sendGridOfEnabledFields()
905   TRY
906   { 
907     int size = this->enabledFields.size();
908     for (int i = 0; i < size; ++i)
909     {
910       this->enabledFields[i]->sendGridOfEnabledFields();
911     }
912   }
913   CATCH_DUMP_ATTR
914
915
916   /*!
917    * Post-process the filter graph for each active field.
918    */
919   void CFile::postProcessFilterGraph()
920   TRY
921   {
922     int size = this->enabledFields.size();
923     for (int i = 0; i < size; ++i)
924     {
925       this->enabledFields[i]->checkIfMustAutoTrigger();
926     }
927   }
928   CATCH_DUMP_ATTR
929
930   /*!
931     Prefetching the data for enabled fields read from file.
932   */
933   void CFile::prefetchEnabledReadModeFields(void)
934   TRY
935   {
936     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
937       return;
938
939     int size = this->enabledFields.size();
940     for (int i = 0; i < size; ++i)
941       this->enabledFields[i]->sendReadDataRequest(CContext::getCurrent()->getCalendar()->getCurrentDate());
942   }
943   CATCH_DUMP_ATTR
944
945   /*!
946     Do all pre timestep operations for enabled fields in read mode:
947      - Check that the data excepted from server has been received
948      - Check if some filters must auto-trigger
949   */
950   void CFile::doPreTimestepOperationsForEnabledReadModeFields(void)
951   TRY
952   {
953     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
954       return;
955
956     int size = this->enabledFields.size();
957     for (int i = 0; i < size; ++i)
958     {
959       this->enabledFields[i]->checkForLateDataFromServer();
960       this->enabledFields[i]->autoTriggerIfNeeded();
961     }
962   }
963   CATCH_DUMP_ATTR
964
965   /*!
966     Do all post timestep operations for enabled fields in read mode:
967      - Prefetch the data read from file when needed
968   */
969   void CFile::doPostTimestepOperationsForEnabledReadModeFields(void)
970   TRY
971   {
972     if (mode.isEmpty() || mode.getValue() != mode_attr::read)
973       return;
974
975     int size = this->enabledFields.size();
976     for (int i = 0; i < size; ++i)
977     {
978       this->enabledFields[i]->sendReadDataRequestIfNeeded();
979     }
980   }
981   CATCH_DUMP_ATTR
982
983   void CFile::solveFieldRefInheritance(bool apply)
984   TRY
985   {
986      // Rsolution des hritages par rfrence de chacun des champs contenus dans le fichier.
987      std::vector<CField*> allF = this->getAllFields();
988      for (unsigned int i = 0; i < allF.size(); i++)
989         allF[i]->solveRefInheritance(apply);
990   }
991   CATCH_DUMP_ATTR
992
993   //----------------------------------------------------------------
994
995   /*!
996   \brief Add a field into file.
997      A field is added into file and it will be written out if the file is enabled and
998   level of this field is smaller than level_output. A new field won't be created if one
999   with id has already existed
1000   \param [in] id String identity of new field
1001   \return Pointer to added (or already existed) field
1002   */
1003   CField* CFile::addField(const string& id)
1004   TRY
1005   {
1006     return vFieldGroup->createChild(id);
1007   }
1008   CATCH_DUMP_ATTR
1009
1010   /*!
1011   \brief Add a field group into file.
1012      A field group is added into file and it will play a role as parents for fields.
1013   A new field group won't be created if one with id has already existed
1014   \param [in] id String identity of new field group
1015   \return Pointer to added (or already existed) field group
1016   */
1017   CFieldGroup* CFile::addFieldGroup(const string& id)
1018   TRY
1019   {
1020     return vFieldGroup->createChildGroup(id);
1021   }
1022   CATCH_DUMP_ATTR
1023
1024   /*!
1025   \brief Add a variable into file.
1026      A variable is added into file and if one with id has already existed, pointer to
1027   it will be returned.
1028      Variable as long as attributes are information container of file.
1029   However, whereas attributes are "fixed" information, variables provides a more flexible way to user
1030   to fill in (extra) information for a file.
1031   \param [in] id String identity of new variable
1032   \return Pointer to added (or already existed) variable
1033   */
1034   CVariable* CFile::addVariable(const string& id)
1035   TRY
1036   {
1037     return vVariableGroup->createChild(id);
1038   }
1039   CATCH_DUMP_ATTR
1040
1041   /*!
1042   \brief Add a variable group into file.
1043      A variable group is added into file and it will play a role as parents for variables.
1044   A new variable group won't be created if one with id has already existed
1045   \param [in] id String identity of new variable group
1046   \return Pointer to added (or already existed) variable group
1047   */
1048   CVariableGroup* CFile::addVariableGroup(const string& id)
1049   TRY
1050   {
1051     return vVariableGroup->createChildGroup(id);
1052   }
1053   CATCH_DUMP_ATTR
1054
1055   void CFile::setContextClient(CContextClient* newContextClient)
1056   TRY
1057   {
1058     client = newContextClient;
1059     size_t size = this->enabledFields.size();
1060     for (size_t i = 0; i < size; ++i)
1061     {
1062       this->enabledFields[i]->setContextClient(newContextClient);
1063     }
1064   }
1065   CATCH_DUMP_ATTR
1066
1067   CContextClient* CFile::getContextClient()
1068   TRY
1069   {
1070     return client;
1071   }
1072   CATCH_DUMP_ATTR
1073
1074   void CFile::setReadContextClient(CContextClient* readContextclient)
1075   TRY
1076   {
1077     read_client = readContextclient;
1078   }
1079   CATCH_DUMP_ATTR
1080
1081   CContextClient* CFile::getReadContextClient()
1082   TRY
1083   {
1084     return read_client;
1085   }
1086   CATCH_DUMP_ATTR
1087
1088   /*!
1089   \brief Send a message to create a field on server side
1090   \param[in] id String identity of field that will be created on server
1091   */
1092   void CFile::sendAddField(const string& id, CContextClient* client)
1093   TRY
1094   {
1095      sendAddItem(id, EVENT_ID_ADD_FIELD, client);
1096   }
1097   CATCH_DUMP_ATTR
1098
1099   /*!
1100   \brief Send a message to create a field group on server side
1101   \param[in] id String identity of field group that will be created on server
1102   */
1103   void CFile::sendAddFieldGroup(const string& id, CContextClient* client)
1104   TRY
1105   {
1106      sendAddItem(id, (int)EVENT_ID_ADD_FIELD_GROUP, client);
1107   }
1108   CATCH_DUMP_ATTR
1109
1110   /*!
1111   \brief Receive a message annoucing the creation of a field on server side
1112   \param[in] event Received event
1113   */
1114   void CFile::recvAddField(CEventServer& event)
1115   TRY
1116   {
1117
1118      CBufferIn* buffer = event.subEvents.begin()->buffer;
1119      string id;
1120      *buffer>>id;
1121      get(id)->recvAddField(*buffer);
1122   }
1123   CATCH
1124
1125   /*!
1126   \brief Receive a message annoucing the creation of a field on server side
1127   \param[in] buffer Buffer containing message
1128   */
1129   void CFile::recvAddField(CBufferIn& buffer)
1130   TRY
1131   {
1132      string id;
1133      buffer>>id;
1134      addField(id);
1135   }
1136   CATCH_DUMP_ATTR
1137
1138   /*!
1139   \brief Receive a message annoucing the creation of a field group on server side
1140   \param[in] event Received event
1141   */
1142   void CFile::recvAddFieldGroup(CEventServer& event)
1143   TRY
1144   {
1145
1146      CBufferIn* buffer = event.subEvents.begin()->buffer;
1147      string id;
1148      *buffer>>id;
1149      get(id)->recvAddFieldGroup(*buffer);
1150   }
1151   CATCH
1152
1153   /*!
1154   \brief Receive a message annoucing the creation of a field group on server side
1155   \param[in] buffer Buffer containing message
1156   */
1157   void CFile::recvAddFieldGroup(CBufferIn& buffer)
1158   TRY
1159   {
1160      string id;
1161      buffer>>id;
1162      addFieldGroup(id);
1163   }
1164   CATCH_DUMP_ATTR
1165
1166   /*!
1167   \brief Send messages to duplicate all variables on server side
1168      Because each variable has also its attributes. So first thing to do is replicate
1169   all these attributes on server side. Because variable can have a value, the second thing
1170   is to duplicate this value on server, too.
1171   */
1172   void CFile::sendAddAllVariables(CContextClient* client)
1173   TRY
1174   {
1175     std::vector<CVariable*> allVar = getAllVariables();
1176     std::vector<CVariable*>::const_iterator it = allVar.begin();
1177     std::vector<CVariable*>::const_iterator itE = allVar.end();
1178
1179     for (; it != itE; ++it)
1180     {
1181       this->sendAddVariable((*it)->getId(), client);
1182       (*it)->sendAllAttributesToServer(client);
1183       (*it)->sendValue(client);
1184     }
1185   }
1186   CATCH_DUMP_ATTR
1187
1188   /*!
1189   \brief Send a message to create a variable group on server side
1190   \param[in] id String identity of variable group that will be created on server
1191   \param [in] client client to which we will send this adding action
1192   */
1193   void CFile::sendAddVariableGroup(const string& id, CContextClient* client)
1194   TRY
1195   {
1196      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
1197   }
1198   CATCH_DUMP_ATTR
1199
1200   /*
1201     Send message to add a variable into a file within a certain client
1202     \param [in] id String identity of a variable
1203     \param [in] client client to which we will send this adding action
1204   */
1205   void CFile::sendAddVariable(const string& id, CContextClient* client)
1206   TRY
1207   {
1208      sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
1209   }
1210   CATCH_DUMP_ATTR
1211
1212   /*!
1213   \brief Receive a message annoucing the creation of a variable on server side
1214   \param[in] event Received event
1215   */
1216   void CFile::recvAddVariable(CEventServer& event)
1217   TRY
1218   {
1219      CBufferIn* buffer = event.subEvents.begin()->buffer;
1220      string id;
1221      *buffer>>id;
1222      get(id)->recvAddVariable(*buffer);
1223   }
1224   CATCH
1225
1226   /*!
1227   \brief Receive a message annoucing the creation of a variable on server side
1228   \param[in] buffer Buffer containing message
1229   */
1230   void CFile::recvAddVariable(CBufferIn& buffer)
1231   TRY
1232   {
1233      string id;
1234      buffer>>id;
1235      addVariable(id);
1236   }
1237   CATCH_DUMP_ATTR
1238
1239   /*!
1240   \brief Receive a message annoucing the creation of a variable group on server side
1241   \param[in] event Received event
1242   */
1243   void CFile::recvAddVariableGroup(CEventServer& event)
1244   TRY
1245   {
1246
1247      CBufferIn* buffer = event.subEvents.begin()->buffer;
1248      string id;
1249      *buffer>>id;
1250      get(id)->recvAddVariableGroup(*buffer);
1251   }
1252   CATCH
1253
1254   /*!
1255   \brief Receive a message annoucing the creation of a variable group on server side
1256   \param[in] buffer Buffer containing message
1257   */
1258   void CFile::recvAddVariableGroup(CBufferIn& buffer)
1259   TRY
1260   {
1261      string id;
1262      buffer>>id;
1263      addVariableGroup(id);
1264   }
1265   CATCH_DUMP_ATTR
1266
1267   /*!
1268     \brief Sending all active (enabled) fields from client to server.
1269   Each field is identified uniquely by its string identity. Not only should we
1270   send the id to server but also we need to send ids of reference domain and reference axis.
1271   With these two id, it's easier to make reference to grid where all data should be written.
1272   Remark: This function must be called AFTER all active (enabled) files have been created on the server side
1273   */
1274   void CFile::sendEnabledFields(CContextClient* client)
1275   TRY
1276   {
1277     size_t size = this->enabledFields.size();
1278     for (size_t i = 0; i < size; ++i)
1279     {
1280       CField* field = this->enabledFields[i];
1281       this->sendAddField(field->getId(), client);
1282       field->checkTimeAttributes();
1283       field->sendAllAttributesToServer(client);
1284       field->sendAddAllVariables(client);
1285     }
1286   }
1287   CATCH_DUMP_ATTR
1288
1289
1290 
1291   /*!
1292    * Send file attribute, related variable and chield field tree to a given file server.
1293    * \param[in] client : the context client where to send file
1294    */
1295   void CFile::sendFileToFileServer(CContextClient* client)
1296   TRY
1297   {
1298     if (sendFileToFileServer_done_.count(client)!=0) return ;
1299     else sendFileToFileServer_done_.insert(client) ;
1300     
1301     StdString fileDefRoot("file_definition");
1302     CFileGroup* cfgrpPtr = CFileGroup::get(fileDefRoot);
1303     cfgrpPtr->sendCreateChild(this->getId(), client);
1304     this->sendAllAttributesToServer(client);
1305     this->sendAddAllVariables(client);
1306     for(auto field : enabledFields) this->sendAddField(field->getId(), client);
1307   }
1308   CATCH_DUMP_ATTR
1309   /*!
1310   \brief Dispatch event received from client
1311      Whenever a message is received in buffer of server, it will be processed depending on
1312   its event type. A new event type should be added in the switch list to make sure
1313   it processed on server side.
1314   \param [in] event: Received message
1315   */
1316   bool CFile::dispatchEvent(CEventServer& event)
1317   TRY
1318   {
1319      if (SuperClass::dispatchEvent(event)) return true;
1320      else
1321      {
1322        switch(event.type)
1323        {
1324           case EVENT_ID_ADD_FIELD :
1325             recvAddField(event);
1326             return true;
1327             break;
1328
1329           case EVENT_ID_ADD_FIELD_GROUP :
1330             recvAddFieldGroup(event);
1331             return true;
1332             break;
1333
1334            case EVENT_ID_ADD_VARIABLE :
1335             recvAddVariable(event);
1336             return true;
1337             break;
1338
1339           case EVENT_ID_ADD_VARIABLE_GROUP :
1340             recvAddVariableGroup(event);
1341             return true;
1342             break;
1343           default :
1344              ERROR("bool CFile::dispatchEvent(CEventServer& event)", << "Unknown Event");
1345           return false;
1346        }
1347      }
1348   }
1349   CATCH
1350
1351   ///--------------------------------------------------------------
1352   /*!
1353   */
1354   StdString CFile::dumpClassAttributes(void)
1355   {
1356     StdString str;
1357     CContext* context = CContext::getCurrent();
1358     str.append("context=\"");
1359     str.append(context->getId());
1360     str.append("\"");
1361     str.append(" enabled fields=\"");
1362     int size = this->enabledFields.size();
1363     for (int i = 0; i < size; ++i)
1364     {
1365       str.append(this->enabledFields[i]->getId());
1366       str.append(" ");
1367     }
1368     str.append("\"");
1369     return str;
1370   }
1371
1372   ///---------------------------------------------------------------
1373
1374} // namespace xios
Note: See TracBrowser for help on using the repository browser.