source: XIOS/dev/dev_ym/XIOS_COUPLING/src/node/field.cpp @ 1930

Last change on this file since 1930 was 1930, checked in by ymipsl, 4 years ago

Big update on on going work related to data distribution and transfer between clients and servers.
Revisite of the source and store filter using "connectors".

YM
  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
  • Property svn:executable set to *
File size: 79.4 KB
Line 
1#include "field.hpp"
2
3#include "attribute_template.hpp"
4#include "object_template.hpp"
5#include "group_template.hpp"
6
7#include "node_type.hpp"
8#include "calendar_util.hpp"
9#include "message.hpp"
10#include "xios_spl.hpp"
11#include "type.hpp"
12#include "timer.hpp"
13#include "context_client.hpp"
14#include "context_server.hpp"
15#include <set>
16#include "garbage_collector.hpp"
17#include "source_filter.hpp"
18#include "store_filter.hpp"
19#include "file_writer_filter.hpp"
20#include "pass_through_filter.hpp"
21#include "filter_expr_node.hpp"
22#include "lex_parser.hpp"
23#include "temporal_filter.hpp"
24#include "spatial_transform_filter.hpp"
25#include "file_server_writer_filter.hpp"
26#include "file_server_reader_filter.hpp"
27#include "server_to_client_filter.hpp"
28#include "server_from_client_source_filter.hpp"
29#include "file_reader_source_filter.hpp"
30#include "tracer.hpp"
31
32namespace xios
33{
34
35  /// ////////////////////// Définitions ////////////////////// ///
36
37  CField::CField(void)
38    : CObjectTemplate<CField>(), CFieldAttributes()
39    , written(false)
40    , nstep(0), nstepMax(0)
41    , hasOutputFile(false)
42    , domAxisScalarIds_(vector<StdString>(3,""))
43    , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
44    , isGridChecked(false)
45    , useCompressedOutput(false)
46    , hasTimeInstant(false)
47    , hasTimeCentered(false)
48    , wasDataRequestedFromServer(false)
49    , wasDataAlreadyReceivedFromServer(false)
50    , mustAutoTrigger(false)
51    , isEOF(false), nstepMaxRead(false)
52  { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
53
54  CField::CField(const StdString& id)
55    : CObjectTemplate<CField>(id), CFieldAttributes()
56    , written(false)
57    , nstep(0), nstepMax(0)
58    , hasOutputFile(false)
59    , domAxisScalarIds_(vector<StdString>(3,""))
60    , areAllReferenceSolved(false), isReferenceSolved(false), isReferenceSolvedAndTransformed(false)
61    , isGridChecked(false)
62    , useCompressedOutput(false)
63    , hasTimeInstant(false)
64    , hasTimeCentered(false)
65    , wasDataRequestedFromServer(false)
66    , wasDataAlreadyReceivedFromServer(false)
67    , mustAutoTrigger(false)
68    , isEOF(false), nstepMaxRead(false)
69  { setVirtualVariableGroup(CVariableGroup::create(getId() + "_virtual_variable_group")); }
70
71  CField::~CField(void)
72  {}
73
74  //----------------------------------------------------------------
75
76  void CField::setVirtualVariableGroup(CVariableGroup* newVVariableGroup)
77  TRY
78  {
79    this->vVariableGroup = newVVariableGroup;
80  }
81  CATCH
82
83  CVariableGroup* CField::getVirtualVariableGroup(void) const
84  TRY
85  {
86     return this->vVariableGroup;
87  }
88  CATCH
89
90  std::vector<CVariable*> CField::getAllVariables(void) const
91  TRY
92  {
93    return this->vVariableGroup->getAllChildren();
94  }
95  CATCH
96
97  void CField::solveDescInheritance(bool apply, const CAttributeMap* const parent)
98  TRY
99  {
100    SuperClassAttribute::setAttributes(parent, apply);
101    this->getVirtualVariableGroup()->solveDescInheritance(apply, NULL);
102  }
103  CATCH_DUMP_ATTR
104
105  //----------------------------------------------------------------
106
107  bool CField::dispatchEvent(CEventServer& event)
108  TRY
109  {
110    if (SuperClass::dispatchEvent(event)) return true;
111    else
112    {
113      switch(event.type)
114      {
115        case EVENT_ID_UPDATE_DATA :
116          recvUpdateData(event);
117          return true;
118          break;
119
120        case EVENT_ID_READ_DATA :
121          recvReadDataRequest(event);
122          return true;
123          break;
124
125        case EVENT_ID_READ_DATA_READY :
126          recvReadDataReady(event);
127          return true;
128          break;
129
130        case EVENT_ID_ADD_VARIABLE :
131          recvAddVariable(event);
132          return true;
133          break;
134
135        case EVENT_ID_ADD_VARIABLE_GROUP :
136          recvAddVariableGroup(event);
137          return true;
138          break;
139     
140        case EVENT_ID_GRID_COMPLETED :
141          recvGridCompleted(event);
142          return true;
143          break;
144        default :
145          ERROR("bool CField::dispatchEvent(CEventServer& event)", << "Unknown Event");
146          return false;
147      }
148    }
149  }
150  CATCH
151
152/* obsolete old interface
153  void CField::sendUpdateData(Time timeStamp, const CArray<double,1>& data, CContextClient* client)
154  TRY
155  {
156    CTimer::get("Field : send data").resume();
157    int receiverSize = client->serverSize;
158
159    CEventClient event(getType(), EVENT_ID_UPDATE_DATA);
160
161    map<int, CArray<int,1> >::iterator it;
162    list<CMessage> list_msg;
163    list<CArray<double,1> > list_data;
164
165    if (!grid_->doGridHaveDataDistributed(client))
166    {
167       if (client->isServerLeader())
168       {
169          for (it = grid_->storeIndex_toSrv_[client].begin(); it != grid_->storeIndex_toSrv_[client].end(); it++)
170          {
171            int rank = it->first;
172            CArray<int,1>& index = it->second;
173
174            list_msg.push_back(CMessage());
175            list_data.push_back(CArray<double,1>(index.numElements()));
176
177            CArray<double,1>& data_tmp = list_data.back();
178            for (int n = 0; n < data_tmp.numElements(); n++) data_tmp(n) = data(index(n));
179
180            list_msg.back() << getId() << timeStamp << data_tmp;
181            event.push(rank, 1, list_msg.back());
182          }
183          client->sendEvent(event);
184        }
185      else client->sendEvent(event);
186    }
187    else
188    {
189      for (it = grid_->storeIndex_toSrv_[client].begin(); it != grid_->storeIndex_toSrv_[client].end(); it++)
190      {
191        int rank = it->first;
192        CArray<int,1>& index = it->second;
193
194        list_msg.push_back(CMessage());
195        list_data.push_back(CArray<double,1>(index.numElements()));
196
197        CArray<double,1>& data_tmp = list_data.back();
198        for (int n = 0; n < data_tmp.numElements(); n++) data_tmp(n) = data(index(n));
199
200        list_msg.back() << getId() << timeStamp << data_tmp;
201        event.push(rank, grid_->nbSenders_[receiverSize][rank], list_msg.back());
202      }
203      client->sendEvent(event);
204    }
205
206    CTimer::get("Field : send data").suspend();
207  }
208  CATCH_DUMP_ATTR
209*/
210
211  void CField::sendUpdateData(Time timeStamp, const CArray<double,1>& data, CContextClient* client)
212  TRY
213  {
214    CTimer::get("Field : send data").resume();
215    CEventClient event(getType(), EVENT_ID_UPDATE_DATA);
216    CMessage message ;
217
218    message<<getId() << timeStamp ;
219    this->getGrid()->getClientToServerConnector(client)->transfer(data, client, event, message) ;
220    CTimer::get("Field : send data").suspend();
221  }
222  CATCH_DUMP_ATTR
223
224  /* old version obsolete
225  void CField::recvUpdateData(CEventServer& event)
226  TRY
227  {
228    std::map<int,CBufferIn*> rankBuffers;
229
230    list<CEventServer::SSubEvent>::iterator it;
231    string fieldId;
232    CTimer::get("Field : recv data").resume();
233    for (it = event.subEvents.begin(); it != event.subEvents.end(); ++it)
234    {
235      int rank = it->rank;
236      CBufferIn* buffer = it->buffer;
237      *buffer >> fieldId;
238      rankBuffers[rank] = buffer;
239    }
240    get(fieldId)->recvUpdateData(rankBuffers);
241    CTimer::get("Field : recv data").suspend();
242  }
243  CATCH
244*/
245
246  void CField::recvUpdateData(CEventServer& event)
247  TRY
248  {
249    string fieldId;
250    for (auto& subEvent : event.subEvents) (*subEvent.buffer) >> fieldId  ;
251    get(fieldId)->receiveUpdateData(event);
252  }
253  CATCH
254
255  void  CField::receiveUpdateData(CEventServer& event)
256  TRY
257  {
258    if (hasCouplerIn()) clientFromClientSourceFilter_->streamData(event) ;
259    else serverFromClientSourceFilter_->streamData(event) ;
260  }
261  CATCH
262/*
263  void  CField::recvUpdateDataFromClient(CEventServer& event)
264  TRY
265  {
266    Time timeStamp ;
267    for (auto& subEvent : event.subEvents) (*subEvent.buffer) >> timeStamp  ;
268
269    CArray<double,1> recvData ;
270    getGrid()->getServerFromClientConnector()->transfer(event,recvData) ;
271    this->setData(recvData);
272  }
273  CATCH
274*/
275
276/* 
277  void CField::recvUpdateDataFromCoupler(CEventServer& event)
278  TRY
279  {
280    CContext* context = CContext::getCurrent();
281    Time timeStamp ;
282    if (wasDataAlreadyReceivedFromServer)
283    { 
284      lastDataReceivedFromServer = lastDataReceivedFromServer + freq_op;
285    }
286    else
287    {
288      // unlikely to input from file server where data are received at ts=0
289      // for coupling, it would be after the first freq_op, because for now we don't have
290      // restart mecanism to send the value at ts=0. It must be changed in future
291      lastDataReceivedFromServer = context->getCalendar()->getInitDate();
292      wasDataAlreadyReceivedFromServer = true;
293    }
294
295    CArray<double,1> recvData ;
296    getGrid()->getServerFromClientConnector()->transfer(event,recvData) ;
297    clientSourceFilter->streamData(lastDataReceivedFromServer, recvData);
298
299  }
300  CATCH_DUMP_ATTR
301*/
302
303
304
305
306  /* old interface to be removed.... */
307  void  CField::recvUpdateDataFromClient(std::map<int,CBufferIn*>& rankBuffers)
308  TRY
309  {
310    // ym to remove latter
311    /*
312    CContext* context = CContext::getCurrent();
313    Time timeStamp ;
314    size_t sizeData = 0;
315    if (0 == recvDataSrv.numElements())
316    {           
317      CArray<int,1>& storeClient = grid_->getStoreIndex_client();
318
319      // Gather all data from different clients     
320      recvDataSrv.resize(storeClient.numElements());
321      recvFoperationSrv = std::shared_ptr<func::CFunctor>(new func::CInstant(recvDataSrv));
322    }
323
324    CArray<double,1> recv_data_tmp(recvDataSrv.numElements());   
325    const CDate& currDate = context->getCalendar()->getCurrentDate();
326    CDuration offsetAllButMonth (freq_offset.getValue().year, 0 , freq_offset.getValue().day,
327                                 freq_offset.getValue().hour, freq_offset.getValue().minute,
328                                 freq_offset.getValue().second, freq_offset.getValue().timestep);
329    const CDate opeDate   = (last_operation_srv - offsetAllButMonth + context->getCalendar()->getTimeStep())
330                             + freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep() + offsetAllButMonth;
331
332    if (opeDate <= currDate)
333    {
334     
335      auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
336      for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
337      {
338        CArray<double,1> tmp;
339        CArray<size_t,1>& indexTmp = it->second;
340        *(rankBuffers[it->first]) >> timeStamp >> tmp;
341        for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
342      }
343    }
344*/
345    Time timeStamp ;
346    CArray<int,1>& storeClient = grid_->getStoreIndex_client(); // replace it with local size
347    CArray<double,1> recv_data_tmp(storeClient.numElements());
348    auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
349    for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
350    {
351      CArray<double,1> tmp;
352      CArray<size_t,1>& indexTmp = it->second;
353      *(rankBuffers[it->first]) >> timeStamp >> tmp;
354      for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
355    }
356
357    this->setData(recv_data_tmp);
358/*
359    // delete incomming flux for server only
360    recvFoperationSrv.reset() ;
361    recvDataSrv.reset() ;
362*/
363  }
364  CATCH_DUMP_ATTR
365
366  /* ym : old interface : to be removed...
367  void CField::writeUpdateData(const CArray<double,1>& data)
368  TRY
369  {
370    CContext* context = CContext::getCurrent();
371
372    const CDate& currDate = context->getCalendar()->getCurrentDate();
373    CDuration offsetAllButMonth (freq_offset.getValue().year, 0 , freq_offset.getValue().day,
374                                   freq_offset.getValue().hour, freq_offset.getValue().minute,
375                                   freq_offset.getValue().second, freq_offset.getValue().timestep);
376    const CDate opeDate   = (last_operation_srv - offsetAllButMonth + context->getCalendar()->getTimeStep())
377                              + freq_op + freq_operation_srv - freq_op - context->getCalendar()->getTimeStep() + offsetAllButMonth;
378    const CDate writeDate = last_Write_srv + freq_write_srv;
379
380    if (opeDate <= currDate)
381    {
382      (*recvFoperationSrv)(data);
383      last_operation_srv = currDate;
384    }
385
386    if (writeDate < (currDate + freq_operation_srv))
387    {
388      recvFoperationSrv->final();
389      last_Write_srv = writeDate;
390      grid_->computeWrittenIndex();
391      writeField();
392      lastlast_Write_srv = last_Write_srv;
393    }
394  }
395  CATCH_DUMP_ATTR
396  */
397
398  void CField::writeUpdateData(const CArray<double,1>& data)
399  TRY
400  {
401    const CDate writeDate = last_Write_srv + freq_write_srv;
402    last_Write_srv = writeDate;
403    // grid_->computeWrittenIndex(); -> obselete function need to be removed
404    /*
405    recvDataSrv.resize(data.numElements()) ;
406    recvDataSrv = data ;
407    */
408    writeField(data);
409    lastlast_Write_srv = last_Write_srv;
410  }
411  CATCH_DUMP_ATTR
412
413
414  void CField::writeField(const CArray<double,1>& data)
415  TRY
416  {
417    if (!getRelFile()->isEmptyZone())
418    {
419      if (grid_->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)
420      {
421        getRelFile()->checkWriteFile();
422        this->incrementNStep();
423        getRelFile()->getDataOutput()->writeFieldData(CField::get(this), data);
424      }
425    }
426  }
427  CATCH_DUMP_ATTR
428
429  /*
430    Send a request for reading data.
431    Client sends a request to server for demanding server to read data and send back to it.
432    For now, this function is called only by client
433    In the future, it can be called by level-1 servers
434    \param [in] tsDataRequested timestamp when the call is made
435  */
436  bool CField::sendReadDataRequest(const CDate& tsDataRequested, CContextClient* client)
437  TRY
438  {
439    CContext* context = CContext::getCurrent();
440
441    lastDataRequestedFromServer = tsDataRequested;
442
443    // No need to send the request if we are sure that we are already at EOF
444    if (!isEOF || context->getCalendar()->getCurrentDate() <= dateEOF)
445    {
446      CEventClient event(getType(), EVENT_ID_READ_DATA);
447      if (client->isServerLeader())
448      {
449        CMessage msg;
450        msg << getId();
451        const std::list<int>& ranks = client->getRanksServerLeader();
452        for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
453          event.push(*itRank, 1, msg);
454        client->sendEvent(event);
455      }
456      else client->sendEvent(event);
457    }
458    else serverSourceFilter->signalEndOfStream(tsDataRequested);
459
460    wasDataRequestedFromServer = true;
461
462    return !isEOF;
463  }
464  CATCH_DUMP_ATTR
465
466  /*!
467  Send request new data read from file if need be, that is the current data is out-of-date.
468  \return true if and only if some data was requested
469  */
470  bool CField::sendReadDataRequestIfNeeded(void)
471  TRY
472  {
473    const CDate& currentDate = CContext::getCurrent()->getCalendar()->getCurrentDate();
474
475    bool dataRequested = false;
476
477    while (currentDate >= lastDataRequestedFromServer)
478    {
479      info(20) << "currentDate : " << currentDate << endl ;
480      info(20) << "lastDataRequestedFromServer : " << lastDataRequestedFromServer << endl ;
481      info(20) << "fileIn_->output_freq.getValue() : " << fileIn_->output_freq.getValue() << endl ;
482      info(20) << "lastDataRequestedFromServer + fileIn_->output_freq.getValue() : " << lastDataRequestedFromServer + fileIn_->output_freq << endl ;
483
484      dataRequested |= sendReadDataRequest(lastDataRequestedFromServer + fileIn_->output_freq, fileIn_->getContextClient());
485    }
486
487    return dataRequested;
488  }
489  CATCH_DUMP_ATTR
490
491  void CField::recvReadDataRequest(CEventServer& event)
492  TRY
493  {
494    CBufferIn* buffer = event.subEvents.begin()->buffer;
495    StdString fieldId;
496    *buffer >> fieldId;
497    get(fieldId)->recvReadDataRequest();
498  }
499  CATCH
500 
501  /*!
502    Receive data request sent from client and process it
503    Every time server receives this request, it will try to read data and sent read data back to client
504    At the moment, this function is called by server level 1
505    In the future, this should (only) be done by the last level servers.
506  */
507  void CField::recvReadDataRequest(void)
508  TRY
509  {
510    fileReaderSourceFilter_->streamData() ;
511  }
512  CATCH_DUMP_ATTR 
513
514/* old interface -> to remove
515  void CField::recvReadDataRequest(CContextServer* server)
516  TRY
517  {
518       
519    CArray<double,1> data ;
520    EReadField hasData = readField(data);
521    CDate date = CContext::getCurrent()->getCalendar()->getCurrentDate();
522    if (hasData == RF_DATA) fileServerReaderFilter_->streamData(date,data) ;
523    else fileServerReaderFilter_->signalEndOfStream(date) ;
524
525  }
526  CATCH_DUMP_ATTR
527*/
528
529  void CField::sendUpdateDataServerToClient(bool isEOF, const CArray<double,1>& data, CContextClient* client)
530  TRY
531  {
532    CEventClient event(getType(), EVENT_ID_READ_DATA_READY);
533    std::list<CMessage> msgs;
534    if (!grid_->doGridHaveDataDistributed(client))
535    {
536       if (client->isServerLeader())
537       {
538          if (0 != data.numElements())
539          {           
540            const std::list<int>& ranks = client->getRanksServerLeader();
541            for (std::list<int>::const_iterator itRank = ranks.begin(), itRankEnd = ranks.end(); itRank != itRankEnd; ++itRank)
542            {
543              msgs.push_back(CMessage());
544              CMessage& msg = msgs.back();
545              msg << getId();
546             
547              if (isEOF)  msg << int(-1);
548              else msg << getNStep() - 1 << data;
549         
550              event.push(*itRank, 1, msg);
551            }
552          }
553          client->sendEvent(event);
554       }
555       else
556       {
557          client->sendEvent(event);
558       }
559    }
560    else
561    {
562      auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
563      for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
564      {
565       
566
567        msgs.push_back(CMessage());
568        CMessage& msg = msgs.back();
569        msg << getId();
570        if (isEOF) msg << int(-1);
571        else 
572        {
573          CArray<size_t,1>& indexTmp = it->second;
574          CArray<double,1> tmp(indexTmp.numElements());
575          for (int idx = 0; idx < indexTmp.numElements(); ++idx) tmp(idx) = data(indexTmp(idx));
576          msg << getNStep() - 1 << tmp;
577        }
578        event.push(it->first, grid_->getNbReadSenders(client)[it->first], msg);
579      }
580      client->sendEvent(event);
581    }
582  }
583  CATCH_DUMP_ATTR
584
585  /*!
586    Read field from a file.
587    A field is read with the distribution of data on the server side
588    \return State of field can be read from a file
589  */
590  // obsolete to remove
591  /*
592  CField::EReadField CField::readField(CArray<double,1>& data)
593  TRY
594  {
595    CContext* context = CContext::getCurrent();
596    grid_->computeWrittenIndex();
597    getRelFile()->initRead();
598    EReadField readState = RF_DATA;
599
600    if (!getRelFile()->isEmptyZone())
601    {     
602      if (grid_->doGridHaveDataToWrite() || getRelFile()->type == CFile::type_attr::one_file)     
603      {
604          CArray<int,1>& storeClient = grid_->getStoreIndex_client();         
605          data.resize(storeClient.numElements());         
606       
607        getRelFile()->checkReadFile();
608
609        if (!nstepMax)
610        {
611          nstepMax = getRelFile()->getDataInput()->getFieldNbRecords(CField::get(this));
612        }
613
614        this->incrementNStep();
615
616        if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
617          readState = RF_EOF;
618
619        if (RF_EOF != readState)
620          getRelFile()->getDataInput()->readFieldData(CField::get(this),data);
621      }
622    }
623    else
624    {
625      this->incrementNStep();
626      if (getNStep() > nstepMax && (getRelFile()->cyclic.isEmpty() || !getRelFile()->cyclic) )
627        readState = RF_EOF;
628      else
629        readState = RF_NODATA;
630
631      if (!nstepMaxRead) // This can be a bug if we try to read field from zero time record
632        readState = RF_NODATA;
633    }
634
635    if (!nstepMaxRead)
636    {
637       MPI_Allreduce(MPI_IN_PLACE, &nstepMax, 1, MPI_INT, MPI_MAX, context->intraComm_);
638       nstepMaxRead = true;
639    }
640
641    return readState;
642  }
643  CATCH_DUMP_ATTR
644  */
645
646  /*
647    Receive read data from server.
648    At the moment, this function is called in the client side.
649    In the future, this function can be called hiearachically (server n-1, server n -2, ..., client)
650    \param event event containing read data
651  */
652  void CField::recvReadDataReady(CEventServer& event)
653  TRY
654  {
655    string fieldId;
656    for (auto& subEvent : event.subEvents) (*subEvent.buffer) >> fieldId  ;
657    get(fieldId)->recvReadDataReady(event);
658  }
659  CATCH
660
661
662  /* old interface to be removed ..*/
663  void CField::recvUpdateDataFromCoupler(std::map<int,CBufferIn*>& rankBuffers)
664  TRY
665  {
666    CContext* context = CContext::getCurrent();
667    Time timeStamp ;
668    if (wasDataAlreadyReceivedFromServer)
669    { 
670      lastDataReceivedFromServer = lastDataReceivedFromServer + freq_op;
671    }
672    else
673    {
674      // unlikely to input from file server where data are received at ts=0
675      // for coupling, it would be after the first freq_op, because for now we don't have
676      // restart mecanism to send the value at ts=0. It mus be changed in future
677      lastDataReceivedFromServer = context->getCalendar()->getInitDate();
678      wasDataAlreadyReceivedFromServer = true;
679    }
680
681    CArray<int,1>& storeClient = grid_->getStoreIndex_client();
682    CArray<double,1> recv_data_tmp(storeClient.numElements()); 
683
684    auto& outLocalIndexStoreOnClient = grid_-> getOutLocalIndexStoreOnClient() ;
685    for (auto it = outLocalIndexStoreOnClient.begin(); it != outLocalIndexStoreOnClient.end(); ++it)
686    {
687      CArray<double,1> tmp;
688      CArray<size_t,1>& indexTmp = it->second;
689      *(rankBuffers[it->first]) >> timeStamp >> tmp;
690      for (int idx = 0; idx < indexTmp.numElements(); ++idx) recv_data_tmp(indexTmp(idx)) = tmp(idx);
691    }
692   
693    clientSourceFilter->streamData(lastDataReceivedFromServer, recv_data_tmp);
694   
695  }
696  CATCH_DUMP_ATTR
697 
698
699  /*!
700    Receive read data from server
701    \param [in] ranks Ranks of sending processes
702    \param [in] buffers buffers containing read data
703  */
704  // old interface to remove
705  void CField::recvReadDataReady(vector<int> ranks, vector<CBufferIn*> buffers)
706  TRY
707  {
708    CContext* context = CContext::getCurrent();
709    std::map<int, CArray<double,1> > data;
710    const bool wasEOF = isEOF;
711
712    for (int i = 0; i < ranks.size(); i++)
713    {
714      int rank = ranks[i];
715      int record;
716      *buffers[i] >> record;
717      isEOF = (record == int(-1));
718
719      if (!isEOF)
720        *buffers[i] >> data[rank];
721      else
722        break;
723    }
724
725    if (wasDataAlreadyReceivedFromServer)
726      lastDataReceivedFromServer = lastDataReceivedFromServer + fileIn_->output_freq;
727    else
728    {
729      lastDataReceivedFromServer = context->getCalendar()->getInitDate();
730      wasDataAlreadyReceivedFromServer = true;
731    }
732
733    if (isEOF)
734    {
735      if (!wasEOF)
736        dateEOF = lastDataReceivedFromServer;
737
738      serverSourceFilter->signalEndOfStream(lastDataReceivedFromServer);
739    }
740    else
741      serverSourceFilter->streamDataFromServer(lastDataReceivedFromServer, data);
742  }
743  CATCH_DUMP_ATTR
744
745
746
747  void CField::receiveReadDataReady(CEventServer& event)
748  TRY
749  {
750    clientFromServerSourceFilter_->streamData(event) ;   
751  }
752  CATCH_DUMP_ATTR
753
754
755
756  void CField::checkForLateDataFromCoupler(void)
757  TRY
758  {
759    CContext* context = CContext::getCurrent();
760    const CDate& currentDate = context->getCalendar()->getCurrentDate();
761
762    CTimer timer("CField::checkForLateDataFromCoupler");
763    timer.resume();
764    traceOff() ;
765    timer.suspend();
766   
767    bool isDataLate; 
768    do
769    {
770      isDataLate=clientFromClientSourceFilter_->isDataLate() ;
771      if (isDataLate)
772      {
773        timer.resume();
774        context->globalEventLoop();
775        timer.suspend();
776      }
777    } while (isDataLate && timer.getCumulatedTime() < CXios::recvFieldTimeout);
778   
779    timer.resume();
780    traceOn() ;
781    timer.suspend() ;
782
783    if (isDataLate) ERROR("void CField::checkForLateDataFromCoupler(void)",
784                            << "Late data at timestep = " << currentDate);
785  }
786  CATCH_DUMP_ATTR
787
788
789  void CField::checkForLateDataFromServer(void)
790  TRY
791  {
792    CContext* context = CContext::getCurrent();
793    // Check if data previously requested has been received as expected
794    if (wasDataRequestedFromServer && !clientFromServerSourceFilter_->isEOF())
795    {
796      CTimer timer("CField::checkForLateDataFromServer");
797      timer.resume();
798      traceOff() ;
799      timer.suspend();
800     
801      bool isDataLate;
802      do
803      {
804        isDataLate=clientFromServerSourceFilter_->isDataLate();
805        if (isDataLate)
806        {
807          timer.resume();
808
809//ym          context->checkBuffersAndListen();
810//ym            context->eventLoop();
811          context->globalEventLoop();
812
813          timer.suspend();
814        }
815      }
816      while (isDataLate && timer.getCumulatedTime() < CXios::recvFieldTimeout);
817      timer.resume();
818      traceOn() ;
819      timer.suspend() ;
820
821
822      if (isDataLate)
823        ERROR("void CField::checkForLateDataFromServer(void)",
824              << "Late data at timestep = " << context->getCalendar()->getCurrentDate());
825    }
826  }
827  CATCH_DUMP_ATTR
828
829  void CField::triggerLateField(void)
830  TRY
831  {
832    if (hasFileIn()) 
833    {
834      checkForLateDataFromServer() ;
835      clientFromServerSourceFilter_->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate()) ;
836    } 
837    else if (hasCouplerIn())
838    {
839      checkForLateDataFromCoupler() ;
840      clientFromClientSourceFilter_->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate()) ;
841    }
842  }
843  CATCH_DUMP_ATTR
844
845
846  void CField::checkIfMustAutoTrigger(void)
847  TRY
848  {
849    mustAutoTrigger = clientFromServerSourceFilter_ ? clientFromServerSourceFilter_->mustAutoTrigger() : false;
850  }
851  CATCH_DUMP_ATTR
852
853  void CField::autoTriggerIfNeeded(void)
854  TRY
855  {
856    if (mustAutoTrigger)
857      clientFromServerSourceFilter_->trigger(CContext::getCurrent()->getCalendar()->getCurrentDate());
858  }
859  CATCH_DUMP_ATTR
860
861  //----------------------------------------------------------------
862/*
863  void CField::setRelFile(CFile* _file)
864  TRY
865  {
866    this->file = _file;
867    hasOutputFile = true;
868  }
869  CATCH_DUMP_ATTR
870*/
871  //----------------------------------------------------------------
872
873  StdString CField::GetName(void)    { return StdString("field"); }
874  StdString CField::GetDefName(void) { return CField::GetName(); }
875  ENodeType CField::GetType(void)    { return eField; }
876
877  //----------------------------------------------------------------
878
879  CGrid* CField::getRelGrid(void) const
880  TRY
881  {
882    return this->grid_;
883  }
884  CATCH
885
886  //----------------------------------------------------------------
887
888  CFile* CField::getRelFile(void) const
889  TRY
890  {
891    if (hasFileIn()) return this->fileIn_;
892    else if (hasFileOut()) return this->fileOut_ ;
893    else return nullptr ;
894  }
895  CATCH
896
897  int CField::getNStep(void) const
898  TRY
899  {
900    return this->nstep;
901  }
902  CATCH
903
904  func::CFunctor::ETimeType CField::getOperationTimeType() const
905  TRY
906  {
907    return operationTimeType;
908  }
909  CATCH
910
911   //----------------------------------------------------------------
912
913  void CField::incrementNStep(void)
914  TRY
915  {
916    this->nstep++;
917  }
918  CATCH_DUMP_ATTR
919
920  void CField::resetNStep(int nstep /*= 0*/)
921  TRY
922  {
923    this->nstep = nstep;
924  }
925  CATCH_DUMP_ATTR
926
927  void CField::resetNStepMax(void)
928  TRY
929  {
930    this->nstepMax = 0;
931    nstepMaxRead = false;
932  }
933  CATCH_DUMP_ATTR
934
935  //----------------------------------------------------------------
936
937  bool CField::isActive(bool atCurrentTimestep /*= false*/) const
938  TRY
939  {
940    if (modelToClientSourceFilter_) 
941      return atCurrentTimestep ? modelToClientSourceFilter_->isDataExpected(CContext::getCurrent()->getCalendar()->getCurrentDate()) : true;
942    else if (clientToModelStoreFilter_)  return true;
943    else if (instantDataFilter)
944      ERROR("bool CField::isActive(bool atCurrentTimestep)",
945            << "Impossible to check if field [ id = " << getId() << " ] is active as it cannot be used to receive nor send data.");
946
947    return false;
948  }
949  CATCH
950
951  //----------------------------------------------------------------
952
953  bool CField::wasWritten() const
954  TRY
955  {
956    return written;
957  }
958  CATCH
959
960  void CField::setWritten()
961  TRY
962  {
963    written = true;
964  }
965  CATCH_DUMP_ATTR
966
967  //----------------------------------------------------------------
968
969  bool CField::getUseCompressedOutput() const
970  TRY
971  {
972    return useCompressedOutput;
973  }
974  CATCH
975
976  void CField::setUseCompressedOutput()
977  TRY
978  {
979    useCompressedOutput = true;
980  }
981  CATCH_DUMP_ATTR
982
983  //----------------------------------------------------------------
984
985  std::shared_ptr<COutputPin> CField::getInstantDataFilter()
986  TRY
987  {
988    return instantDataFilter;
989  }
990  CATCH_DUMP_ATTR
991
992  //----------------------------------------------------------------
993
994  /*!
995    Build up graph of grids which plays role of destination and source in grid transformation
996    This function should be called before \func solveGridReference()
997  */
998  void CField::buildGridTransformationGraph()
999  TRY
1000  {
1001    CContext* context = CContext::getCurrent();
1002    if (context->getServiceType()==CServicesManager::CLIENT)
1003    {
1004      if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
1005      {
1006        grid_->addTransGridSource(getDirectFieldReference()->grid_);
1007      }
1008    }
1009  }
1010  CATCH_DUMP_ATTR
1011
1012  /*!
1013    Generate a new grid destination if there are more than one grid source pointing to a same grid destination
1014  */
1015  void CField::generateNewTransformationGridDest()
1016  TRY
1017  {
1018    CContext* context = CContext::getCurrent();
1019    if (context->getServiceType()==CServicesManager::CLIENT)
1020    {
1021      std::map<CGrid*,std::pair<bool,StdString> >& gridSrcMap = grid_->getTransGridSource();
1022      if (1 < gridSrcMap.size())
1023      {
1024        // Search for grid source
1025        CGrid* gridSrc = grid_;
1026        CField* currField = this;
1027        std::vector<CField*> hieraField;
1028        while (currField->hasDirectFieldReference() && (gridSrc == grid_))
1029        {
1030          hieraField.push_back(currField);
1031          CField* tmp = currField->getDirectFieldReference();
1032          currField = tmp;
1033          gridSrc = currField->grid_;
1034        }
1035
1036        if (gridSrcMap.end() != gridSrcMap.find(gridSrc))
1037        {
1038          CGrid* gridTmp;
1039          std::pair<bool,StdString> newGridDest = gridSrcMap[gridSrc];
1040          if (newGridDest.first)
1041          {
1042            StdString newIdGridDest = newGridDest.second;
1043            if (!CGrid::has(newIdGridDest))
1044            {
1045              ERROR("CGrid* CGrid::generateNewTransformationGridDest()",
1046                << " Something wrong happened! Grid whose id " << newIdGridDest
1047                << "should exist ");
1048            }
1049            gridTmp = CGrid::get(newIdGridDest);
1050          }
1051          else
1052          {
1053            StdString newIdGridDest = CGrid::generateId(gridSrc, grid_);
1054            gridTmp = CGrid::cloneGrid(newIdGridDest, grid_);
1055
1056            (gridSrcMap[gridSrc]).first = true;
1057            (gridSrcMap[gridSrc]).second = newIdGridDest;
1058          }
1059
1060          // Update all descendants
1061          for (std::vector<CField*>::iterator it = hieraField.begin(); it != hieraField.end(); ++it)
1062          {
1063            (*it)->grid_ = gridTmp;
1064            (*it)->updateRef((*it)->grid_);
1065          }
1066        }
1067      }
1068    }
1069  }
1070  CATCH_DUMP_ATTR
1071
1072  void CField::updateRef(CGrid* grid)
1073  TRY
1074  {
1075    if (!grid_ref.isEmpty()) grid_ref.setValue(grid->getId());
1076    else
1077    {
1078      std::vector<CAxis*> axisTmp = grid->getAxis();
1079      std::vector<CDomain*> domainTmp = grid->getDomains();
1080      if ((1<axisTmp.size()) || (1<domainTmp.size()))
1081        ERROR("void CField::updateRef(CGrid* grid)",
1082          << "More than one domain or axis is available for domain_ref/axis_ref of field " << this->getId());
1083
1084      if ((!domain_ref.isEmpty()) && (domainTmp.empty()))
1085        ERROR("void CField::updateRef(CGrid* grid)",
1086          << "Incoherent between available domain and domain_ref of field " << this->getId());
1087      if ((!axis_ref.isEmpty()) && (axisTmp.empty()))
1088        ERROR("void CField::updateRef(CGrid* grid)",
1089          << "Incoherent between available axis and axis_ref of field " << this->getId());
1090
1091      if (!domain_ref.isEmpty()) domain_ref.setValue(domainTmp[0]->getId());
1092      if (!axis_ref.isEmpty()) axis_ref.setValue(axisTmp[0]->getId());
1093    }
1094  }
1095  CATCH_DUMP_ATTR
1096   
1097  /*!
1098    Solve reference of all enabled fields even the source fields .
1099    In this step, we do transformations.
1100  */
1101  void CField::solveAllEnabledFieldsAndTransform()
1102  TRY
1103  {
1104    CContext* context = CContext::getCurrent();
1105
1106    if (!isReferenceSolvedAndTransformed)
1107    {
1108      isReferenceSolvedAndTransformed = true;
1109
1110      if (context->getServiceType()==CServicesManager::CLIENT)
1111      {
1112        solveRefInheritance(true);
1113        if (hasDirectFieldReference()) getDirectFieldReference()->solveAllEnabledFieldsAndTransform();
1114      }
1115
1116      if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1117        solveServerOperation();
1118
1119      solveGridReference();
1120
1121      if (context->getServiceType()==CServicesManager::CLIENT)
1122      {
1123        solveGenerateGrid();
1124        buildGridTransformationGraph();
1125      }
1126
1127      solveGridDomainAxisRef(false);
1128
1129      if (context->getServiceType()==CServicesManager::CLIENT)
1130      {
1131        solveTransformedGrid();
1132      }
1133
1134      solveGridDomainAxisRef(false);
1135    }
1136  }
1137  CATCH_DUMP_ATTR
1138
1139  void CField::checkGridOfEnabledFields()
1140  TRY
1141  {
1142    if (!isGridChecked)
1143    {
1144      isGridChecked = true;
1145      solveCheckMaskIndex(false);
1146    }
1147  }
1148  CATCH_DUMP_ATTR
1149
1150  void CField::sendGridComponentOfEnabledFields()
1151  TRY
1152  {
1153    solveGridDomainAxisRef(true);
1154    // solveCheckMaskIndex(true);
1155  }
1156  CATCH_DUMP_ATTR
1157
1158  void CField::sendGridOfEnabledFields()
1159  TRY
1160  {
1161    // solveGridDomainAxisRef(true);
1162    solveCheckMaskIndex(true);
1163  }   
1164  CATCH_DUMP_ATTR
1165
1166  void CField::solveOnlyReferenceEnabledField(void)
1167  TRY
1168  {
1169    CContext* context = CContext::getCurrent();
1170    if (!isReferenceSolved)
1171    {
1172      isReferenceSolved = true;
1173
1174      if (context->getServiceType()==CServicesManager::CLIENT)
1175      {
1176        solveRefInheritance(true);
1177        if (hasDirectFieldReference()) getDirectFieldReference()->solveOnlyReferenceEnabledField();
1178      }
1179
1180      if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1181        solveServerOperation();
1182
1183      solveGridReference();
1184      grid_->solveElementsRefInheritance(true); // make it again to solve grid reading from file
1185
1186      if (context->getServiceType()==CServicesManager::CLIENT)
1187      {
1188        solveGenerateGrid();
1189        buildGridTransformationGraph();
1190      }
1191    }
1192  }
1193  CATCH_DUMP_ATTR
1194
1195  void CField::solveAllReferenceEnabledField(bool doSending2Server)
1196  TRY
1197  {
1198    CContext* context = CContext::getCurrent();
1199    solveOnlyReferenceEnabledField();
1200
1201    if (!areAllReferenceSolved)
1202    {
1203      areAllReferenceSolved = true;
1204     
1205      if (context->getServiceType()==CServicesManager::CLIENT)
1206      {
1207        solveRefInheritance(true);
1208        if (hasDirectFieldReference()) getDirectFieldReference()->solveAllReferenceEnabledField(false);
1209      }
1210      else if (context->getServiceType()==CServicesManager::GATHERER || context->getServiceType()==CServicesManager::OUT_SERVER)
1211        solveServerOperation();
1212
1213      solveGridReference();
1214    }
1215
1216    solveGridDomainAxisRef(doSending2Server);
1217
1218    if (context->getServiceType()==CServicesManager::CLIENT)
1219    {
1220      solveTransformedGrid();
1221    }
1222
1223    solveCheckMaskIndex(doSending2Server);
1224  }
1225  CATCH_DUMP_ATTR
1226
1227  /*!
1228   * Compute the required buffer size to send the fields data.
1229   * \param [in/out] bufferSize Modifying the bufferSize for the client context
1230   * \param [in/out] maxEventSize Modifying the maximum event size for the client context
1231   * \param [in] bufferForWriting True if buffers are used for sending data for writing
1232   */ 
1233  void CField::setContextClientDataBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
1234                                              map<CContextClient*,map<int,size_t>>& maxEventSize, 
1235                                              bool bufferForWriting)
1236  {
1237    auto& contextBufferSize = bufferSize[client] ;
1238    auto& contextMaxEventSize = maxEventSize[client] ;
1239    const std::map<int, size_t> mapSize = grid_->getDataBufferSize(client, getId(), bufferForWriting);
1240    for(auto& it : mapSize )
1241    {
1242      // If contextBufferSize[it.first] does not exist, it will be zero-initialized
1243      // so we can use it safely without checking for its existance
1244      if (CXios::isOptPerformance) contextBufferSize[it.first] += it.second;
1245      else if (contextBufferSize[it.first] < it.second) contextBufferSize[it.first] = it.second;
1246
1247      if (contextMaxEventSize[it.first] < it.second) contextMaxEventSize[it.first] = it.second;
1248    }
1249
1250  }
1251
1252  void CField::setContextClientAttributesBufferSize(map<CContextClient*,map<int,size_t>>& bufferSize, 
1253                                                   map<CContextClient*,map<int,size_t>>& maxEventSize, 
1254                                                   bool bufferForWriting)
1255  {
1256    auto& contextBufferSize = bufferSize[client] ;
1257    auto& contextMaxEventSize = maxEventSize[client] ;
1258    const std::map<int, size_t> mapSize = grid_->getAttributesBufferSize(client, bufferForWriting);
1259    for(auto& it : mapSize )
1260    {
1261      // If contextBufferSize[it.first] does not exist, it will be zero-initialized
1262      // so we can use it safely without checking for its existance
1263      if (contextBufferSize[it.first] < it.second) contextBufferSize[it.first] = it.second;
1264      if (contextMaxEventSize[it.first] < it.second) contextMaxEventSize[it.first] = it.second;
1265    }
1266
1267  }
1268
1269
1270// ym obsolete to be removed
1271  std::map<int, StdSize> CField::getGridAttributesBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
1272  TRY
1273  {
1274    return grid_->getAttributesBufferSize(client, bufferForWriting);
1275  }
1276  CATCH_DUMP_ATTR
1277
1278// ym obsolete to be removed
1279  std::map<int, StdSize> CField::getGridDataBufferSize(CContextClient* client, bool bufferForWriting /*= "false"*/)
1280  TRY
1281  {
1282    return grid_->getDataBufferSize(client, getId(), bufferForWriting);
1283  }
1284  CATCH_DUMP_ATTR
1285
1286
1287
1288  size_t CField::getGlobalWrittenSize()
1289  TRY
1290  {
1291    return grid_->getGlobalWrittenSize();
1292  }
1293  CATCH_DUMP_ATTR
1294
1295  //----------------------------------------------------------------
1296
1297  void CField::solveServerOperation(void)
1298  TRY
1299  {
1300    CContext* context = CContext::getCurrent();
1301
1302    if (freq_op.isEmpty()) freq_op.setValue(TimeStep);
1303
1304    if (freq_offset.isEmpty()) freq_offset.setValue(NoneDu);
1305
1306    freq_operation_srv = fileOut_->output_freq.getValue();
1307    freq_write_srv     = fileOut_->output_freq.getValue();
1308
1309    lastlast_Write_srv = context->getCalendar()->getInitDate();
1310    last_Write_srv     = context->getCalendar()->getInitDate();
1311    last_operation_srv = context->getCalendar()->getInitDate();
1312
1313    const CDuration toffset = freq_operation_srv - freq_offset.getValue() - context->getCalendar()->getTimeStep();
1314    last_operation_srv     = last_operation_srv - toffset;
1315
1316    if (operation.isEmpty())
1317      ERROR("void CField::solveServerOperation(void)",
1318            << "An operation must be defined for field \"" << getId() << "\".");
1319
1320    std::shared_ptr<func::CFunctor> functor;
1321    CArray<double, 1> dummyData;
1322
1323#define DECLARE_FUNCTOR(MType, mtype) \
1324    if (operation.getValue().compare(#mtype) == 0) \
1325    { \
1326      functor.reset(new func::C##MType(dummyData)); \
1327    }
1328
1329#include "functor_type.conf"
1330
1331    if (!functor)
1332      ERROR("void CField::solveServerOperation(void)",
1333            << "\"" << operation << "\" is not a valid operation.");
1334
1335    operationTimeType = functor->timeType();
1336  }
1337  CATCH_DUMP_ATTR
1338
1339 //----------------------------------------------------------------
1340
1341  bool CField::buildWorkflowGraph(CGarbageCollector& gc)
1342  {
1343    if (buildWorkflowGraphDone_) return true ;
1344   
1345    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1346    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1347
1348    if (!inputFilter) inputFilter = std::shared_ptr<CPassThroughFilter>(new CPassThroughFilter(gc)); 
1349     
1350    if (hasDirectFieldReference())
1351    {
1352      CField* fieldRef = getDirectFieldReference();
1353      bool ret=fieldRef->buildWorkflowGraph(gc); 
1354      if (!ret) return false ; // workflow graph cannot be built at this stage
1355    }
1356
1357    // now construct grid and check if element are enabled
1358    solveGridReference() ; // grid_ is now defined
1359    if (!isGridCompleted()) return false;
1360
1361    // Check if we have an expression to parse
1362    std::shared_ptr<COutputPin> filterExpr ;
1363    if (hasExpression())
1364    {
1365      boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
1366      filterExpr = expr->reduce(gc, *this);
1367      if (!filterExpr) return false ; // workflow graph cannot be built at this stage
1368    }
1369   
1370    // prepare transformation. Need to know before if workflow of auxillary field can be built
1371    if (hasDirectFieldReference())
1372    {
1373      auto gridPath=getGridPath() ;
1374      gridPath.push_back(grid_) ;
1375
1376      CGrid* gridSrc=getDirectFieldReference()->getGrid() ;
1377      for(auto grid : gridPath)
1378      {
1379        grid->solveElementsRefInheritance() ;
1380        grid->completeGrid(gridSrc); // grid generation, to be checked
1381        grid->checkElementsAttributes() ;
1382        grid->prepareTransformGrid(gridSrc) ; // prepare the grid tranformation
1383        for(auto fieldId : grid->getAuxInputTransformGrid()) // try to build workflow graph for auxillary field tranformation
1384          if (!CField::get(fieldId)->buildWorkflowGraph(gc)) return false ;
1385        gridSrc=grid ;
1386      }
1387     
1388      std::shared_ptr<COutputPin> lastFilter ;
1389      if (filterExpr) lastFilter=filterExpr ;
1390      else lastFilter = inputFilter ;
1391     
1392      gridSrc=getDirectFieldReference()->getGrid() ;
1393      for(auto grid : gridPath) 
1394      {
1395        grid->makeTransformGrid() ; // make the grid transformation
1396        if (grid->hasTransform()) 
1397        {
1398          std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridSrc, grid, detectMissingValues, defaultValue); 
1399          lastFilter->connectOutput(filters.first, 0);
1400          lastFilter = filters.second;
1401          gridSrc=grid ;
1402        }
1403      }
1404      instantDataFilter = lastFilter ;
1405     
1406      // connect the input Filter to the reference
1407      getDirectFieldReference()->getInstantDataFilter()->connectOutput(inputFilter,0);
1408    }
1409    else 
1410    {
1411      if (hasFileIn()) // input file, attemp to read the grid from file
1412      {
1413         // must be checked
1414         fileIn_->initRead() ;
1415         fileIn_->checkReadFile();
1416         grid_->solveElementsRefInheritance() ;
1417         if (fileIn_->isClientSide()) fileIn_->readFieldAttributesMetaData(this);
1418         grid_->completeGrid(); // grid generation, to be checked
1419         if (fileIn_->isClientSide()) fileIn_->readFieldAttributesValues(this);
1420         grid_->checkElementsAttributes() ;
1421         grid_->solveDomainAxisBaseRef();
1422         // probably in future tag grid incomplete if coming from a reading
1423         instantDataFilter=inputFilter ;
1424      } 
1425      else if (hasCouplerIn())
1426      {
1427        grid_->checkElementsAttributes() ;
1428        instantDataFilter=inputFilter ;
1429      }
1430      else
1431      {
1432        setModelIn() ; // no reference, the field is potentially a source field from model
1433
1434        grid_->solveElementsRefInheritance() ;
1435        grid_->completeGrid(); // grid generation, to be checked
1436        grid_->checkElementsAttributes() ;
1437        instantDataFilter=inputFilter ;
1438      }
1439    }
1440   
1441    if (hasFileOut())
1442    {
1443      if (fileOut_->isServerSide())
1444      {
1445        this->solveServerOperation() ;
1446      }
1447    }
1448
1449    buildWorkflowGraphDone_ = true ;
1450    workflowEnabled_ = true ;
1451    return true ;
1452  }
1453   
1454  /*!
1455   * Connect field to filter to send data to server. A temporal filter is inserted before accordingly to the
1456   * output frequency of the file
1457   * \param gc the garbage collector to use when building the filter graph
1458   */
1459  void CField::connectToFileServer(CGarbageCollector& gc)
1460  {
1461    // insert temporal filter before sending to files
1462    fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, client));
1463    // insert temporal filter before sending to files
1464    getTemporalDataFilter(gc, fileOut_->output_freq)->connectOutput(fileWriterFilter, 0);
1465  } 
1466
1467  void CField::connectToCouplerOut(CGarbageCollector& gc)
1468  {
1469    // insert temporal filter before sending to files
1470    fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, client));
1471    instantDataFilter->connectOutput(fileWriterFilter, 0);
1472  } 
1473
1474  /*!
1475   * Compute grid index needed to send grid and data to server
1476   */
1477  void CField::computeGridIndexToFileServer(void)
1478  {
1479    grid_->computeGridIndexToFileServer(client) ;
1480  }
1481
1482  /*!
1483   * Connect field to a source filter to receive data from model.
1484   */
1485  void CField::connectToModelInput(CGarbageCollector& gc)
1486  {
1487    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1488    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1489
1490    if (check_if_active.isEmpty()) check_if_active = false; 
1491    modelToClientSourceFilter_ = std::shared_ptr<CModelToClientSourceFilter>(new CModelToClientSourceFilter(gc, grid_, detectMissingValues, defaultValue));
1492    modelToClientSourceFilter_ -> connectOutput(inputFilter,0) ;
1493  } 
1494 
1495  /*!
1496   * Connect field to a source filter to receive data from a client (on server side).
1497   */
1498  void CField::connectToClientInput(CGarbageCollector& gc)
1499  {
1500    serverFromClientSourceFilter_ = std::shared_ptr<CServerFromClientSourceFilter>(new CServerFromClientSourceFilter(gc,  grid_));
1501    serverFromClientSourceFilter_ -> connectOutput(inputFilter,0) ;
1502  } 
1503
1504
1505  /*!
1506   * Connect field to a source filter to receive data from a server (on client side).
1507   */
1508  void CField::connectToServerInput(CGarbageCollector& gc)
1509  {
1510    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1511    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1512
1513    checkTimeAttributes();
1514    serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1515                                                                          detectMissingValues, defaultValue));
1516    //serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc,  grid_, false, false));
1517    serverSourceFilter -> connectOutput(inputFilter,0) ;
1518  } 
1519
1520  /*!
1521   * Connect field to a source filter to receive data from coupler (on client side).
1522   */
1523   void CField::connectToCouplerIn(CGarbageCollector& gc)
1524  {
1525    CContext* context = CContext::getCurrent();
1526
1527    if (freq_op.isEmpty()) freq_op.setValue(TimeStep);
1528    if (freq_offset.isEmpty()) freq_offset.setValue(freq_op.getValue() - TimeStep);
1529   
1530    /* old
1531
1532    freq_operation_srv = freq_op ;
1533    last_operation_srv = context->getCalendar()->getInitDate();
1534    const CDuration toffset = freq_operation_srv - freq_offset.getValue() - context->getCalendar()->getTimeStep();
1535    last_operation_srv     = last_operation_srv - toffset;
1536
1537    clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, false, freq_offset, true)) ;
1538    clientSourceFilter -> connectOutput(inputFilter,0) ;
1539
1540    */
1541    // new
1542
1543    clientFromClientSourceFilter_ = std::shared_ptr<CClientFromClientSourceFilter>(new CClientFromClientSourceFilter(gc, this)) ;
1544    clientFromClientSourceFilter_ -> connectOutput(inputFilter,0) ;
1545   
1546  } 
1547
1548  /*!
1549   * Connect field to a file writer filter to write data in file (on server side).
1550   */
1551  void CField::connectToFileWriter(CGarbageCollector& gc)
1552  {
1553    fileServerWriterFilter = std::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this));
1554    instantDataFilter->connectOutput(fileServerWriterFilter, 0);
1555  } 
1556
1557  /*!
1558   * Connect field to a file reader filter to read data from file (on server side).
1559   */
1560  void CField::connectToFileReader(CGarbageCollector& gc)
1561  {
1562    fileReaderSourceFilter_ = std::shared_ptr<CFileReaderSourceFilter>(new CFileReaderSourceFilter(gc, this));
1563    instantDataFilter->connectOutput(inputFilter, 0);
1564  }
1565
1566
1567  /*!
1568   * Connect field to a store filter to output data to model on client Side
1569   */
1570  void CField::connectToModelOutput(CGarbageCollector& gc)
1571  {
1572    clientToModelStoreFilter_ = std::shared_ptr<CClientToModelStoreFilter>(new CClientToModelStoreFilter(gc, this));
1573    instantDataFilter->connectOutput(clientToModelStoreFilter_, 0);
1574  }
1575
1576
1577 
1578  void CField::connectToServerToClient(CGarbageCollector& gc)
1579  {
1580    serverToClientFilter_ = std::shared_ptr<CServerToClientFilter>(new CServerToClientFilter(gc, this, client));
1581    instantDataFilter->connectOutput(serverToClientFilter_, 0);
1582  }
1583
1584  /*!
1585   * Transform the grid_path attribut into vector of grid.
1586   * \return the vector CGrid* containing the list of grid path for tranformation
1587   */ 
1588  vector<CGrid*> CField::getGridPath(void)
1589  {
1590    std::vector<CGrid*> gridPath;
1591
1592    if (hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
1593    {
1594      if (!grid_path.isEmpty())
1595      {
1596        std::string gridId;
1597        size_t start = 0, end;
1598
1599        do
1600        {
1601          end = grid_path.getValue().find(',', start);
1602          if (end != std::string::npos)
1603          {
1604            gridId = grid_path.getValue().substr(start, end - start);
1605            start = end + 1;
1606          }
1607          else gridId = grid_path.getValue().substr(start);
1608
1609          if (!CGrid::has(gridId))
1610            ERROR("void CField::solveTransformedGrid()",
1611               << "Invalid grid_path, the grid '" << gridId << "' does not exist.");
1612
1613          gridPath.push_back(CGrid::get(gridId));
1614        }
1615        while (end != std::string::npos);
1616      }
1617    }
1618    return gridPath ;
1619  }
1620
1621  /*!
1622   * Constructs the graph filter for the field, enabling or not the data output.
1623   * This method should not be called more than once with enableOutput equal to true.
1624   *
1625   * \param gc the garbage collector to use when building the filter graph
1626   * \param enableOutput must be true when the field data is to be
1627   *                     read by the client or/and written to a file
1628   */
1629   // ym obselete : to be removed later....
1630  void CField::buildFilterGraph(CGarbageCollector& gc, bool enableOutput)
1631  TRY
1632  {     
1633    if (!isReferenceSolvedAndTransformed) solveAllEnabledFieldsAndTransform();
1634    if (!isGridChecked) checkGridOfEnabledFields();
1635
1636    const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1637    const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1638
1639    CContext* context = CContext::getCurrent();
1640    bool hasWriterServer = context->getServiceType()==CServicesManager::OUT_SERVER ;
1641    bool hasIntermediateServer = context->getServiceType()==CServicesManager::GATHERER ;
1642
1643    if (hasWriterServer)
1644    {
1645      if (!instantDataFilter)
1646        instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false));
1647
1648
1649      // If the field data is to be read by the client or/and written to a file
1650      if (enableOutput && !clientToModelStoreFilter_ && !fileWriterFilter)
1651      {
1652        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1653        {
1654          fileServerWriterFilter = std::shared_ptr<CFileServerWriterFilter>(new CFileServerWriterFilter(gc, this));
1655          instantDataFilter->connectOutput(fileServerWriterFilter, 0);
1656        }
1657      }
1658    }
1659    else if (hasIntermediateServer)
1660    {
1661      if (!instantDataFilter)
1662        instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, false));
1663
1664      // If the field data is to be read by the client or/and written to a file
1665      if (enableOutput && !clientToModelStoreFilter_ && !fileWriterFilter)
1666      {
1667        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1668        {
1669          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient()));
1670          instantDataFilter->connectOutput(fileWriterFilter, 0);
1671        }
1672      }
1673    }
1674    else
1675    {
1676      // Start by building a filter which can provide the field's instant data
1677      if (!instantDataFilter)
1678      {
1679        // Check if we have an expression to parse
1680        if (hasExpression())
1681        {
1682          boost::scoped_ptr<IFilterExprNode> expr(parseExpr(getExpression() + '\0'));
1683          std::shared_ptr<COutputPin> filter = expr->reduce(gc, *this);
1684
1685          // Check if a spatial transformation is needed
1686          if (!field_ref.isEmpty())
1687          {
1688            CGrid* gridRef = CField::get(field_ref)->grid_;
1689            if (grid_ && grid_ != gridRef && grid_->hasTransform())
1690            {
1691                std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters = CSpatialTransformFilter::buildFilterGraph(gc, gridRef, grid_, detectMissingValues, defaultValue); 
1692
1693              filter->connectOutput(filters.first, 0);
1694              filter = filters.second;
1695            }
1696          }
1697
1698          instantDataFilter = filter;
1699        }
1700        // Check if we have a reference on another field
1701        else if (!field_ref.isEmpty()) instantDataFilter = getFieldReference(gc);
1702        // Check if the data is to be read from a file
1703        else if (getRelFile() && !getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::read)
1704        {
1705          checkTimeAttributes();
1706          instantDataFilter = serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1707                                                                                                       detectMissingValues, defaultValue));
1708        }
1709        else // The data might be passed from the model
1710        {
1711          if (check_if_active.isEmpty()) check_if_active = false; 
1712          instantDataFilter = clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, false, true, NoneDu, false,
1713                                                                                                      detectMissingValues, defaultValue));
1714        }
1715      }
1716
1717      // If the field data is to be read by the client or/and written to a file
1718      if (enableOutput && !clientToModelStoreFilter_ && !fileWriterFilter)
1719      {
1720        if (!read_access.isEmpty() && read_access)
1721        {
1722          clientToModelStoreFilter_ = std::shared_ptr<CClientToModelStoreFilter>(new CClientToModelStoreFilter(gc, this));
1723          instantDataFilter->connectOutput(clientToModelStoreFilter_, 0);
1724        }
1725
1726        if (getRelFile() && (getRelFile()->mode.isEmpty() || getRelFile()->mode == CFile::mode_attr::write))
1727        {
1728          fileWriterFilter = std::shared_ptr<CFileWriterFilter>(new CFileWriterFilter(gc, this, getRelFile()->getContextClient()));
1729          getTemporalDataFilter(gc, getRelFile()->output_freq)->connectOutput(fileWriterFilter, 0);
1730        }
1731      }
1732    }
1733  }
1734  CATCH_DUMP_ATTR
1735
1736  /*!
1737   * Returns the filter needed to handle the field reference.
1738   * This method should only be called when building the filter graph corresponding to the field.
1739   *
1740   * \param gc the garbage collector to use
1741   * \return the output pin corresponding to the field reference
1742   */
1743  std::shared_ptr<COutputPin> CField::getFieldReference(CGarbageCollector& gc)
1744  TRY
1745  {
1746    if (instantDataFilter || field_ref.isEmpty())
1747      ERROR("COutputPin* CField::getFieldReference(CGarbageCollector& gc)",
1748            "Impossible to get the field reference for a field which has already been parsed or which does not have a field_ref.");
1749
1750    CField* fieldRef = CField::get(field_ref);
1751    fieldRef->buildFilterGraph(gc, false);
1752
1753    std::pair<std::shared_ptr<CFilter>, std::shared_ptr<CFilter> > filters;
1754    // Check if a spatial transformation is needed
1755    if (grid_ && grid_ != fieldRef->grid_ && grid_->hasTransform())
1756    {       
1757      bool hasMissingValue = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1758      double defaultValue  = hasMissingValue ? default_value : (!default_value.isEmpty() ? default_value : 0.0);                               
1759      filters = CSpatialTransformFilter::buildFilterGraph(gc, fieldRef->grid_, grid_, hasMissingValue, defaultValue);
1760    }
1761    else
1762      filters.first = filters.second = std::shared_ptr<CFilter>(new CPassThroughFilter(gc));
1763
1764    fieldRef->getInstantDataFilter()->connectOutput(filters.first, 0);
1765
1766    return filters.second;
1767  }
1768  CATCH_DUMP_ATTR
1769
1770  /*!
1771   * Returns the filter needed to handle a self reference in the field's expression.
1772   * If the needed filter does not exist, it is created, otherwise it is reused.
1773   * This method should only be called when building the filter graph corresponding
1774   * to the field's expression.
1775   *
1776   * \param gc the garbage collector to use
1777   * \return the output pin corresponding to a self reference
1778   */
1779
1780/* old version
1781  std::shared_ptr<COutputPin> CField::getSelfReference(CGarbageCollector& gc)
1782  TRY
1783  {
1784    if (instantDataFilter || !hasExpression())
1785      ERROR("COutputPin* CField::getSelfReference(CGarbageCollector& gc)",
1786            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1787
1788    if (!selfReferenceFilter)
1789    {
1790      const bool detectMissingValues = (!detect_missing_value.isEmpty() && !default_value.isEmpty() && detect_missing_value == true);
1791      const double defaultValue  = detectMissingValues ? default_value : (!default_value.isEmpty() ? default_value : 0.0);
1792
1793      if (file && !file->mode.isEmpty() && file->mode == CFile::mode_attr::read)
1794      {
1795        if (!serverSourceFilter)
1796        {
1797          checkTimeAttributes();
1798          serverSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, false, freq_offset, true,
1799                                                              detectMissingValues, defaultValue));
1800         }
1801
1802        selfReferenceFilter = serverSourceFilter;
1803      }
1804      else if (!field_ref.isEmpty())
1805      {
1806        CField* fieldRef = CField::get(field_ref);
1807        fieldRef->buildFilterGraph(gc, false);
1808        selfReferenceFilter = fieldRef->getInstantDataFilter();
1809      }
1810      else
1811      {
1812        if (!clientSourceFilter)
1813        {
1814          if (check_if_active.isEmpty()) check_if_active = false;
1815          clientSourceFilter = std::shared_ptr<CSourceFilter>(new CSourceFilter(gc, grid_, true, true, NoneDu, false,
1816                                                                                detectMissingValues, defaultValue));
1817        }
1818
1819        selfReferenceFilter = clientSourceFilter;
1820      }
1821    }
1822
1823    return selfReferenceFilter;
1824  }
1825  CATCH_DUMP_ATTR
1826*/
1827  std::shared_ptr<COutputPin> CField::getSelfReference(CGarbageCollector& gc)
1828  TRY
1829  {
1830    return inputFilter ;
1831  } 
1832  CATCH_DUMP_ATTR
1833
1834  /*!
1835   * Returns the temporal filter corresponding to the field's temporal operation
1836   * for the specified operation frequency. The filter is created if it does not
1837   * exist, otherwise it is reused.
1838   *
1839   * \param gc the garbage collector to use
1840   * \param outFreq the operation frequency, i.e. the frequency at which the output data will be computed
1841   * \return the output pin corresponding to the requested temporal filter
1842   */
1843  std::shared_ptr<COutputPin> CField::getTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1844  TRY
1845  {
1846    std::map<CDuration, std::shared_ptr<COutputPin> >::iterator it = temporalDataFilters.find(outFreq);
1847
1848    if (it == temporalDataFilters.end())
1849    {
1850      if (operation.isEmpty())
1851        ERROR("void CField::getTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1852              << "An operation must be defined for field \"" << getId() << "\".");
1853
1854      checkTimeAttributes(&outFreq);
1855
1856      const bool detectMissingValues = (!detect_missing_value.isEmpty()  && detect_missing_value == true);
1857      std::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
1858                                                                             CContext::getCurrent()->getCalendar()->getInitDate(),
1859                                                                             freq_op, freq_offset, outFreq, detectMissingValues));
1860
1861      instantDataFilter->connectOutput(temporalFilter, 0);
1862
1863      it = temporalDataFilters.insert(std::make_pair(outFreq, temporalFilter)).first;
1864    }
1865
1866    return it->second;
1867  }
1868  CATCH_DUMP_ATTR
1869
1870  /*!
1871    * Returns the temporal filter corresponding to the field's temporal operation
1872    * for the specified operation frequency.
1873    *
1874    * \param gc the garbage collector to use
1875    * \param outFreq the operation frequency, i.e. the frequency at which the output data will be computed
1876    * \return the output pin corresponding to the requested temporal filter
1877    */
1878
1879  std::shared_ptr<COutputPin> CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1880  TRY
1881  {
1882    if (instantDataFilter || !hasExpression())
1883      ERROR("COutputPin* CField::getSelfTemporalDataFilter(CGarbageCollector& gc)",
1884            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1885   
1886    if (selfTemporalDataFilter) return selfTemporalDataFilter;
1887
1888    if (hasDirectFieldReference())
1889    {
1890      CField* fieldRef=getDirectFieldReference();
1891      return fieldRef->getTemporalDataFilter(gc, outFreq) ;
1892    }
1893    else
1894    {
1895      if (selfTemporalDataFilter) return selfTemporalDataFilter ;
1896
1897      if (operation.isEmpty())
1898        ERROR("void CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1899              << "An operation must be defined for field \"" << getId() << "\".");
1900
1901      checkTimeAttributes(&outFreq); //bof
1902
1903      const bool detectMissingValues = (!detect_missing_value.isEmpty() && detect_missing_value == true);
1904      selfTemporalDataFilter = std::shared_ptr<CTemporalFilter>(new CTemporalFilter(gc, operation,
1905                                                                CContext::getCurrent()->getCalendar()->getInitDate(),
1906                                                                freq_op, freq_offset, outFreq, detectMissingValues));
1907
1908      inputFilter->connectOutput(selfTemporalDataFilter, 0);
1909      return selfTemporalDataFilter ;
1910    }
1911  }
1912  CATCH_DUMP_ATTR
1913
1914/* old version   
1915  std::shared_ptr<COutputPin> CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)
1916  TRY
1917  {
1918    if (instantDataFilter || !hasExpression())
1919      ERROR("COutputPin* CField::getSelfTemporalDataFilter(CGarbageCollector& gc)",
1920            "Impossible to add a self reference to a field which has already been parsed or which does not have an expression.");
1921
1922    if (!selfReferenceFilter) getSelfReference(gc) ;
1923
1924    if (serverSourceFilter || clientSourceFilter)
1925    {
1926      if (operation.isEmpty())
1927        ERROR("void CField::getSelfTemporalDataFilter(CGarbageCollector& gc, CDuration outFreq)",
1928              << "An operation must be defined for field \"" << getId() << "\".");
1929
1930      checkTimeAttributes(&outFreq);
1931
1932      const bool detectMissingValues = (!detect_missing_value.isEmpty() && detect_missing_value == true);
1933      std::shared_ptr<CTemporalFilter> temporalFilter(new CTemporalFilter(gc, operation,
1934                                                                          CContext::getCurrent()->getCalendar()->getInitDate(),
1935                                                                          freq_op, freq_offset, outFreq, detectMissingValues));
1936
1937      selfReferenceFilter->connectOutput(temporalFilter, 0);
1938      return temporalFilter ;
1939    }
1940    else if (!field_ref.isEmpty())
1941    {
1942      CField* fieldRef = CField::get(field_ref);
1943      fieldRef->buildFilterGraph(gc, false);
1944      return fieldRef->getTemporalDataFilter(gc, outFreq) ;
1945    }
1946  }
1947  CATCH_DUMP_ATTR
1948*/
1949
1950  //----------------------------------------------------------------
1951/*
1952   void CField::fromBinary(StdIStream& is)
1953   {
1954      SuperClass::fromBinary(is);
1955#define CLEAR_ATT(name_)\
1956      SuperClassAttribute::operator[](#name_)->reset()
1957
1958         CLEAR_ATT(domain_ref);
1959         CLEAR_ATT(axis_ref);
1960#undef CLEAR_ATT
1961
1962   }
1963*/
1964   //----------------------------------------------------------------
1965
1966  void CField::solveGridReference(void)
1967  TRY
1968  {
1969    if (grid_!=nullptr) return ; // already done
1970
1971    if (grid_ref.isEmpty() && domain_ref.isEmpty() && axis_ref.isEmpty() && scalar_ref.isEmpty())
1972    {
1973      ERROR("CField::solveGridReference(void)",
1974            << "A grid must be defined for field '" << getFieldOutputName() << "' .");
1975    }
1976    else if (!grid_ref.isEmpty() && (!domain_ref.isEmpty() || !axis_ref.isEmpty() || !scalar_ref.isEmpty()))
1977    {
1978      ERROR("CField::solveGridReference(void)",
1979            << "Field '" << getFieldOutputName() << "' has both a grid and a domain/axis/scalar." << std::endl
1980            << "Please define either 'grid_ref' or 'domain_ref'/'axis_ref'/'scalar_ref'.");
1981    }
1982
1983    if (grid_ref.isEmpty())
1984    {
1985      std::vector<CDomain*> vecDom;
1986      std::vector<CAxis*> vecAxis;
1987      std::vector<CScalar*> vecScalar;
1988      std::vector<int> axisDomainOrderTmp;
1989
1990      std::vector<CDomain*> vecDomRef;
1991      std::vector<CAxis*> vecAxisRef;
1992      std::vector<CScalar*> vecScalarRef;
1993       
1994      if (!domain_ref.isEmpty())
1995      {
1996        StdString tmp = domain_ref.getValue();
1997        if (CDomain::has(domain_ref))
1998        {
1999          vecDom.push_back(CDomain::get(domain_ref));
2000          vecDomRef.push_back(CDomain::createDomain());
2001          vecDomRef.back()->domain_ref=domain_ref;
2002          axisDomainOrderTmp.push_back(2);
2003        }
2004        else  ERROR("CField::solveGridReference(void)",
2005                    << "Invalid reference to domain '" << domain_ref.getValue() << "'.");
2006      }
2007
2008      if (!axis_ref.isEmpty())
2009      {
2010        if (CAxis::has(axis_ref))
2011        {
2012          vecAxis.push_back(CAxis::get(axis_ref));
2013          vecAxisRef.push_back(CAxis::createAxis());
2014          vecAxisRef.back()->axis_ref=axis_ref;
2015          axisDomainOrderTmp.push_back(1);
2016        }
2017        else  ERROR("CField::solveGridReference(void)",
2018                    << "Invalid reference to axis '" << axis_ref.getValue() << "'.");
2019      }
2020
2021      if (!scalar_ref.isEmpty())
2022      {
2023        if (CScalar::has(scalar_ref))
2024        {
2025          vecScalar.push_back(CScalar::get(scalar_ref));
2026          vecScalarRef.push_back(CScalar::createScalar());
2027          vecScalarRef.back()->scalar_ref=scalar_ref;
2028          axisDomainOrderTmp.push_back(0);
2029        }
2030        else ERROR("CField::solveGridReference(void)",
2031                   << "Invalid reference to scalar '" << scalar_ref.getValue() << "'.");
2032      }
2033       
2034      CArray<int,1> axisDomainOrder(axisDomainOrderTmp.size());
2035      for (int idx = 0; idx < axisDomainOrderTmp.size(); ++idx)
2036      {
2037        axisDomainOrder(idx) = axisDomainOrderTmp[idx];
2038      }
2039
2040      // Warning: the gridId shouldn't be set as the grid_ref since it could be inherited
2041      StdString gridId = CGrid::generateId(vecDom, vecAxis, vecScalar,axisDomainOrder);
2042      if (CGrid::has(gridId)) this->grid_ = CGrid::get(gridId);
2043      else  this->grid_ = CGrid::createGrid(gridId, vecDomRef, vecAxisRef, vecScalarRef,axisDomainOrder);
2044    }
2045    else
2046    {
2047      if (CGrid::has(grid_ref)) this->grid_ = CGrid::get(grid_ref);
2048      else  ERROR("CField::solveGridReference(void)",
2049                   << "Invalid reference to grid '" << grid_ref.getValue() << "'.");
2050    }
2051  }
2052  CATCH_DUMP_ATTR
2053
2054  void CField::solveGridDomainAxisRef(bool checkAtt)
2055  TRY
2056  {
2057    grid_->solveDomainAxisRef(checkAtt);
2058  }
2059  CATCH_DUMP_ATTR
2060
2061  void CField::solveCheckMaskIndex(bool doSendingIndex)
2062  TRY
2063  {
2064    grid_->checkMaskIndex(doSendingIndex);
2065  }
2066  CATCH_DUMP_ATTR
2067
2068  void CField::solveTransformedGrid()
2069  TRY
2070  {
2071    if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
2072    {
2073      std::vector<CGrid*> grids;
2074      // Source grid
2075      grids.push_back(getDirectFieldReference()->grid_);
2076      auto gridPath = getGridPath() ;
2077      grids.insert(grids.begin(), gridPath.begin(), gridPath.end());
2078
2079      for (size_t i = 0, count = grids.size() - 1; i < count; ++i)
2080      {
2081        CGrid *gridSrc  = grids[i];
2082        CGrid *gridDest = grids[i + 1];
2083        if (!gridDest->isTransformed()) gridDest->transformGrid(gridSrc);
2084      }
2085    }
2086    else if (grid_ && grid_->hasTransform() && !grid_->isTransformed())
2087    {
2088      // Temporarily deactivate the self-transformation of grid
2089      // grid_->transformGrid(grid_);
2090    }
2091  }
2092  CATCH_DUMP_ATTR
2093
2094  void CField::solveGenerateGrid()
2095  TRY
2096  {
2097    if (grid_ && !grid_->isTransformed() && hasDirectFieldReference() && grid_ != getDirectFieldReference()->grid_)
2098      grid_->completeGrid(getDirectFieldReference()->grid_);
2099    else grid_->completeGrid();
2100  }
2101  CATCH_DUMP_ATTR
2102
2103  void CField::solveGridDomainAxisBaseRef()
2104  TRY
2105  {
2106    grid_->solveDomainAxisRef(false);
2107    grid_->solveDomainAxisBaseRef();
2108  }
2109  CATCH_DUMP_ATTR
2110
2111  ///-------------------------------------------------------------------
2112
2113  template <>
2114  void CGroupTemplate<CField, CFieldGroup, CFieldAttributes>::solveRefInheritance(void)
2115  TRY
2116  {
2117    if (this->group_ref.isEmpty()) return;
2118    StdString gref = this->group_ref.getValue();
2119
2120    if (!CFieldGroup::has(gref))
2121      ERROR("CGroupTemplate<CField, CFieldGroup, CFieldAttributes>::solveRefInheritance(void)",
2122         << "[ gref = " << gref << "]"
2123         << " invalid group name !");
2124
2125    CFieldGroup* group = CFieldGroup::get(gref);
2126    CFieldGroup* owner = CFieldGroup::get(boost::polymorphic_downcast<CFieldGroup*>(this));
2127    owner->setAttributes(group); // inherite of attributes of group reference
2128     
2129    std::vector<CField*> allChildren  = group->getAllChildren();
2130    std::vector<CField*>::iterator it = allChildren.begin(), end = allChildren.end();
2131
2132    for (; it != end; it++)
2133    {
2134      CField* child = *it;
2135     if (child->hasId()) owner->createChild()->field_ref.setValue(child->getId());
2136    }
2137  }
2138  CATCH_DUMP_ATTR
2139
2140  void CField::scaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset)
2141  TRY
2142  {
2143    data = (data - addOffset) / scaleFactor;
2144  }
2145  CATCH_DUMP_ATTR
2146
2147  void CField::invertScaleFactorAddOffset(CArray<double,1>& data, double scaleFactor, double addOffset)
2148  TRY
2149  {
2150    data = data * scaleFactor + addOffset;
2151  }
2152  CATCH_DUMP_ATTR
2153
2154  void CField::outputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2155  TRY
2156  { 
2157    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2158    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2159    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2160    {
2161      dataOut(outIndexServer(idx)) = dataIn(outIndexClient(idx));
2162    }
2163  }
2164  CATCH_DUMP_ATTR
2165
2166  void CField::inputField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2167  TRY
2168  {
2169    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2170    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2171    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2172    {
2173      dataOut(outIndexClient(idx)) = dataIn(outIndexServer(idx));
2174    }
2175  }
2176  CATCH_DUMP_ATTR
2177
2178 void CField::outputCompressedField(const CArray<double,1>& dataIn, CArray<double,1>& dataOut)
2179 TRY
2180 {
2181    CArray<size_t,1>& outIndexClient = grid_->localIndexToWriteOnClient_;
2182    CArray<size_t,1>& outIndexServer = grid_->localIndexToWriteOnServer_;
2183    for (size_t idx = 0; idx < outIndexServer.numElements(); ++idx)
2184    {
2185      dataOut((idx)) = dataIn(outIndexClient(idx));
2186    }
2187  }
2188  CATCH_DUMP_ATTR
2189
2190  ///-------------------------------------------------------------------
2191
2192  void CField::parse(xml::CXMLNode& node)
2193  TRY
2194  {
2195    string newContent ;
2196    SuperClass::parse(node);
2197    if (node.goToChildElement())
2198    {
2199      do
2200      {
2201        if (node.getElementName() == "variable" || node.getElementName() == "variable_group") this->getVirtualVariableGroup()->parseChild(node);
2202        else if (node.getElementName() == "expr") if (node.getContent(newContent)) content+=newContent ;
2203      } while (node.goToNextElement());
2204      node.goToParentElement();
2205    }
2206    if (node.getContent(newContent)) content=newContent ;
2207  }
2208  CATCH_DUMP_ATTR
2209
2210 /*!
2211   This function retrieves Id of corresponding domain_ref and axis_ref (if any)
2212   of a field. In some cases, only domain exists but axis doesn't
2213   \return pair of Domain and Axis id
2214  */
2215  const std::vector<StdString>& CField::getRefDomainAxisIds()
2216  TRY
2217  {
2218    CGrid* cgPtr = getRelGrid();
2219    if (NULL != cgPtr)
2220    {
2221      std::vector<StdString>::iterator it;
2222      if (!domain_ref.isEmpty())
2223      {
2224        std::vector<StdString> domainList = cgPtr->getDomainList();
2225        it = std::find(domainList.begin(), domainList.end(), domain_ref.getValue());
2226        if (domainList.end() != it) domAxisScalarIds_[0] = *it;
2227      }
2228
2229      if (!axis_ref.isEmpty())
2230      {
2231        std::vector<StdString> axisList = cgPtr->getAxisList();
2232        it = std::find(axisList.begin(), axisList.end(), axis_ref.getValue());
2233        if (axisList.end() != it) domAxisScalarIds_[1] = *it;
2234      }
2235
2236      if (!scalar_ref.isEmpty())
2237      {
2238        std::vector<StdString> scalarList = cgPtr->getScalarList();
2239        it = std::find(scalarList.begin(), scalarList.end(), scalar_ref.getValue());
2240        if (scalarList.end() != it) domAxisScalarIds_[2] = *it;
2241      }
2242    }
2243    return (domAxisScalarIds_);
2244  }
2245  CATCH_DUMP_ATTR
2246
2247  CVariable* CField::addVariable(const string& id)
2248  TRY
2249  {
2250    return vVariableGroup->createChild(id);
2251  }
2252  CATCH
2253
2254  CVariableGroup* CField::addVariableGroup(const string& id)
2255  TRY
2256  {
2257    return vVariableGroup->createChildGroup(id);
2258  }
2259  CATCH
2260
2261  void CField::setContextClient(CContextClient* contextClient)
2262  TRY
2263  {
2264    CContext* context = CContext::getCurrent();
2265    client = contextClient;
2266 
2267    // A grid is sent by a client (both for read or write) or by primary server (write only)
2268    if (context->getServiceType()==CServicesManager::GATHERER)
2269    {
2270      if (getRelFile()->mode.isEmpty() || (!getRelFile()->mode.isEmpty() && getRelFile()->mode == CFile::mode_attr::write))
2271        grid_->setContextClient(contextClient);
2272    }
2273    else if (context->getServiceType()==CServicesManager::CLIENT)
2274      grid_->setContextClient(contextClient);
2275  }
2276  CATCH_DUMP_ATTR
2277
2278  void CField::sendCloseDefinition(void)
2279  {
2280    CContext::getCurrent()->sendCloseDefinition(client) ;
2281  }
2282
2283  void CField::sendFieldToFileServer(void)
2284  {
2285    CContext::getCurrent()->sendContextToFileServer(client);
2286    getRelFile()->sendFileToFileServer(client);
2287    grid_->sendGridToFileServer(client);
2288    this->sendAllAttributesToServer(client);
2289    this->sendAddAllVariables(client);
2290  }
2291
2292  void CField::sendFieldToCouplerOut(void)
2293  {
2294    if (sendFieldToCouplerOut_done_) return ;
2295    else sendFieldToCouplerOut_done_=true ;
2296    grid_->sendGridToCouplerOut(client, this->getId());
2297    this->sendGridCompleted();
2298
2299  }
2300 
2301  void CField::makeGridAliasForCoupling(void) 
2302  { 
2303    grid_->makeAliasForCoupling(this->getId()); 
2304  }
2305
2306 //! Client side: Send a message  announcing that the grid definition has been received from a coupling context
2307   void CField::sendGridCompleted(void)
2308   TRY
2309   {
2310      CEventClient event(getType(),EVENT_ID_GRID_COMPLETED);
2311
2312      if (client->isServerLeader())
2313      {
2314        CMessage msg;
2315        msg<<this->getId();
2316        for (auto& rank : client->getRanksServerLeader()) event.push(rank,1,msg);
2317        client->sendEvent(event);
2318      }
2319      else client->sendEvent(event);
2320   }
2321   CATCH_DUMP_ATTR
2322
2323   //! Server side: Receive a message announcing that the grid definition has been received from a coupling context
2324   void CField::recvGridCompleted(CEventServer& event)
2325   TRY
2326   {
2327      CBufferIn* buffer=event.subEvents.begin()->buffer;
2328      string id;
2329      *buffer>>id ;
2330      get(id)->recvGridCompleted(*buffer);
2331   }
2332   CATCH
2333
2334   //! Server side: Receive a message  message  announcing that the grid definition has been received from a coupling context
2335   void CField::recvGridCompleted(CBufferIn& buffer)
2336   TRY
2337   {
2338      setGridCompleted() ;
2339   }
2340   CATCH_DUMP_ATTR
2341
2342
2343
2344
2345
2346  void CField::sendFieldToInputFileServer(void)
2347  {
2348    CContext::getCurrent()->sendContextToFileServer(client);
2349    getRelFile()->sendFileToFileServer(client);
2350    grid_->sendGridToFileServer(client);
2351    read_access=true ; // not the best solution, but on server side, the field must be a starting point of the workflow
2352                       // must be replace by a better solution when implementing filters for reading and send to client
2353                       // on server side
2354    this->sendAllAttributesToServer(client);
2355    this->sendAddAllVariables(client);
2356  }
2357
2358  void CField::sendAddAllVariables(CContextClient* client)
2359  TRY
2360  {
2361    std::vector<CVariable*> allVar = getAllVariables();
2362    std::vector<CVariable*>::const_iterator it = allVar.begin();
2363    std::vector<CVariable*>::const_iterator itE = allVar.end();
2364
2365    for (; it != itE; ++it)
2366    {
2367      this->sendAddVariable((*it)->getId(), client);
2368      (*it)->sendAllAttributesToServer(client);
2369      (*it)->sendValue(client);
2370    }
2371  }
2372  CATCH_DUMP_ATTR
2373
2374  /*!
2375   * Send all Attributes to server. This method is overloaded, since only grid_ref attribute
2376   * must be sent to server and not domain_ref/axis_ref/scalar_ref.
2377   */
2378   
2379  void CField::sendAllAttributesToServer(CContextClient* client)
2380  TRY
2381  {
2382    if (grid_ref.isEmpty())
2383    {
2384      grid_ref=grid_->getId() ;
2385      SuperClass::sendAllAttributesToServer(client) ;
2386      domain_ref.reset() ;
2387      axis_ref.reset() ;
2388      scalar_ref.reset() ;
2389      grid_ref.reset();
2390    }
2391    else SuperClass::sendAllAttributesToServer(client) ;
2392  }
2393  CATCH_DUMP_ATTR
2394   
2395  void CField::sendAddVariable(const string& id, CContextClient* client)
2396  TRY
2397  {
2398    sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE, client);
2399  }
2400  CATCH_DUMP_ATTR
2401
2402  void CField::sendAddVariableGroup(const string& id, CContextClient* client)
2403  TRY
2404  {
2405    sendAddItem(id, (int)EVENT_ID_ADD_VARIABLE_GROUP, client);
2406  }
2407  CATCH_DUMP_ATTR
2408
2409  void CField::recvAddVariable(CEventServer& event)
2410  TRY
2411  {
2412    CBufferIn* buffer = event.subEvents.begin()->buffer;
2413    string id;
2414    *buffer >> id;
2415    get(id)->recvAddVariable(*buffer);
2416  }
2417  CATCH
2418
2419  void CField::recvAddVariable(CBufferIn& buffer)
2420  TRY
2421  {
2422    string id;
2423    buffer >> id;
2424    addVariable(id);
2425  }
2426  CATCH_DUMP_ATTR
2427
2428  void CField::recvAddVariableGroup(CEventServer& event)
2429  TRY
2430  {
2431    CBufferIn* buffer = event.subEvents.begin()->buffer;
2432    string id;
2433    *buffer >> id;
2434    get(id)->recvAddVariableGroup(*buffer);
2435  }
2436  CATCH
2437
2438  void CField::recvAddVariableGroup(CBufferIn& buffer)
2439  TRY
2440  {
2441    string id;
2442    buffer >> id;
2443    addVariableGroup(id);
2444  }
2445  CATCH_DUMP_ATTR
2446
2447  /*!
2448   * Check on freq_off and freq_op attributes.
2449   */
2450  void CField::checkTimeAttributes(CDuration* freqOp)
2451  TRY
2452  {
2453    if (hasFileIn() && !(operation.getValue() == "instant" || operation.getValue() == "once") )     
2454      ERROR("void CField::checkTimeAttributes(void)",
2455         << "Unsupported operation for field '" << getFieldOutputName() << "'." << std::endl
2456         << "Currently only \"instant\" is supported for fields read from file.")
2457
2458    if (freq_op.isEmpty())
2459    {
2460      if (operation.getValue() == "instant")
2461      {
2462        if (hasFileIn() || hasFileOut()) freq_op.setValue(getRelFile()->output_freq.getValue());
2463        else freq_op=*freqOp ;
2464      }
2465      else freq_op.setValue(TimeStep);
2466    }
2467    if (freq_offset.isEmpty()) freq_offset.setValue(hasFileIn() ? NoneDu : (freq_op.getValue() - TimeStep));
2468  }
2469  CATCH_DUMP_ATTR
2470
2471  /*!
2472   * Returns string arithmetic expression associated to the field.
2473   * \return if content is defined return content string, otherwise, if "expr" attribute is defined, return expr string.
2474   */
2475  const string& CField::getExpression(void)
2476  TRY
2477  {
2478    if (!expr.isEmpty() && content.empty())
2479    {
2480      content = expr;
2481      expr.reset();
2482    }
2483
2484    return content;
2485  }
2486  CATCH_DUMP_ATTR
2487
2488  bool CField::hasExpression(void) const
2489  TRY
2490  {
2491    return (!expr.isEmpty() || !content.empty());
2492  }
2493  CATCH
2494
2495  bool CField::hasGridMask(void) const
2496  TRY
2497  {
2498    return (this->grid_->hasMask());
2499  }
2500  CATCH
2501
2502  DEFINE_REF_FUNC(Field,field)
2503} // namespace xios
Note: See TracBrowser for help on using the repository browser.