Changeset 1642 for XIOS/dev/branch_openmp/src/filter

Timestamp:

01/23/19 10:31:44 (5 years ago)

Author:

yushan

Message:

dev on ADA. add flag switch _usingEP/_usingMPI

Location:

XIOS/dev/branch_openmp/src/filter

Files:

• file_writer_filter.cpp (modified) (1 diff)
• source_filter.cpp (modified) (3 diffs)
• source_filter.hpp (modified) (3 diffs)
• spatial_transform_filter.cpp (modified) (10 diffs)
• spatial_transform_filter.hpp (modified) (1 diff)
• temporal_filter.cpp (modified) (1 diff)

XIOS/dev/branch_openmp/src/filter/file_writer_filter.cpp

@@ -17 +17 @@
   void CFileWriterFilter::onInputReady(std::vector<CDataPacketPtr> data)
   {
-    const bool detectMissingValue = (!field->detect_missing_value.isEmpty()
-                                      && !field->default_value.isEmpty()
-                                      && field->detect_missing_value == true);
+    const bool detectMissingValue = ( !field->default_value.isEmpty() &&
+                               ( (!field->detect_missing_value.isEmpty() || field->detect_missing_value == true)
+                                 || field->hasGridMask()) );
 
     CArray<double, 1> dataArray = (detectMissingValue) ? data[0]->data.copy() : data[0]->data;

XIOS/dev/branch_openmp/src/filter/source_filter.cpp

@@ -7 +7 @@
 namespace xios
 {
-  CSourceFilter::CSourceFilter(CGarbageCollector& gc, CGrid* grid, bool compression, 
+  CSourceFilter::CSourceFilter(CGarbageCollector& gc, CGrid* grid,
+                               bool compression /*= true*/, bool mask /*= false*/,
                                const CDuration offset /*= NoneDu*/, bool manualTrigger /*= false*/,
                                bool hasMissingValue /*= false*/,
@@ -14 +15 @@
     , grid(grid)
     , compression(compression)
+    , mask(mask)
     , offset(offset)
     , hasMissingValue(hasMissingValue), defaultValue(defaultValue)
@@ -40 +42 @@
     }
     else
-      grid->inputField(data, packet->data);
-
-    
-    
-    // if (compression) grid->inputField(data, packet->data) ;
-    // else
-    // {
-    //   // just make a flat copy
-    //   CArray<double, N> data_tmp(data.copy()) ; // supress const attribute
-    //   CArray<double,1> dataTmp2(data_tmp.dataFirst(),shape(data.numElements()),neverDeleteData) ;
-    //   packet->data = dataTmp2 ;
-    // }
+    {
+      if (mask)
+        grid->maskField(data, packet->data);
+      else
+        grid->inputField(data, packet->data);
+    }
     // Convert missing values to NaN
     if (hasMissingValue)

XIOS/dev/branch_openmp/src/filter/source_filter.hpp

@@ -21 +21 @@
        * \param gc the garbage collector associated with this filter
        * \param grid the grid to which the data is attached
+       * \param compression
+       * \param mask
        * \param offset the offset applied to the timestamp of all packets
        * \param manualTrigger whether the output should be triggered manually
@@ -27 +29 @@
        */
       CSourceFilter(CGarbageCollector& gc, CGrid* grid,
-                    bool compression=true,
+                    bool compression = true,
+                    bool mask = false,
                     const CDuration offset = NoneDu, bool manualTrigger = false,
                     bool hasMissingValue = false,
@@ -61 +64 @@
 
     private:
-      CGrid* grid; //!< The grid attached to the data the filter can accept
-      const CDuration offset; //!< The offset applied to the timestamp of all packets
+      CGrid* grid;             //!< The grid attached to the data the filter can accept
+      const CDuration offset;  //!< The offset applied to the timestamp of all packets
       const bool hasMissingValue;
       const double defaultValue;
-      const bool compression ; //!< indicate if the data need to be compressed : on client size : true, on server side : false
+      const bool compression ; //!< indicates if data need to be compressed : on client side : true, on server side : false
+      const bool mask ;        //!< indicates whether grid mask should be applied (true for clients, false for servers)
   }; // class CSourceFilter
 } // namespace xios

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.cpp

@@ -1 @@
-#include "mpi.hpp"
 #include "spatial_transform_filter.hpp"
 #include "grid_transformation.hpp"
@@ -5 +4 @@
 #include "context_client.hpp"
 #include "timer.hpp"
-using namespace ep_lib;
 
 namespace xios
@@ -70 +68 @@
       onOutputReady(outputPacket);
   }
-
-
-
-
 
   CSpatialTemporalFilter::CSpatialTemporalFilter(CGarbageCollector& gc, CSpatialTransformFilterEngine* engine, CGridTransformation* gridTransformation, double outputValue, size_t inputSlotsCount)
@@ -142 +136 @@
   }
 
-  std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> > *CSpatialTransformFilterEngine::engines_ptr = 0;
+  std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> > CSpatialTransformFilterEngine::engines;
 
   CSpatialTransformFilterEngine* CSpatialTransformFilterEngine::get(CGridTransformation* gridTransformation)
@@ -149 +143 @@
       ERROR("CSpatialTransformFilterEngine& CSpatialTransformFilterEngine::get(CGridTransformation* gridTransformation)",
             "Impossible to get the requested engine, the grid transformation is invalid.");
-    
-    if(engines_ptr == NULL) engines_ptr = new std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> >;
-
-
-    std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines_ptr->find(gridTransformation);
-    if (it == engines_ptr->end())
+
+    std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines.find(gridTransformation);
+    if (it == engines.end())
     {
       std::shared_ptr<CSpatialTransformFilterEngine> engine(new CSpatialTransformFilterEngine(gridTransformation));
-      it = engines_ptr->insert(std::make_pair(gridTransformation, engine)).first;
+      it = engines.insert(std::make_pair(gridTransformation, engine)).first;
     }
 
@@ -206 +197 @@
     const std::list<CGridTransformation::RecvIndexGridDestinationMap>& listLocalIndexToReceive = gridTransformation->getLocalIndexToReceiveOnGridDest();
     const std::list<size_t>& listNbLocalIndexToReceive = gridTransformation->getNbLocalIndexToReceiveOnGridDest();
-    const std::list<std::vector<bool> >& listLocalIndexMaskOnDest = gridTransformation->getLocalMaskIndexOnGridDest();
     const std::vector<CGenericAlgorithmTransformation*>& listAlgos = gridTransformation->getAlgos();
 
@@ -215 +205 @@
     std::list<CGridTransformation::RecvIndexGridDestinationMap>::const_iterator itListRecv = listLocalIndexToReceive.begin();
     std::list<size_t>::const_iterator itNbListRecv = listNbLocalIndexToReceive.begin();
-    std::list<std::vector<bool> >::const_iterator itLocalMaskIndexOnDest = listLocalIndexMaskOnDest.begin();
     std::vector<CGenericAlgorithmTransformation*>::const_iterator itAlgo = listAlgos.begin();
 
-    for (; itListSend != iteListSend; ++itListSend, ++itListRecv, ++itNbListRecv, ++itLocalMaskIndexOnDest, ++itAlgo)
+    for (; itListSend != iteListSend; ++itListSend, ++itListRecv, ++itNbListRecv, ++itAlgo)
     {
       CArray<double,1> dataCurrentSrc(dataCurrentDest);
@@ -235 +224 @@
 
       idxSendBuff = 0;
-      std::vector<MPI_Request> sendRecvRequest(localIndexToSend.size() + itListRecv->size());
-      int position = 0;
+      std::vector<ep_lib::MPI_Request> sendRecvRequest;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
       {
@@ -246 +234 @@
           sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
         }
-        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position++]);
+        sendRecvRequest.push_back(ep_lib::MPI_Request());
+        ep_lib::MPI_Isend(sendBuff[idxSendBuff], countSize, EP_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest.back());
       }
 
@@ -263 +252 @@
         int srcRank = itRecv->first;
         int countSize = itRecv->second.size();
-        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position++]);
+        sendRecvRequest.push_back(ep_lib::MPI_Request());
+        ep_lib::MPI_Irecv(recvBuff + currentBuff, countSize, EP_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest.back());
         currentBuff += countSize;
       }
-      std::vector<MPI_Status> status(sendRecvRequest.size());
-      MPI_Waitall(sendRecvRequest.size(), &sendRecvRequest[0], &status[0]);
+      std::vector<ep_lib::MPI_Status> status(sendRecvRequest.size());
+      ep_lib::MPI_Waitall(sendRecvRequest.size(), &sendRecvRequest[0], &status[0]);
 
       dataCurrentDest.resize(*itNbListRecv);
-      const std::vector<bool>& localMaskDest = *itLocalMaskIndexOnDest;
-      for (int i = 0; i < localMaskDest.size(); ++i)
-        if (localMaskDest[i]) dataCurrentDest(i) = 0.0;
-        else dataCurrentDest(i) = defaultValue;
+      dataCurrentDest = 0.0;
 
       std::vector<bool> localInitFlag(dataCurrentDest.numElements(), true);

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.hpp

@@ -141 +141 @@
 
       //! The allocated engines
-
-      static std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> > *engines_ptr;
-      #pragma omp threadprivate(engines_ptr)
+      static std::map<CGridTransformation*, std::shared_ptr<CSpatialTransformFilterEngine> > engines;
   }; // class CSpatialTransformFilterEngine
 } // namespace xios

XIOS/dev/branch_openmp/src/filter/temporal_filter.cpp

@@ -22 +22 @@
                         this->samplingOffset.second, this->samplingOffset.timestep)
     , initDate(initDate)
+//    , nextSamplingDate(initDate + (this->samplingOffset + initDate.getRelCalendar().getTimeStep()))
     , nextSamplingDate(initDate + offsetMonth + ( offsetAllButMonth + initDate.getRelCalendar().getTimeStep()))
     , nbOperationDates(1)