Changeset 2386

Timestamp:

07/26/22 15:26:52 (23 months ago)

Author:

jderouillat

Message:

Set the code structure to compute the hash value of an element based on its attributs, use for now before writing an element in a file

Location:

XIOS3/trunk/src

Files:

• attribute.hpp (modified) (1 diff)
• attribute_array.hpp (modified) (1 diff)
• attribute_array_impl.hpp (modified) (1 diff)
• attribute_enum.hpp (modified) (1 diff)
• attribute_enum_impl.hpp (modified) (1 diff)
• attribute_map.cpp (modified) (1 diff)
• attribute_map.hpp (modified) (1 diff)
• attribute_template.hpp (modified) (1 diff)
• attribute_template_impl.hpp (modified) (1 diff)
• io/nc4_data_output.cpp (modified) (2 diffs)
• node/axis.cpp (modified) (1 diff)
• node/axis.hpp (modified) (1 diff)
• node/domain.cpp (modified) (1 diff)
• node/domain.hpp (modified) (1 diff)

XIOS3/trunk/src/attribute.hpp

@@ -44 +44 @@
             virtual StdString dumpGraph(void) const = 0;
             virtual bool isEqual(const CAttribute& ) = 0;
+            virtual int computeHash(void) = 0;
 
             //! Returns true if and only if the attribute should be publicly exposed in the API

XIOS3/trunk/src/attribute_array.hpp

@@ -44 +44 @@
                         
             bool isEqual(const CAttribute& attr);
+            int computeHash(void);
 
             /// Destructeur ///

XIOS3/trunk/src/attribute_array_impl.hpp

@@ -117 +117 @@
       else return false;  
     }
+  
+    template <typename T_numtype, int N_rank>
+    int CAttributeArray<T_numtype,N_rank>::computeHash(void)
+    {
+      ERROR("CAttributeArray::computeHash(void)",
+       << "NOT IMPLEMENTED FOR NOW !");
+      return std::hash<string>{}( toString() );
+    }
 
     template <typename T_numtype, int N_rank>

XIOS3/trunk/src/attribute_enum.hpp

@@ -52 +52 @@
             
             bool isEqual(const CAttribute& attr );
+            int computeHash(void);
 
             /// Destructeur ///

XIOS3/trunk/src/attribute_enum_impl.hpp

@@ -126 +126 @@
   }
 
+  template <class T>
+  int CAttributeEnum<T>::computeHash(void)
+  {
+    // Basic hash computation through string
+    return std::hash<string>{}( toString() );
+  }
+
   //---------------------------------------------------------------
 

XIOS3/trunk/src/attribute_map.cpp

@@ -269 +269 @@
       }
 
+      /*!
+        Compute the hash value of current attribute map excluding some attributs.
+        \param [in] excludedAttrs attribute to be excluded from hash computation
+      */
+      int CAttributeMap::computeGlobalAttributesHash(const vector<StdString>& excludedAttrs)
+      {
+         int attrs_hash( 0 );
+         typedef std::pair<StdString, CAttribute*> StdStrAttPair;
+
+         SuperClassMap::const_iterator it = this->begin(), end = this->end();
+         for (; it != end; it++)
+         {
+            const StdStrAttPair& el = *it;
+            if ( std::find( excludedAttrs.begin(), excludedAttrs.end(), el.first ) == excludedAttrs.end() )
+            {
+              if (this->hasAttribute(el.first))
+              {
+                if (!el.second->isEmpty())
+                {
+                  attrs_hash += el.second->computeHash();
+                }
+              }
+            }
+         }
+         return attrs_hash;
+      }
+  
       //---------------------------------------------------------------
 /*

XIOS3/trunk/src/attribute_map.hpp

@@ -37 +37 @@
             void setAttributes(const CAttributeMap * const _parent, bool apply=true);
             void duplicateAttributes(const CAttributeMap* const _parent);
+            int computeGlobalAttributesHash(const vector<StdString>& excludedAttrs);
             void clearAllAttributes(void);
             StdString dumpXiosAttributes(void) const;

XIOS3/trunk/src/attribute_template.hpp

@@ -61 +61 @@
             
             bool isEqual(const CAttribute& attr );
+            int computeHash(void);
 
             /// Destructeur ///

XIOS3/trunk/src/attribute_template_impl.hpp

@@ -168 +168 @@
       if (!this->isEmpty() && !attr.isEmpty())  return (this->getValue() == attr.getValue());
       else return false;
+    }
+
+    template <class T>
+    int CAttributeTemplate<T>::computeHash(void)
+    {
+      // Basic hash computation through string
+      return std::hash<string>{}( toString() );
     }
 

XIOS3/trunk/src/io/nc4_data_output.cpp

@@ -78 +78 @@
          if (!domain->lonvalue.isEmpty() )
          {
-           int globalSize = domain->ni_glo.getValue()*domain->nj_glo.getValue();
-           CArray<size_t,1> globalIndex; // No redundancy globalIndex will be computed with the connector
-           shared_ptr<CGridTransformConnector> gridTransformConnector;
-           // Compute a without redundancy element FULL view to enable a consistent hash computation
-           domain->getLocalView(CElementView::FULL)->createWithoutRedundancyFullViewConnector( globalSize, comm_file, gridTransformConnector, globalIndex );
-           int localSize = globalIndex.numElements();
-           
-           CArray<double,1> lon_distributedValue, lat_distributedValue ;
-           gridTransformConnector->transfer(domain->lonvalue, lon_distributedValue );
-           gridTransformConnector->transfer(domain->latvalue, lat_distributedValue );
-
-           // Compute the distributed hash (v0) of the element
-           // it will be associated to the default element name (= map key), and to the name really written
-           int localHash = 0;
-           for (int iloc=0; iloc<localSize ; iloc++ ) localHash+=globalIndex(iloc)*lon_distributedValue(iloc)*lat_distributedValue(iloc);
-           int globalHash(0);
-           MPI_Allreduce( &localHash, &globalHash, 1, MPI_INT, MPI_SUM, comm_file  );
-         
+           // The hash of the element will be associated to the default element name (= map key), and to the name really written
+           int globalHash = domain->computeAttributesHash( comm_file ); // Need a MPI_Comm to distribute without redundancy some attributs (value)
+
            StdString defaultNameKey = domain->getDomainOutputName();
            if ( !relDomains_.count ( defaultNameKey ) )
@@ -1056 +1041 @@
         if (!axis->value.isEmpty() )
         {
-           int globalSize = axis->n_glo.getValue();
-           CArray<size_t,1> globalIndex; // No redundancy globalIndex will be computed with the connector
-           shared_ptr<CGridTransformConnector> gridTransformConnector;
-           // Compute a without redundancy element FULL view to enable a consistent hash computation
-           axis->getLocalView(CElementView::FULL)->createWithoutRedundancyFullViewConnector( globalSize, comm_file, gridTransformConnector, globalIndex );
-           int localSize = globalIndex.numElements();
-
-          CArray<double,1> distributedValue ;
-          gridTransformConnector->transfer(axis->value, distributedValue );
-        
-          // Compute the distributed hash (v0) of the element
-          // it will be associated to the default element name (= map key), and to the name really written
-          int localHash = 0;
-          for (int iloc=0; iloc<localSize ; iloc++ ) localHash+=globalIndex(iloc)*distributedValue(iloc);
-          int globalHash(0);
-          MPI_Allreduce( &localHash, &globalHash, 1, MPI_INT, MPI_SUM, comm_file  );
+          // The hash of the element will be associated to the default element name (= map key), and to the name really written
+          int globalHash = axis->computeAttributesHash( comm_file ); // Need a MPI_Comm to distribute without redundancy some attributs (value)
 
           StdString defaultNameKey = axis->getAxisOutputName();

XIOS3/trunk/src/node/axis.cpp

@@ -302 +302 @@
      bounds.reset();
      label.reset() ;
+   }
+
+   int CAxis::computeAttributesHash( MPI_Comm comm )
+   {
+     int axis_hash = 0;
+     
+     // Compute the hash of distributed attributs (value ...)
+     int globalSize = this->n_glo.getValue();
+     CArray<size_t,1> globalIndex; // No redundancy globalIndex will be computed with the connector
+     shared_ptr<CGridTransformConnector> gridTransformConnector;
+     // Compute a without redundancy element FULL view to enable a consistent hash computation (and a distributed globalIndex)
+     this->getLocalView(CElementView::FULL)->createWithoutRedundancyFullViewConnector( globalSize, comm, gridTransformConnector, globalIndex );
+     int localSize = globalIndex.numElements();
+
+     CArray<double,1> distributedValue ;
+     gridTransformConnector->transfer(this->value, distributedValue );
+        
+     int localHash = 0;
+     for (int iloc=0; iloc<localSize ; iloc++ ) localHash+=globalIndex(iloc)*distributedValue(iloc);
+     int distributedHash = 0;
+     MPI_Allreduce( &localHash, &distributedHash, 1, MPI_INT, MPI_SUM, comm  );
+
+     // Compute the hash of global attributs (unit, prec ...)
+     vector<StdString> excludedAttr;
+     //excludedAttr.push_back("name");
+     // internal attributs
+     excludedAttr.insert(excludedAttr.end(), { "index", "data_n", "data_begin", "data_index"  });
+     excludedAttr.insert(excludedAttr.end(), { "begin", "n" });     
+     excludedAttr.push_back("axis_ref");
+     // in distributed
+     excludedAttr.push_back("value");
+     // should be considered in distributed
+     excludedAttr.push_back("bounds");
+     excludedAttr.push_back("label");
+     excludedAttr.push_back("mask"); // ???
+
+     int globalHash = this->computeGlobalAttributesHash( excludedAttr );
+
+     return distributedHash + globalHash;
    }
 

XIOS3/trunk/src/node/axis.hpp

@@ -130 +130 @@
          void resetGeometricAttributes(void) ;
 
+         int computeAttributesHash( MPI_Comm comm );
+
          size_t getGlobalWrittenSize(void) ;
 

XIOS3/trunk/src/node/domain.cpp

@@ -1770 +1770 @@
    CATCH_DUMP_ATTR
 
+   int CDomain::computeAttributesHash( MPI_Comm comm )
+   {
+     int domain_hash = 0;
+     
+     // Compute the hash of distributed attributs (value ...)
+     int globalSize = this->ni_glo.getValue()*this->nj_glo.getValue();
+     CArray<size_t,1> globalIndex; // No redundancy globalIndex will be computed with the connector
+     shared_ptr<CGridTransformConnector> gridTransformConnector;
+     // Compute a without redundancy element FULL view to enable a consistent hash computation
+     this->getLocalView(CElementView::FULL)->createWithoutRedundancyFullViewConnector( globalSize, comm, gridTransformConnector, globalIndex );
+     int localSize = globalIndex.numElements();
+           
+     CArray<double,1> lon_distributedValue, lat_distributedValue ;
+     gridTransformConnector->transfer(this->lonvalue, lon_distributedValue );
+     gridTransformConnector->transfer(this->latvalue, lat_distributedValue );
+
+     // Compute the distributed hash (v0) of the element
+     // it will be associated to the default element name (= map key), and to the name really written
+     int localHash = 0;
+     for (int iloc=0; iloc<localSize ; iloc++ ) localHash+=globalIndex(iloc)*lon_distributedValue(iloc)*lat_distributedValue(iloc);
+     int distributedHash = 0;
+     MPI_Allreduce( &localHash, &distributedHash, 1, MPI_INT, MPI_SUM, comm  );
+     
+     // Compute the hash of global attributs (unit, prec ...)
+     vector<StdString> excludedAttr;
+     //excludedAttr.push_back("name");
+     // internal attributs
+     excludedAttr.insert(excludedAttr.end(), { "ibegin", "jbegin", "ni", "nj", "i_index", "j_index" });
+     excludedAttr.insert(excludedAttr.end(), { "data_ni", "data_nj", "data_ibegin", "data_jbegin" });
+     excludedAttr.insert(excludedAttr.end(), { "data_i_index", "data_j_index", "domain_ref" });
+     // in distributed through lonvalue and latvalue
+     excludedAttr.insert(excludedAttr.end(), { "lonvalue_1d", "latvalue_1d", "lonvalue_2d", "latvalue_2d" });
+     // should be considered in distributed
+     excludedAttr.insert(excludedAttr.end(), { "mask_1d", "mask_2d" }); // ???
+     excludedAttr.insert(excludedAttr.end(), { "bounds_lon_1d", "bounds_lat_1d", "bounds_lon_2d", "bounds_lat_2d" });
+     excludedAttr.insert(excludedAttr.end(), { "area" });
+     // private
+     excludedAttr.insert(excludedAttr.end(), { "lon_start", "lon_end", "lat_start", "lat_end" });
+     excludedAttr.insert(excludedAttr.end(), { "bounds_lon_start", "bounds_lon_end", "bounds_lat_start", "bounds_lat_end" });
+     excludedAttr.insert(excludedAttr.end(), { "has_lat_in_read_file", "has_lon_in_read_file", "has_bounds_lat_in_read_file", "has_bounds_lon_in_read_file" });
+     excludedAttr.insert(excludedAttr.end(), { "lonvalue_rectilinear_read_from_file", "latvalue_rectilinear_read_from_file", "lonvalue_curvilinear_read_from_file", "latvalue_curvilinear_read_from_file" });
+     excludedAttr.insert(excludedAttr.end(), { "bounds_lonvalue_curvilinear_read_from_file", "bounds_latvalue_curvilinear_read_from_file", "lonvalue_unstructured_read_from_file", "latvalue_unstructured_read_from_file" });
+     excludedAttr.insert(excludedAttr.end(), { "bounds_lonvalue_unstructured_read_from_file", "bounds_latvalue_unstructured_read_from_file" });
+     
+     int globalHash = this->computeGlobalAttributesHash( excludedAttr );
+
+     return distributedHash + globalHash;
+   }
 
    void CDomain::initializeLocalElement(void)

XIOS3/trunk/src/node/domain.hpp

@@ -91 +91 @@
          bool checkAttributes_done_ = false ;
 
+         int computeAttributesHash( MPI_Comm comm );
          
       //////////////////////////