Changeset 2529 for XIOS3/trunk/src

Timestamp:

07/18/23 15:12:50 (12 months ago)

Author:

jderouillat

Message:

Enable parallel compression regarding NetCDF & HDF5 configurations at compile time. Implement an API to use HDF5 lossy compression plugins (SZ is available for now).

Location:

XIOS3/trunk/src

Files:

• config/field_attribute.conf (modified) (1 diff)
• io/nc4_data_output.cpp (modified) (1 diff)
• io/netCdfInterface.cpp (modified) (2 diffs)
• io/netCdfInterface.hpp (modified) (2 diffs)
• io/onetcdf4.cpp (modified) (3 diffs)
• io/onetcdf4.hpp (modified) (1 diff)
• io/onetcdf4_plugin.hpp (added)

XIOS3/trunk/src/config/field_attribute.conf

@@ -33 +33 @@
 DECLARE_ATTRIBUTE(double,    scale_factor)
 
+DECLARE_ATTRIBUTE(StdString, compression_type)
 DECLARE_ATTRIBUTE(int,       compression_level)
+DECLARE_ARRAY(double , 1,    compression_params)
 DECLARE_ATTRIBUTE(double,    chunking_blocksize_target)
 

XIOS3/trunk/src/io/nc4_data_output.cpp

@@ -1948 +1948 @@
             if (field->compression_level.isEmpty())
               field->compression_level = field->getRelFile()->compression_level.isEmpty() ? 0 : field->getRelFile()->compression_level;
-            SuperClassWriter::setCompressionLevel(fieldid, field->compression_level);
+            else
+              field->compression_type.setValue( "gzip" );
+            if (field->compression_type.isEmpty())
+              field->compression_type = "None";
+            if (field->compression_params.isEmpty())
+              field->compression_params.resize( 0 );
+            SuperClassWriter::setCompressionLevel(fieldid, field->compression_type, field->compression_level, field->compression_params);
 
            {  // Ecriture des coordonnes

XIOS3/trunk/src/io/netCdfInterface.cpp

@@ -11 +11 @@
 #include "netCdfException.hpp"
 #include "mem_checker.hpp"
+#include <string>
+#include <sys/stat.h>
 
 namespace xios
@@ -776 +778 @@
     throw CNetCdfException(e);
   }
-
+  
+  return status;
+}
+
+int CNetCdfInterface::defVarFilter(int ncid, int varId, unsigned int filterId , size_t filterNbrParams, unsigned int* filterParams)
+{
+  const char* cplugin = getenv("HDF5_PLUGIN_PATH") ;
+  bool filterAvailable( false );
+  
+  StdString pluginLibName;
+  if ( filterId == 32017 ) // 32017 = SZ
+  {
+    pluginLibName = "libhdf5sz.so";
+  }
+  if (cplugin)
+  {
+    string plugin( cplugin );
+    string libsz( plugin+"/"+pluginLibName );
+    struct stat sb;
+    if (stat(libsz.c_str(), &sb) == 0) filterAvailable = true;
+  }
+  if (!filterAvailable)
+  {
+    StdStringStream sstr;
+    sstr << "Error when calling function nc_def_var_filter(...)" << std::endl;
+    sstr << "The SZ filter required for " << varId << " is not available."
+         << "Check that $HDF5_PLUGIN_PATH/"<< pluginLibName <<" exists." ;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+ 
+  int status = nc_def_var_filter(ncid, varId, filterId, filterNbrParams, filterParams);
+  if (NC_NOERR != status)
+  {
+    StdString errormsg(nc_strerror(status));
+    StdStringStream sstr;
+    
+    sstr << "Error when calling function nc_def_var_filter(...)" << std::endl;
+    sstr << errormsg << std::endl;
+    sstr << "Unable to set the compression filter of the variable with id: " << varId
+         << " and plugin: " << filterId << std::endl;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+  
   return status;
 }

XIOS3/trunk/src/io/netCdfInterface.hpp

@@ -116 +116 @@
     static int defVarDeflate(int ncid, int varId, int compressionLevel);
 
+    //! Define variable filter
+    static int defVarFilter(int ncid, int varId, unsigned int filterId , size_t filterNbrParams, unsigned int* filterParams );
+
     //! Set or unset the fill mode
     static int setFill(int ncid, bool fill);
@@ -121 +124 @@
     //! Define variable fill parameters
     static int defVarFill(int ncid, int varId, int noFill, void* fillValue);
-
 
     //! Change access type of a variable

XIOS3/trunk/src/io/onetcdf4.cpp

@@ -2 +2 @@
 
 #include "onetcdf4.hpp"
+#include "onetcdf4_plugin.hpp"
 #include "group_template.hpp"
 #include "mpi.hpp"
@@ -505 +506 @@
             CNetCdfInterface::defVarFill(grpid, varid, true, NULL);
          }
-
-         setCompressionLevel(name, compressionLevel) ;
          
          return varid;
@@ -513 +512 @@
       //---------------------------------------------------------------
 
-      void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)
+      void CONetCDF4::setCompressionLevel(const StdString& varname, const StdString& compressionType, int compressionLevel, const CArray<double,1>& compressionParams)
       {
          if (compressionLevel < 0 || compressionLevel > 9)
            ERROR("void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)",
                  "Invalid compression level for variable \"" << varname << "\", the value should range between 0 and 9.");
-         if (compressionLevel && wmpi)
+#ifndef PARALLEL_COMPRESSION
+         if ( ((compressionLevel)||(compressionParams.numElements())) && wmpi)
            ERROR("void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)",
                  "Impossible to use compression for variable \"" << varname << "\" when using parallel mode.");
+#endif
          int grpid = this->getCurrentGroup();
          int varid = this->getVariable(varname);
-         CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
+         if (compressionType=="None")
+         {
+         }
+         else if (compressionType=="gzip")
+         {
+           CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
+         }
+         else
+         {
+           size_t cd_nelmts;
+           unsigned int* cd_values = NULL;
+           if (compressionType=="SZ")
+           {
+             CONetCDF4Plugin::interpretParametersSZ(compressionParams, &cd_nelmts, &cd_values);
+             CNetCdfInterface::defVarFilter(grpid, varid, 32017, cd_nelmts, cd_values);
+           }
+           else
+           {
+               ERROR("void CONetCDF4::setCompressionLevel(...)", "compression_type = " << compressionType << " is not managed");
+           }
+           if (cd_values!=NULL) delete [] cd_values;
+         }
+                      
       }
 

XIOS3/trunk/src/io/onetcdf4.hpp

@@ -59 +59 @@
                void setDefaultValue(const StdString& varname, const T* value = NULL);
 
-            void setCompressionLevel(const StdString& varname, int compressionLevel);
+            void setCompressionLevel(const StdString& varname, const StdString& compressionType, int compressionLevel, 
+                                     const CArray<double,1>& compressionParam );
 
             template <class T>  void addAttribute (const StdString& name, const T& value, const StdString* varname = NULL);