Changeset 2530

Timestamp:

07/19/23 09:07:54 (18 months ago)

Author:

jderouillat

Message:

Enable parallel compression regarding NetCDF & HDF5 configurations at compile time. Implement an API to use HDF5 lossy compression plugins (SZ is available for now).

Location:

XIOS3/branches/xios-3.0-beta

Files:

• make_xios (modified) (1 diff)
• src/config/field_attribute.conf (modified) (1 diff)
• src/io/nc4_data_output.cpp (modified) (1 diff)
• src/io/netCdfInterface.cpp (modified) (2 diffs)
• src/io/netCdfInterface.hpp (modified) (2 diffs)
• src/io/onetcdf4.cpp (modified) (3 diffs)
• src/io/onetcdf4.hpp (modified) (1 diff)
• src/io/onetcdf4_plugin.hpp (added)

XIOS3/branches/xios-3.0-beta/make_xios

@@ -212 +212 @@
 fi 
 
+
+# Detecting if parallel compression is available :
+#   - NetCDF is HDF5 based and >= 4.7.4  ?
+#   - HDF5   is parallel   and >= 1.10.3 ?
+USE_HDF5=0
+if [[ ! -z $NETCDF_INCDIR ]]
+then
+    NETCDF_LOOKUP=`echo ${NETCDF_INCDIR} | sed 's/\-I/\ /g'`
+    netcdf_meta_file=`find $NETCDF_LOOKUP -name netcdf_meta.h 2> /dev/null`
+else
+    for dir in `echo $CPATH | sed 's/\:/\ /g'` ; do
+        if [ -f "$dir/netcdf_meta.h" ]
+        then
+            netcdf_meta_file=$dir/netcdf_meta.h
+            break
+        fi
+    done
+fi
+if [[ ! -z $netcdf_meta_file ]]
+then
+    NC_MAJOR=`grep "NC_VERSION_MAJOR" $netcdf_meta_file | awk '{ print $3 }'`
+    NC_MINOR=`grep "NC_VERSION_MINOR" $netcdf_meta_file | awk '{ print $3 }'`
+    NC_PATCH=`grep "NC_VERSION_PATCH" $netcdf_meta_file | awk '{ print $3 }'`
+    if ( [[ "$NC_MAJOR" -ge 5 || ( "$NC_MAJOR" -eq 4 &&  "$NC_MINOR" -ge 8 )|| ( "$NC_MAJOR" -eq 4 && "$NC_MINOR" -eq 7 && "$NC_PATCH" -ge 4 ) ]] )
+    then
+        USE_HDF5=`grep NC_HAS_HDF5 $netcdf_meta_file | awk '{ print $3 }'`
+    fi
+fi
+
+if [ "$USE_HDF5" == "1" ]
+then
+    # https://forum.hdfgroup.org/t/parallel-compression-support-detection/5398    
+    if [[ ! -z $HDF5_LIBDIR ]]
+    then
+        HDF5_LOOKUP=`echo ${HDF5_LIBDIR} | sed 's/\-L/\ /g'`
+        hdf5_settings_file=`find $HDF5_LOOKUP -name libhdf5.settings 2> /dev/null`
+    else
+        for dir in `echo $LD_LIBRARY_PATH | sed 's/\:/\ /g'` ; do
+            if [ -f "$dir/libhdf5.settings" ]
+            then
+                hdf5_settings_file=$dir/libhdf5.settings
+                break
+            fi
+        done
+    fi
+    HDF5_VERSION=`grep "HDF5 Version" $hdf5_settings_file | awk -F  ':' '{ print $2 }'`
+    HDF5_MAJOR_VERSION=`echo $HDF5_VERSION | awk -F  '.' '{ print $2 }'`
+    HDF5_MINOR_VERSION=`echo $HDF5_VERSION | awk -F  '.' '{ print $3 }'`
+    HDF5_PARALLEL=`grep "Parallel HDF5" $hdf5_settings_file | awk -F  ':' '{ print $2 }'`
+    HDF5_DEFLATE=`grep "I/O filters (external)" $hdf5_settings_file | awk -F  ':' '{ print $2 }'`
+    if ( [[ "$HDF5_MAJOR_VERSION" -ge 11 || ( "$HDF5_MAJOR_VERSION" -eq 10 &&  "$HDF5_MINOR_VERSION" -ge 3 ) ]] )
+    then
+        if [[ "$HDF5_PARALLEL" =~ "yes" && "$HDF5_DEFLATE" =~ "zlib" ]]
+        then 
+            XIOS_CPPKEY="$XIOS_CPPKEY PARALLEL_COMPRESSION"
+        fi
+    fi          
+fi
+if [[ "$XIOS_CPPKEY" =~ "PARALLEL_COMPRESSION" ]]
+then
+    echo "->Parallel compression is available"
+else
+    echo "->Parallel compression is NOT available"    
+fi
+# End "Detecting if parallel compression is available"
+
+
 # Setting path for boost
 if [[ "$use_extern_boost" == "true" ]]

XIOS3/branches/xios-3.0-beta/src/config/field_attribute.conf

@@ -33 +33 @@
 DECLARE_ATTRIBUTE(double,    scale_factor)
 
+DECLARE_ATTRIBUTE(StdString, compression_type)
 DECLARE_ATTRIBUTE(int,       compression_level)
+DECLARE_ARRAY(double , 1,    compression_params)
 DECLARE_ATTRIBUTE(double,    chunking_blocksize_target)
 

XIOS3/branches/xios-3.0-beta/src/io/nc4_data_output.cpp

@@ -1948 +1948 @@
             if (field->compression_level.isEmpty())
               field->compression_level = field->getRelFile()->compression_level.isEmpty() ? 0 : field->getRelFile()->compression_level;
-            SuperClassWriter::setCompressionLevel(fieldid, field->compression_level);
+            else
+              field->compression_type.setValue( "gzip" );
+            if (field->compression_type.isEmpty())
+              field->compression_type = "None";
+            if (field->compression_params.isEmpty())
+              field->compression_params.resize( 0 );
+            SuperClassWriter::setCompressionLevel(fieldid, field->compression_type, field->compression_level, field->compression_params);
 
            {  // Ecriture des coordonnes

XIOS3/branches/xios-3.0-beta/src/io/netCdfInterface.cpp

@@ -11 +11 @@
 #include "netCdfException.hpp"
 #include "mem_checker.hpp"
+#include <string>
+#include <sys/stat.h>
 
 namespace xios
@@ -776 +778 @@
     throw CNetCdfException(e);
   }
-
+  
+  return status;
+}
+
+int CNetCdfInterface::defVarFilter(int ncid, int varId, unsigned int filterId , size_t filterNbrParams, unsigned int* filterParams)
+{
+  const char* cplugin = getenv("HDF5_PLUGIN_PATH") ;
+  bool filterAvailable( false );
+  
+  StdString pluginLibName;
+  if ( filterId == 32017 ) // 32017 = SZ
+  {
+    pluginLibName = "libhdf5sz.so";
+  }
+  if (cplugin)
+  {
+    string plugin( cplugin );
+    string libsz( plugin+"/"+pluginLibName );
+    struct stat sb;
+    if (stat(libsz.c_str(), &sb) == 0) filterAvailable = true;
+  }
+  if (!filterAvailable)
+  {
+    StdStringStream sstr;
+    sstr << "Error when calling function nc_def_var_filter(...)" << std::endl;
+    sstr << "The SZ filter required for " << varId << " is not available."
+         << "Check that $HDF5_PLUGIN_PATH/"<< pluginLibName <<" exists." ;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+ 
+  int status = nc_def_var_filter(ncid, varId, filterId, filterNbrParams, filterParams);
+  if (NC_NOERR != status)
+  {
+    StdString errormsg(nc_strerror(status));
+    StdStringStream sstr;
+    
+    sstr << "Error when calling function nc_def_var_filter(...)" << std::endl;
+    sstr << errormsg << std::endl;
+    sstr << "Unable to set the compression filter of the variable with id: " << varId
+         << " and plugin: " << filterId << std::endl;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+  
   return status;
 }

XIOS3/branches/xios-3.0-beta/src/io/netCdfInterface.hpp

@@ -116 +116 @@
     static int defVarDeflate(int ncid, int varId, int compressionLevel);
 
+    //! Define variable filter
+    static int defVarFilter(int ncid, int varId, unsigned int filterId , size_t filterNbrParams, unsigned int* filterParams );
+
     //! Set or unset the fill mode
     static int setFill(int ncid, bool fill);
@@ -121 +124 @@
     //! Define variable fill parameters
     static int defVarFill(int ncid, int varId, int noFill, void* fillValue);
-
 
     //! Change access type of a variable

XIOS3/branches/xios-3.0-beta/src/io/onetcdf4.cpp

@@ -2 +2 @@
 
 #include "onetcdf4.hpp"
+#include "onetcdf4_plugin.hpp"
 #include "group_template.hpp"
 #include "mpi.hpp"
@@ -505 +506 @@
             CNetCdfInterface::defVarFill(grpid, varid, true, NULL);
          }
-
-         setCompressionLevel(name, compressionLevel) ;
          
          return varid;
@@ -513 +512 @@
       //---------------------------------------------------------------
 
-      void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)
+      void CONetCDF4::setCompressionLevel(const StdString& varname, const StdString& compressionType, int compressionLevel, const CArray<double,1>& compressionParams)
       {
          if (compressionLevel < 0 || compressionLevel > 9)
            ERROR("void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)",
                  "Invalid compression level for variable \"" << varname << "\", the value should range between 0 and 9.");
-         if (compressionLevel && wmpi)
+#ifndef PARALLEL_COMPRESSION
+         if ( ((compressionLevel)||(compressionParams.numElements())) && wmpi)
            ERROR("void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)",
                  "Impossible to use compression for variable \"" << varname << "\" when using parallel mode.");
+#endif
          int grpid = this->getCurrentGroup();
          int varid = this->getVariable(varname);
-         CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
+         if (compressionType=="None")
+         {
+         }
+         else if (compressionType=="gzip")
+         {
+           CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
+         }
+         else
+         {
+           size_t cd_nelmts;
+           unsigned int* cd_values = NULL;
+           if (compressionType=="SZ")
+           {
+             CONetCDF4Plugin::interpretParametersSZ(compressionParams, &cd_nelmts, &cd_values);
+             CNetCdfInterface::defVarFilter(grpid, varid, 32017, cd_nelmts, cd_values);
+           }
+           else
+           {
+               ERROR("void CONetCDF4::setCompressionLevel(...)", "compression_type = " << compressionType << " is not managed");
+           }
+           if (cd_values!=NULL) delete [] cd_values;
+         }
+                      
       }
 

XIOS3/branches/xios-3.0-beta/src/io/onetcdf4.hpp

@@ -59 +59 @@
                void setDefaultValue(const StdString& varname, const T* value = NULL);
 
-            void setCompressionLevel(const StdString& varname, int compressionLevel);
+            void setCompressionLevel(const StdString& varname, const StdString& compressionType, int compressionLevel, 
+                                     const CArray<double,1>& compressionParam );
 
             template <class T>  void addAttribute (const StdString& name, const T& value, const StdString* varname = NULL);