Changeset 606 for XIOS/trunk

Timestamp:

06/03/15 15:06:25 (9 years ago)

Author:

rlacroix

Message:

Support NetCDF4 compression.

Only available for non-parallel output so either if only one server is used or if the multiple file mode is enabled).

Location:

XIOS/trunk/src

Files:

• config/field_attribute.conf (modified) (1 diff)
• config/file_attribute.conf (modified) (1 diff)
• output/nc4_data_output.cpp (modified) (1 diff)
• output/netCdfInterface.cpp (modified) (1 diff)
• output/netCdfInterface.hpp (modified) (1 diff)
• output/onetcdf4.cpp (modified) (5 diffs)
• output/onetcdf4.hpp (modified) (5 diffs)

XIOS/trunk/src/config/field_attribute.conf

@@ -25 +25 @@
 DECLARE_ATTRIBUTE(double,    add_offset)
 DECLARE_ATTRIBUTE(double,    scale_factor)
+
+DECLARE_ATTRIBUTE(int,       compression_level)

XIOS/trunk/src/config/file_attribute.conf

@@ -17 +17 @@
 DECLARE_ENUM2(mode, read, write)
 
-// DECLARE_ATTRIBUTE_DEF(bool, enabled, true)
+DECLARE_ATTRIBUTE(int,       compression_level)

XIOS/trunk/src/output/nc4_data_output.cpp

@@ -756 +756 @@
               SuperClassWriter::setDefaultValue(fieldid, (double*)NULL);
 
+            if (field->compression_level.isEmpty())
+              field->compression_level = field->file->compression_level.isEmpty() ? 0 : field->file->compression_level;
+            if (field->compression_level < 0 || field->compression_level > 9)
+              ERROR("void CNc4DataOutput::writeField_(CField* field)",
+                    "Invalid compression level, the value should range between 0 and 9.");
+            SuperClassWriter::setCompressionLevel(fieldid, field->compression_level);
+
            {  // Ecriture des coordonnées
 

XIOS/trunk/src/output/netCdfInterface.cpp

@@ -525 +525 @@
 
 /*!
+This function sets the compression level to the specified variable
+\param [in] ncid Groud id (or file id)
+\param [in] varId Id of the variable
+\param [in] compressionLevel The compression level from 0 to 9 (0 disables the compression, 9 is the higher compression)
+\return Status code
+*/
+int CNetCdfInterface::defVarDeflate(int ncid, int varId, int compressionLevel)
+{
+  int status = nc_def_var_deflate(ncid, varId, false, (compressionLevel > 0), compressionLevel);
+  if (NC_NOERR != status)
+  {
+    StdString errormsg(nc_strerror(status));
+    StdStringStream sstr;
+
+    sstr << "Error in calling function " << "nc_def_var_deflate(ncid, varId, false, (compressionLevel > 0), compressionLevel)" << std::endl;
+    sstr << errormsg << std::endl;
+    sstr << "Unable to set the compression level of the variable with id: " << varId
+         << " and compression level: " << compressionLevel << std::endl;
+    StdString e = sstr.str();
+    throw CNetCdfException(e);
+  }
+
+  return status;
+}
+
+/*!
 Set or unset the fill mode for a NetCDF file specified by its file id.
 \param [in] ncid File id

XIOS/trunk/src/output/netCdfInterface.hpp

@@ -90 +90 @@
     static int defVarChunking(int ncid, int varId, int storage, StdSize chunkSize[]);
 
+    //! Define variable compression level
+    static int defVarDeflate(int ncid, int varId, int compressionLevel);
+
     //! Set or unset the fill mode
     static int setFill(int ncid, bool fill);

XIOS/trunk/src/output/onetcdf4.cpp

@@ -84 +84 @@
          // This is done per variable for the NetCDF4 format.
          if (useClassicFormat)
-            CNetCdfInterface::setFill(this->ncidp, false);    
+            CNetCdfInterface::setFill(this->ncidp, false);
       }
 
@@ -277 +277 @@
 
       int CONetCDF4::addVariable(const StdString & name, nc_type type,
-                                  const std::vector<StdString> & dim)
+                                 const std::vector<StdString> & dim)
       {
          int varid = 0;
          std::vector<int> dimids;
-         std::vector<StdSize> dimsizes ;
-         StdSize size ;
-         StdSize totalSize ;
-         StdSize maxSize=1024*1024*256 ; // == 2GB/8 if output double
-
-         int grpid = this->getCurrentGroup();
-
-         std::vector<StdString>::const_iterator
-            it  = dim.begin(), end = dim.end();
-
-         for (;it != end; it++)
+         std::vector<StdSize> dimsizes;
+         StdSize size;
+         StdSize totalSize;
+         StdSize maxSize = 1024 * 1024 * 256; // == 2GB/8 if output double
+
+         int grpid = this->getCurrentGroup();
+
+         std::vector<StdString>::const_iterator it = dim.begin(), end = dim.end();
+
+         for (; it != end; it++)
          {
             const StdString & dimid = *it;
             dimids.push_back(this->getDimension(dimid));
-            (CNetCdfInterface::inqDimLen(grpid, this->getDimension(dimid), size));
-            if (size==NC_UNLIMITED) size=1 ;
-            dimsizes.push_back(size) ;
-         }
-
-         CNetCdfInterface::defVar(grpid, name, type, dimids.size(), &(dimids[0]), varid);
+            CNetCdfInterface::inqDimLen(grpid, this->getDimension(dimid), size);
+            if (size == NC_UNLIMITED) size = 1;
+            dimsizes.push_back(size);
+         }
+
+         CNetCdfInterface::defVar(grpid, name, type, dimids.size(), &dimids[0], varid);
 
          // The classic NetCDF format does not support chunking nor fill parameters
@@ -318 +317 @@
          }
 
-         return (varid);
+         return varid;
+      }
+
+      //---------------------------------------------------------------
+
+      void CONetCDF4::setCompressionLevel(const StdString& varname, int compressionLevel)
+      {
+         int grpid = this->getCurrentGroup();
+         int varid = this->getVariable(varname);
+         CNetCdfInterface::defVarDeflate(grpid, varid, compressionLevel);
       }
 
@@ -449 +457 @@
          {
             sstart.push_back(record + recordOffset);
-            scount.push_back(1); 
+            scount.push_back(1);
             if ((start == NULL) &&
                 (count == NULL)) i++;
@@ -456 +464 @@
 
          for (;it != end; it++)
-         { 
+         {
             if ((start != NULL) && (count != NULL))
             {

XIOS/trunk/src/output/onetcdf4.hpp

@@ -48 +48 @@
             int addVariable(const StdString & name, nc_type type,
                             const std::vector<StdString> & dim);
-                            
+
       //----------------------------------------------------------------
          public :
-         
+
             template <class T>
                void setDefaultValue(const StdString & varname, const T * value = NULL);
-         
+
+            void setCompressionLevel(const StdString& varname, int compressionLevel);
+
             template <class T>  void addAttribute (const StdString & name, const T & value, const StdString * varname = NULL);
 
@@ -64 +66 @@
                               const std::vector<StdSize> * count = NULL);
 
-            void writeData(const CArray<int, 2>& data, const StdString & name);     
+            void writeData(const CArray<int, 2>& data, const StdString & name);
             void writeTimeAxisData(const CArray<double,1>& data, const StdString & name,
                                    bool collective, StdSize record, bool Isroot) ;
@@ -72 +74 @@
             /// Destructeur ///
             virtual ~CONetCDF4(void);
-            
+
       //----------------------------------------------------------------
-      
+
          protected :
 
@@ -97 +99 @@
 
       //----------------------------------------------------------------
-      
+
          private :
-         
+
             template <class T>
                void writeData_(int grpid, int varid,
@@ -125 +127 @@
 
       ///---------------------------------------------------------------
-           
+