Changeset 2598

Timestamp:

11/20/23 13:05:46 (6 months ago)

Author:

jderouillat

Message:

Add a field attribute, conversion_by_NetCDF, to operate type conversion in XIOS, and not in NetCDF

Location:

XIOS3/branches/xios-3.0-beta/src

Files:

• config/field_attribute.conf (modified) (1 diff)
• io/nc4_data_output.cpp (modified) (4 diffs)
• io/nc4_data_output.hpp (modified) (1 diff)
• io/netCdfInterface.cpp (modified) (2 diffs)
• io/netCdfInterface_decl.cpp (modified) (1 diff)
• io/onetcdf4.cpp (modified) (1 diff)
• io/onetcdf4_decl.cpp (modified) (1 diff)

XIOS3/branches/xios-3.0-beta/src/config/field_attribute.conf

@@ -13 +13 @@
 DECLARE_ATTRIBUTE(int,       level)
 DECLARE_ATTRIBUTE(int,       prec)
+DECLARE_ATTRIBUTE(bool,      conversion_by_NetCDF)
 DECLARE_ATTRIBUTE(StdString, expr, false)
 

XIOS3/branches/xios-3.0-beta/src/io/nc4_data_output.cpp

@@ -12 +12 @@
 #include "timer.hpp"
 #include "uuid.hpp"
+
+#include <limits.h>
+#define X_FLOAT_MAX     FLT_MAX
+#define X_FLOAT_MIN     FLT_MIN
+#define X_SHORT_MAX     SHRT_MAX
+#define X_SHORT_MIN     SHRT_MIN
+
 namespace xios
 {
@@ -2265 +2272 @@
               {
                  CTimer::get("Files : writing data").resume();
-                 SuperClassWriter::writeData(data, fieldid, isCollective, nstep - 1);
+                 writeAndConvertData(field, data, nstep - 1);
                  CTimer::get("Files : writing data").suspend();
                  if (wtime)
@@ -2429 +2436 @@
 
                 CTimer::get("Files : writing data").resume();
-                SuperClassWriter::writeData(data, fieldid, isCollective, nstep - 1, &start, &count);
+                writeAndConvertData(field, data, nstep - 1, &start, &count);
                 CTimer::get("Files : writing data").suspend();
 
@@ -2463 +2470 @@
            ERROR("CNc4DataOutput::writeFieldData_ (CField*  field)", << msg);
          }
+      }
+
+      void CNc4DataOutput::writeAndConvertData(CField* field, const CArray<double,1>& data, StdSize record, const std::vector<StdSize> *start, const std::vector<StdSize> *count)
+      {
+        StdString fieldid = field->getFieldOutputName();
+        nc_type type ;
+        if (field->prec.isEmpty()) type =  NC_FLOAT ;
+        else
+        {
+          if (field->prec==2) type = NC_SHORT ;
+          else if (field->prec==4)  type =  NC_FLOAT ;
+          else if (field->prec==8)   type =  NC_DOUBLE ;
+        }
+
+        bool conversionByNetCDF = true; // default : conversion operated by NetCDF for now (legacy behaviour)
+        if (!field->conversion_by_NetCDF.isEmpty())
+        {
+          // use conversion_by_NetCDF = ".false." to  bypass NetCDF conversion
+          //   poor performances from NC_DOUBLE to NC_FLOAT with isgreater/isless in recent NetCDF available at TGCC
+          conversionByNetCDF = field->conversion_by_NetCDF;
+        }
+        if ( ( type == NC_DOUBLE ) || ( conversionByNetCDF ) )
+        {
+          SuperClassWriter::writeData(data, fieldid, isCollective, record, start, count);
+        }
+        else if ( type == NC_FLOAT )
+        {
+          CArray<float,1> f_data;
+          f_data.resize( data.numElements() );
+          for (int i = 0; i < data.numElements(); i++) 
+          {
+            if (data(i) <= X_FLOAT_MAX || data(i) >= X_FLOAT_MIN)
+              f_data(i) = data(i);
+            else if (data(i) > X_FLOAT_MAX)
+              f_data(i) = X_FLOAT_MAX;
+            else if (data(i) < X_FLOAT_MIN)
+              f_data(i) = X_FLOAT_MIN;
+          }
+          SuperClassWriter::writeData(f_data, fieldid, isCollective, record, start, count);
+        }
+        else if ( type == NC_SHORT )
+        {
+          CArray<short,1> s_data;
+          s_data.resize( data.numElements() );
+          for (int i = 0; i < data.numElements(); i++) 
+          {
+            if (data(i) <= X_SHORT_MAX || data(i) >= X_SHORT_MIN)
+              s_data(i) = data(i);
+            else if (data(i) > X_SHORT_MAX)
+              s_data(i) = X_SHORT_MAX;
+            else if (data(i) < X_SHORT_MIN)
+              s_data(i) = X_SHORT_MIN;
+          }
+          SuperClassWriter::writeData(s_data, fieldid, isCollective, record, start, count);
+        }
+        else
+        {
+            ERROR("CNc4DataOutput::writeAndConvertData(...)", << "Type conversion not managed for " << fieldid ); 
+        }
+        
       }
 

XIOS3/branches/xios-3.0-beta/src/io/nc4_data_output.hpp

@@ -57 +57 @@
             virtual void writeAttribute_(CVariable* var, const string& fieldId);
             virtual int  writeFieldData_(CField* field, const CArray<double,1>& data, const CDate& lastWrite, const CDate& currentWrite, int nstep);
+            void writeAndConvertData(CField* field, const CArray<double,1>& data, StdSize record, const std::vector<StdSize> *start = NULL, const std::vector<StdSize> *count = NULL);
             virtual void writeFile_     (CFile* file);
             virtual void closeFile_     (void);

XIOS3/branches/xios-3.0-beta/src/io/netCdfInterface.cpp

@@ -1054 +1054 @@
 
 template<>
+int CNetCdfInterface::ncGetVaraType(int ncid, int varid, const StdSize* start, const StdSize* count, short* data)
+{
+  return nc_get_vara_short(ncid, varid, start, count, data);
+}
+
+template<>
 int CNetCdfInterface::ncGetVaraType(int ncid, int varid, const StdSize* start, const StdSize* count, size_t* data)
 {
@@ -1082 +1088 @@
 {
   return nc_put_vara_int(ncid, varid, start, count, data);
+}
+
+template<>
+int CNetCdfInterface::ncPutVaraType(int ncid, int varid, const StdSize* start, const StdSize* count, const short* data)
+{
+  return nc_put_vara_short(ncid, varid, start, count, data);
 }
 

XIOS3/branches/xios-3.0-beta/src/io/netCdfInterface_decl.cpp

@@ -34 +34 @@
   macroPutVar(float)
   macroPutVar(int)
+  macroPutVar(short)
   macroPutVar(size_t)
   macroPutVar(char)

XIOS3/branches/xios-3.0-beta/src/io/onetcdf4.cpp

@@ -735 +735 @@
           CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
       }
+      
+      template <>
+      void CONetCDF4::writeData_(int grpid, int varid,
+                                 const std::vector<StdSize>& sstart,
+                                 const std::vector<StdSize>& scount, const short* data)
+      {
+          CNetCdfInterface::putVaraType(grpid, varid, &sstart[0], &scount[0], data);
+      }
       //---------------------------------------------------------------
 

XIOS3/branches/xios-3.0-beta/src/io/onetcdf4_decl.cpp

@@ -16 +16 @@
   macro(double, 2)
   macro(double, 3)
+  macro(float, 1)
+  macro(short, 1)
   macro(StdString, 1)