Changeset 2019 for XIOS/dev/dev_trunk_graph/src/client.cpp

Timestamp:

01/22/21 12:00:29 (3 years ago)

Author:

yushan

Message:

Graph intermedia commit to a tmp branch

File:

• XIOS/dev/dev_trunk_graph/src/client.cpp (modified) (9 diffs)

XIOS/dev/dev_trunk_graph/src/client.cpp

@@ -12 +12 @@
 #include "buffer_client.hpp"
 #include "string_tools.hpp"
+#include "ressources_manager.hpp"
+#include "services_manager.hpp"
+#include <functional>
+#include <cstdio>
+#include "workflow_graph.hpp"
 
 namespace xios
 {
 
+    const double serverPublishDefaultTimeout=10;
+
     MPI_Comm CClient::intraComm ;
     MPI_Comm CClient::interComm ;
+    MPI_Comm CClient::clientsComm_ ;
+
     std::list<MPI_Comm> CClient::contextInterComms;
     int CClient::serverLeader ;
@@ -24 +33 @@
     StdOFStream CClient::m_infoStream;
     StdOFStream CClient::m_errorStream;
+    CPoolRessource* CClient::poolRessource_=nullptr ;
+
     MPI_Comm& CClient::getInterComm(void)   { return (interComm); }
      
@@ -37 +48 @@
     void CClient::initialize(const string& codeId, MPI_Comm& localComm, MPI_Comm& returnComm)
     {
+    
+       MPI_Comm clientComm ;
+      // initialize MPI if not initialized
       int initialized ;
       MPI_Initialized(&initialized) ;
       if (initialized) is_MPI_Initialized=true ;
       else is_MPI_Initialized=false ;
-      int rank ;
-
-// don't use OASIS
-      if (!CXios::usingOasis)
-      {
-// localComm isn't given
-        if (localComm == MPI_COMM_NULL)
-        {
+      
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+
+      /////////////////////////////////////////
+      ///////////// PART 1 ////////////////////
+      /////////////////////////////////////////
+      
+
+      // localComm isn't given
+      if (localComm == MPI_COMM_NULL)
+      {
+          
+        // don't use OASIS
+        if (!CXios::usingOasis)
+        {
+
           if (!is_MPI_Initialized)
           {
@@ -54 +76 @@
           }
           CTimer::get("XIOS").resume() ;
-          CTimer::get("XIOS init/finalize").resume() ;
-          boost::hash<string> hashString ;
-
-          unsigned long hashClient=hashString(codeId) ;
-          unsigned long hashServer=hashString(CXios::xiosCodeId) ;
-          unsigned long* hashAll ;
-          int size ;
-          int myColor ;
-          int i,c ;
-          MPI_Comm newComm ;
-
-          MPI_Comm_size(CXios::globalComm,&size) ;
-          MPI_Comm_rank(CXios::globalComm,&rank_);
-
-          hashAll=new unsigned long[size] ;
-
-          MPI_Allgather(&hashClient,1,MPI_LONG,hashAll,1,MPI_LONG,CXios::globalComm) ;
-
-          map<unsigned long, int> colors ;
-          map<unsigned long, int> leaders ;
-
-          for(i=0,c=0;i<size;i++)
+          CTimer::get("XIOS init/finalize",false).resume() ;
+          
+          // split the global communicator
+          // get hash from all model to attribute a unique color (int) and then split to get client communicator
+          // every mpi process of globalComm (MPI_COMM_WORLD) must participate
+
+          int commRank, commSize ;
+          MPI_Comm_rank(globalComm,&commRank) ;
+          MPI_Comm_size(globalComm,&commSize) ;
+
+          std::hash<string> hashString ;
+          size_t hashClient=hashString(codeId) ;
+          
+          size_t* hashAll = new size_t[commSize] ;
+          MPI_Allgather(&hashClient,1,MPI_UNSIGNED_LONG,hashAll,1,MPI_LONG,globalComm) ;
+          
+          int color=0 ;
+          set<size_t> listHash ;
+          for(int i=0 ; i<=commRank ; i++) 
+            if (listHash.count(hashAll[i])==0)
+            {
+              listHash.insert(hashAll[i]) ;
+              color=color+1 ;
+            }
+          delete[] hashAll ;
+
+          MPI_Comm_split(globalComm, color, commRank, &clientComm) ;
+        }
+        else // using oasis to split communicator
+        {
+          if (!is_MPI_Initialized) oasis_init(codeId) ;
+          oasis_get_localcomm(clientComm) ;
+        }
+      }
+      else // localComm is given
+      {
+        MPI_Comm_dup(localComm,&clientComm) ;
+      }
+      
+     
+      /////////////////////////////////////////
+      ///////////// PART 2 ////////////////////
+      /////////////////////////////////////////
+      
+
+      // Create the XIOS communicator for every process which is related
+      // to XIOS, as well on client side as on server side
+      
+      MPI_Comm xiosGlobalComm ;
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+      if (strIds.empty())
+      {
+         // no code Ids given, suppose XIOS initialisation is global            
+         int commRank, commGlobalRank, serverLeader, clientLeader,serverRemoteLeader,clientRemoteLeader ;
+         MPI_Comm splitComm,interComm ;
+         MPI_Comm_rank(globalComm,&commGlobalRank) ;
+         MPI_Comm_split(globalComm, 0, commGlobalRank, &splitComm) ;
+         int splitCommSize, globalCommSize ;
+        
+         MPI_Comm_size(splitComm,&splitCommSize) ;
+         MPI_Comm_size(globalComm,&globalCommSize) ;
+         if (splitCommSize==globalCommSize) // no server
+         {
+           MPI_Comm_dup(globalComm,&xiosGlobalComm) ;
+           CXios::setXiosComm(xiosGlobalComm) ;
+         }
+         else
+         {
+           MPI_Comm_rank(splitComm,&commRank) ;
+           if (commRank==0) clientLeader=commGlobalRank ;
+           else clientLeader=0 ;
+           serverLeader=0 ;
+           MPI_Allreduce(&clientLeader,&clientRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+           MPI_Allreduce(&serverLeader,&serverRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ;
+           MPI_Intercomm_create(splitComm, 0, globalComm, serverRemoteLeader,1341,&interComm) ;
+           MPI_Intercomm_merge(interComm,true,&xiosGlobalComm) ;
+           CXios::setXiosComm(xiosGlobalComm) ;
+         }
+      }
+      else
+      {
+
+        xiosGlobalCommByFileExchange(clientComm, codeId) ;
+      
+      }
+
+      int commRank ;
+      MPI_Comm_rank(CXios::getXiosComm(), &commRank) ;
+      MPI_Comm_split(CXios::getXiosComm(),false,commRank, &clientsComm_) ;
+      
+      // is using server or not ?
+      int xiosCommSize, clientsCommSize ; 
+      MPI_Comm_size(CXios::getXiosComm(), &xiosCommSize) ;
+      MPI_Comm_size(clientsComm_, &clientsCommSize) ;
+      if (xiosCommSize==clientsCommSize) CXios::setUsingServer() ;
+      else CXios::setNotUsingServer() ;
+
+
+      CXios::setGlobalRegistry(new CRegistry(clientsComm_)) ;
+      /////////////////////////////////////////
+      ///////////// PART 3 ////////////////////
+      /////////////////////////////////////////
+     
+      CXios::launchDaemonsManager(false) ;
+      poolRessource_ = new CPoolRessource(clientComm, codeId) ;
+
+      /////////////////////////////////////////
+      ///////////// PART 4 ////////////////////
+      /////////////////////////////////////////      
+      
+      returnComm = clientComm ;
+    }
+
+
+    void CClient::xiosGlobalCommByFileExchange(MPI_Comm clientComm, const string& codeId)
+    {
+ 
+      MPI_Comm globalComm=CXios::getGlobalComm() ;
+      MPI_Comm xiosGlobalComm ;
+
+      string strIds=CXios::getin<string>("clients_code_id","") ;
+      vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ;
+
+      int commRank, globalRank, clientRank, serverRank ;
+      MPI_Comm_rank(clientComm, &commRank) ;
+      MPI_Comm_rank(globalComm, &globalRank) ;
+      string clientFileName("__xios_publisher::"+codeId+"__to_remove__") ;
+           
+      int error ;
+
+      if (commRank==0) // if root process publish name
+      {  
+        std::ofstream ofs (clientFileName, std::ofstream::out);
+        ofs<<globalRank ;
+        ofs.close();
+        
+  // get server root rank
+
+        std::ifstream ifs ;
+        string fileName=("__xios_publisher::"+CXios::xiosCodeId+"__to_remove__") ;
+      
+        double timeout = CXios::getin<double>("server_puplish_timeout",serverPublishDefaultTimeout) ;
+        double time ;
+          
+        do
+        {
+          CTimer::get("server_publish_timeout").resume() ;  
+          ifs.clear() ;
+          ifs.open(fileName, std::ifstream::in) ;
+          CTimer::get("server_publish_timeout").suspend() ;
+        } while (ifs.fail() && CTimer::get("server_publish_timeout").getCumulatedTime()<timeout) ;
+        
+        if (CTimer::get("server_publish_timeout").getCumulatedTime()>=timeout || ifs.fail())
+        {
+          ifs.clear() ;
+          ifs.close() ;
+          ifs.clear() ;
+          error=true ;            
+        }
+        else 
+        {
+          ifs>>serverRank ;
+          ifs.close() ;
+          error=false ;
+        } 
+
+      } 
+      
+      MPI_Bcast(&error,1,MPI_INT,0,clientComm) ;
+      
+      if (error==false)  // you have a server
+      {
+        MPI_Comm intraComm ;
+        MPI_Comm_dup(clientComm,&intraComm) ;
+        MPI_Comm interComm ;
+        
+        int pos=0 ;
+        for(int i=0 ; codeId!=clientsCodeId[i]; i++) pos=pos+1 ;
+
+        bool high=true ;
+        for(int i=pos ; i<clientsCodeId.size(); i++)
+        {  
+          MPI_Intercomm_create(intraComm, 0, globalComm, serverRank, 3141, &interComm);
+          MPI_Comm_free(&intraComm) ;
+          MPI_Intercomm_merge(interComm,high, &intraComm ) ;
+          high=false ;
+        }
+        xiosGlobalComm=intraComm ;
+      }
+      else  // no server detected
+      {
+        vector<int> clientsRank(clientsCodeId.size()) ;
+        
+        if (commRank==0)
+        {  
+          for(int i=0;i<clientsRank.size();i++)
           {
-            if (colors.find(hashAll[i])==colors.end())
+            std::ifstream ifs ;
+            string fileName=("__xios_publisher::"+clientsCodeId[i]+"__to_remove__") ;
+            do
             {
-              colors[hashAll[i]] =c ;
-              leaders[hashAll[i]]=i ;
-              c++ ;
+              ifs.clear() ;
+              ifs.open(fileName, std::ifstream::in) ;
+            } while (ifs.fail()) ;
+            ifs>>clientsRank[i] ;
+            ifs.close() ;
+          }
+        }
+         
+        int client ;
+        MPI_Comm intraComm ;
+        MPI_Comm_dup(clientComm,&intraComm) ;
+        MPI_Comm interComm ;
+        
+        int pos=0 ;
+        for(int i=0 ; codeId!=clientsCodeId[i]; i++) pos=pos+1 ;
+        
+        bool high=true ;
+        for(int i=pos+1 ; i<clientsCodeId.size(); i++)
+        {  
+          if (codeId==clientsCodeId[0])   // first model play the server rule
+          {          
+            MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[i], 3141, &interComm);
+            MPI_Intercomm_merge(interComm,false, &intraComm ) ;
+          }
+          else
+          {          
+            MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[0], 3141, &interComm);
+            MPI_Intercomm_merge(interComm,high, &intraComm ) ;
+            high=false ;
+          }
+        }
+        xiosGlobalComm=intraComm ;
+      }
+
+      MPI_Barrier(xiosGlobalComm);
+      if (commRank==0) std::remove(clientFileName.c_str()) ;         
+      MPI_Barrier(xiosGlobalComm);
+  
+      CXios::setXiosComm(xiosGlobalComm) ;
+
+      MPI_Comm commUnfree ;
+      MPI_Comm_dup(clientComm, &commUnfree ) ;
+ 
+    }
+
+// to check on other architecture
+    void CClient::xiosGlobalCommByPublishing(MPI_Comm clientComm, const string& codeId)
+    {
+
+      // untested. need to be developped an a true MPI compliant library
+
+/*
+        // try to discover other client/server
+        // do you have a xios server ?
+        char portName[MPI_MAX_PORT_NAME];
+        int ierr ;
+        int commRank ;
+        MPI_Comm_rank(clientComm,&commRank) ;
+
+        MPI_Barrier(globalComm) ;
+        if (commRank==0)
+        {
+             
+          MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN );
+          const char* serviceName=CXios::xiosCodeId.c_str() ;
+          ierr=MPI_Lookup_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+          MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
+        }
+        ierr=MPI_SUCCESS ;
+        MPI_Bcast(&ierr,1,MPI_INT,0,clientComm) ;
+
+        if (ierr==MPI_SUCCESS) // you have a server
+        {  
+          MPI_Comm intraComm=clientComm ;
+          MPI_Comm interComm ;
+          for(int i=0 ; i<clientsCodeId.size(); i++)
+          {  
+            MPI_Comm_connect(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+            MPI_Intercomm_merge(interComm, true, &intraComm ) ;
+          }
+          xiosGlobalComm=intraComm ;
+        }
+        else  // you don't have any server
+        {
+          if (codeId==clientsCodeId[0]) // first code will publish his name
+          {
+
+            if (commRank==0) // if root process publish name
+            {  
+              MPI_Open_port(MPI_INFO_NULL, portName);
+              MPI_Publish_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+            }
+
+            MPI_Comm intraComm=clientComm ;
+            MPI_Comm interComm ;
+            for(int i=0 ; i<clientsCodeId.size()-1; i++)
+            {  
+              MPI_Comm_accept(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+              MPI_Intercomm_merge(interComm,false, &intraComm ) ;
             }
           }
-
-          // Verify whether we are on server mode or not
-          CXios::setNotUsingServer();
-          for (i=0; i < size; ++i)
+          else  // other clients are connecting to the first one
           {
-            if (hashServer == hashAll[i])
+            if (commRank==0)
             {
-              CXios::setUsingServer();
-              break;
+
+              MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN );
+              ierr=MPI_Lookup_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName);
+              MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
+             }
+
+            MPI_Bcast(&ierr,1,MPI_INT,0,clientComm) ;
+
+            if (ierr==MPI_SUCCESS) // you can connect
+            {  
+              MPI_Comm intraComm=clientComm ;
+              MPI_Comm interComm ;
+              for(int i=0 ; i<clientsCodeId.size()-1; i++)
+              {  
+                MPI_Comm_connect(portName, MPI_INFO_NULL, 0, intraComm, &interComm);
+                MPI_Intercomm_merge(interComm, true, &intraComm ) ;
+              }
+              xiosGlobalComm=intraComm ;
             }
           }
-
-          myColor=colors[hashClient];
-          MPI_Comm_split(CXios::globalComm,myColor,rank_,&intraComm) ;
-
-          if (CXios::usingServer)
-          {
-            int clientLeader=leaders[hashClient] ;
-            serverLeader=leaders[hashServer] ;
-            int intraCommSize, intraCommRank ;
-            MPI_Comm_size(intraComm,&intraCommSize) ;
-            MPI_Comm_rank(intraComm,&intraCommRank) ;
-            info(50)<<"intercommCreate::client "<<rank_<<" intraCommSize : "<<intraCommSize
-                   <<" intraCommRank :"<<intraCommRank<<"  clientLeader "<< serverLeader<<endl ;
-             MPI_Intercomm_create(intraComm, 0, CXios::globalComm, serverLeader, 0, &interComm) ;
-             //rank_ = intraCommRank;
-          }
-          else
-          {
-            MPI_Comm_dup(intraComm,&interComm) ;
-          }
-          delete [] hashAll ;
-        }
-        // localComm argument is given
-        else
-        {
-          if (CXios::usingServer)
-          {
-            //ERROR("void CClient::initialize(const string& codeId,MPI_Comm& localComm,MPI_Comm& returnComm)", << " giving a local communictor is not compatible with using server mode") ;
-          }
-          else
-          {
-            MPI_Comm_dup(localComm,&intraComm) ;
-            MPI_Comm_dup(intraComm,&interComm) ;
-          }
-        }
-      }
-      // using OASIS
-      else
-      {
-        // localComm isn't given
-        if (localComm == MPI_COMM_NULL)
-        {
-          if (!is_MPI_Initialized) oasis_init(codeId) ;
-          oasis_get_localcomm(localComm) ;
-        }
-        MPI_Comm_dup(localComm,&intraComm) ;
-
-        CTimer::get("XIOS").resume() ;
-        CTimer::get("XIOS init/finalize").resume() ;
-
-        if (CXios::usingServer)
-        {
-          MPI_Status status ;
-          MPI_Comm_rank(intraComm,&rank_) ;
-
-          oasis_get_intercomm(interComm,CXios::xiosCodeId) ;
-          if (rank_==0) MPI_Recv(&serverLeader,1, MPI_INT, 0, 0, interComm, &status) ;
-          MPI_Bcast(&serverLeader,1,MPI_INT,0,intraComm) ;
-        }
-        else MPI_Comm_dup(intraComm,&interComm) ;
-      }
-
-      MPI_Comm_dup(intraComm,&returnComm) ;
-    }
+        }  
+      */
+    }
+
 
 ///---------------------------------------------------------------
@@ -170 +418 @@
     void CClient::registerContext(const string& id, MPI_Comm contextComm)
     {
-      CContext::setCurrent(id) ;
-      CContext* context=CContext::create(id);
-      StdString idServer(id);
-      idServer += "_server";
-
-      if (CXios::isServer && !context->hasServer)
-      // Attached mode
-      {
-        MPI_Comm contextInterComm ;
-        MPI_Comm_dup(contextComm,&contextInterComm) ;
-        CContext* contextServer = CContext::create(idServer);
-
-        // Firstly, initialize context on client side
-        context->initClient(contextComm,contextInterComm, contextServer);
-
-        // Secondly, initialize context on server side
-        contextServer->initServer(contextComm,contextInterComm, context);
-
-        // Finally, we should return current context to context client
-        CContext::setCurrent(id);
-
-        contextInterComms.push_back(contextInterComm);
-      }
-      else
-      {
-        int size,rank,globalRank ;
-        size_t message_size ;
-        int leaderRank ;
-        MPI_Comm contextInterComm ;
-
-        MPI_Comm_size(contextComm,&size) ;
-        MPI_Comm_rank(contextComm,&rank) ;
-        MPI_Comm_rank(CXios::globalComm,&globalRank) ;
-        if (rank!=0) globalRank=0 ;
-
-        CMessage msg ;
-        msg<<idServer<<size<<globalRank ;
-//        msg<<id<<size<<globalRank ;
-
-        int messageSize=msg.size() ;
-        char * buff = new char[messageSize] ;
-        CBufferOut buffer((void*)buff,messageSize) ;
-        buffer<<msg ;
-
-        MPI_Send((void*)buff,buffer.count(),MPI_CHAR,serverLeader,1,CXios::globalComm) ;
-
-        MPI_Intercomm_create(contextComm,0,CXios::globalComm,serverLeader,10+globalRank,&contextInterComm) ;
-        info(10)<<"Register new Context : "<<id<<endl ;
-        MPI_Comm inter ;
-        MPI_Intercomm_merge(contextInterComm,0,&inter) ;
-        MPI_Barrier(inter) ;
-
-        context->initClient(contextComm,contextInterComm) ;
-
-        contextInterComms.push_back(contextInterComm);
-        MPI_Comm_free(&inter);
-        delete [] buff ;
-
-      }
-    }
+      int commRank, commSize ;
+      MPI_Comm_rank(contextComm,&commRank) ;
+      MPI_Comm_size(contextComm,&commSize) ;
+
+      getPoolRessource()->createService(contextComm, id, 0, CServicesManager::CLIENT, 1) ;
+      getPoolRessource()->createService(contextComm, CXios::defaultServerId, 0, CServicesManager::IO_SERVER, 1) ;
+
+      if (commRank==0) while (!CXios::getServicesManager()->hasService(getPoolRessource()->getId(), id, 0)) { CXios::getDaemonsManager()->eventLoop();}
+
+      if (commRank==0) CXios::getContextsManager()->createServerContext(getPoolRessource()->getId(), id, 0, id) ;
+      int type=CServicesManager::CLIENT ;
+      string name = CXios::getContextsManager()->getServerContextName(getPoolRessource()->getId(), id, 0, type, id) ;
+      while (!CXios::getContextsManager()->hasContext(name, contextComm) )
+      {
+        CXios::getDaemonsManager()->eventLoop() ;
+      }
+
+    }
+
+
 
 /*!
@@ -260 +468 @@
     }
 
-
     void CClient::finalize(void)
     {
-      int rank ;
-      int msg=0 ;
-
-      MPI_Comm_rank(intraComm,&rank) ;
- 
-      if (!CXios::isServer)
-      {
-        MPI_Comm_rank(intraComm,&rank) ;
-        if (rank==0)
-        {
-          MPI_Send(&msg,1,MPI_INT,0,0,interComm) ;
-        }
-      }
-
-      for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++)
-        MPI_Comm_free(&(*it));
-      MPI_Comm_free(&interComm);
-      MPI_Comm_free(&intraComm);
-
-      CTimer::get("XIOS init/finalize").suspend() ;
+     
+      MPI_Barrier(clientsComm_) ;
+      int commRank ;
+      MPI_Comm_rank(clientsComm_, &commRank) ;
+      if (commRank==0) CXios::getRessourcesManager()->finalize() ;
+      
+      auto globalRegistry=CXios::getGlobalRegistry() ;
+      globalRegistry->hierarchicalGatherRegistry() ;
+
+      if (commRank==0)
+      {
+        info(80)<<"Write data base Registry"<<endl<<globalRegistry->toString()<<endl ;
+        globalRegistry->toFile("xios_registry.bin") ;
+      }
+      delete globalRegistry ;
+
+      CTimer::get("XIOS init/finalize",false).suspend() ;
       CTimer::get("XIOS").suspend() ;
+      
+      CXios::finalizeDaemonsManager() ;
 
       if (!is_MPI_Initialized)
@@ -301 +507 @@
       report(0)<< " Memory report : increasing it by a factor will increase performance, depending of the volume of data wrote in file at each time step of the file"<<endl ;
       report(100)<<CTimer::getAllCumulatedTime()<<endl ;
-   }
+    
+      CWorkflowGraph::drawWorkFlowGraph_client();
+    }
+    
 
     /*!
@@ -325 +534 @@
       int size = 0;
       int rank;
-      MPI_Comm_size(CXios::globalComm, &size);
+      MPI_Comm_size(CXios::getGlobalComm(), &size);
+      MPI_Comm_rank(CXios::getGlobalComm(),&rank);
       while (size)
       {
@@ -332 +542 @@
       }
 
-      if (CXios::usingOasis)
-      {
-        MPI_Comm_rank(CXios::globalComm,&rank);
-        fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << rank << ext;
-      }
-      else
-        fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << getRank() << ext;
-
+      fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << rank << ext;
 
       fb->open(fileNameClient.str().c_str(), std::ios::out);