source: XIOS/dev/dev_ym/XIOS_SERVICES/src/manager/server_context.cpp @ 1761

#include "server_context.hpp"
#include "contexts_manager.hpp"
#include "cxios.hpp"
#include "mpi.hpp"
#include "context.hpp"
#include "register_context_info.hpp"
#include "services.hpp"


namespace xios
{
  using namespace std ;

  map<string, tuple<bool,MPI_Comm,MPI_Comm> > CServerContext::overlapedComm_ ;

  CServerContext::CServerContext(CService* parentService, MPI_Comm contextComm, const std::string& poolId, const std::string& serviceId, 
                                 const int& partitionId, const std::string& contextId) : finalizeSignal_(false), parentService_(parentService)
  {
   int localRank, globalRank, commSize ;

    MPI_Comm_dup(contextComm, &contextComm_) ;
    xiosComm_=CXios::getXiosComm() ;
  
    MPI_Comm_rank(xiosComm_,&globalRank) ;
    MPI_Comm_rank(contextComm_,&localRank) ;
 
    winNotify_ = new CWindowManager(contextComm_, maxBufferSize_) ;
    
    int type;
    if (localRank==localLeader_) 
    {
      globalLeader_=globalRank ;
      MPI_Comm_rank(contextComm_,&commSize) ;
      
      CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ;
      SRegisterContextInfo contextInfo = {poolId, serviceId, partitionId, type, contextId, commSize, globalLeader_} ;
      name_ = CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ;
      CXios::getContextsManager()->registerContext(name_, contextInfo) ;
    }
    MPI_Bcast(&type, 1, MPI_INT, localLeader_,contextComm_) ;
    name_ = CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ;
    context_=CContext::create(name_);

    context_->init(this, contextComm, type) ;

    info(10)<<"Context "<< CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId)<<" created, on local rank "<<localRank
                        <<" and global rank "<<globalRank<<endl  ;
  }

  
  bool CServerContext::createIntercomm(const string& poolId, const string& serviceId, const int& partitionId, const string& contextId, 
                                       const MPI_Comm& intraComm, MPI_Comm& interCommClient, MPI_Comm& interCommServer, bool wait)
  {
    int intraCommRank ;
    MPI_Comm_rank(intraComm, &intraCommRank) ;
    int contextLeader ;

    bool ok ;
    int type ;
    MPI_Comm newInterCommClient, newInterCommServer ;
    MPI_Comm_dup(contextComm_,&newInterCommClient) ;
    MPI_Comm_dup(contextComm_,&newInterCommServer) ;
    overlapedComm_[name_]=tuple<bool, MPI_Comm, MPI_Comm>(false, newInterCommClient, newInterCommServer) ;
    MPI_Barrier(contextComm_) ;

    if (intraCommRank==0)
    {
      ok=CXios::getContextsManager()->createServerContextIntercomm(poolId, serviceId, partitionId, contextId, name_, wait) ;
      if (ok) 
      {
        CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ;
        string name=CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ;
        CXios::getContextsManager()->getContextLeader(name, contextLeader) ;
      }
    }
    
    auto eventScheduler=parentService_->getEventScheduler() ;
    std::hash<string> hashString ;
    size_t hashId = hashString(name_) ;
    size_t currentTimeLine=0 ;
    eventScheduler->registerEvent(currentTimeLine,hashId); 

    while (!eventScheduler->queryEvent(currentTimeLine,hashId))
    {
       CXios::getDaemonsManager()->eventLoop() ;
    }  
    
    MPI_Bcast(&ok, 1, MPI_INT, 0, intraComm) ;

    if (ok)  
    {
      int overlap  ;
      if (get<0>(overlapedComm_[name_])) overlap=1 ;
      else overlap=0 ;

      int nOverlap ;  
      MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ;
      int commSize ;
      MPI_Comm_size(contextComm_,&commSize ) ;
      if (nOverlap==commSize)
      {
        cout<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ;
        interCommClient=newInterCommClient ;
        interCommServer=newInterCommServer ;
      }
      else if (nOverlap==0)
      { 
        cout<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ;
        MPI_Intercomm_create(intraComm, 0, xiosComm_, contextLeader, 3141, &interCommClient) ;
        MPI_Comm_dup(interCommClient, &interCommServer) ;
        MPI_Comm_free(&newInterCommClient) ;
        MPI_Comm_free(&newInterCommServer) ;
      }
      else
      {
        cout<<"CServerContext::createIntercomm : partial overlap ==> not managed"<<endl ;
      }
    }
    overlapedComm_.erase(name_) ;
    return ok ;
  }


  void CServerContext::createIntercomm(int remoteLeader, const string& sourceContext)
  {
     int commSize ;
     MPI_Comm_size(contextComm_,&commSize) ;
     for(int rank=0; rank<commSize; rank++)
     {
       notifyType_=NOTIFY_CREATE_INTERCOMM ;
       notifyCreateIntercomm_ = make_tuple(remoteLeader, sourceContext) ;
       sendNotification(rank) ;
     }
  }
  
  void CServerContext::sendNotification(int rank)
  {
    winNotify_->lockWindow(rank,0) ;
    winNotify_->pushToWindow(rank, this, &CServerContext::notificationsDumpOut) ;
    winNotify_->unlockWindow(rank,0) ;
  }

  
  void CServerContext::notificationsDumpOut(CBufferOut& buffer)
  {
    
    buffer.realloc(maxBufferSize_) ;
    
    if (notifyType_==NOTIFY_CREATE_INTERCOMM)
    {
      auto& arg=notifyCreateIntercomm_ ;
      buffer << notifyType_ << std::get<0>(arg)<<std::get<1>(arg) ;
    }
  }

  void CServerContext::notificationsDumpIn(CBufferIn& buffer)
  {
    if (buffer.bufferSize() == 0) notifyType_= NOTIFY_NOTHING ;
    else
    {
      buffer>>notifyType_;
      if (notifyType_==NOTIFY_CREATE_INTERCOMM)
      {
        auto& arg=notifyCreateIntercomm_ ;
        buffer >> std::get<0>(arg)>> std::get<1>(arg) ;
      }
    }
  }

  void CServerContext::checkNotifications(void)
  {
    int commRank ;
    MPI_Comm_rank(contextComm_, &commRank) ;
    winNotify_->lockWindow(commRank,0) ;
    winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ;
    winNotify_->unlockWindow(commRank,0) ;
    if (notifyType_==NOTIFY_CREATE_INTERCOMM) createIntercomm() ;
  }

  bool CServerContext::eventLoop(void)
  {
    bool finished=false ;
    checkNotifications() ;
    if (context_!=nullptr)  
    {
      if (context_->eventLoop())
      {
        context_=nullptr ;
        // destroy context ??? --> later
      }
    }

    if (context_==nullptr && finalizeSignal_) finished=true ;
    return finished ;
  }

  void CServerContext::createIntercomm(void)
  {
     MPI_Comm interCommServer, interCommClient ;
     int remoteLeader=get<0>(notifyCreateIntercomm_) ;
     string sourceContext=get<1>(notifyCreateIntercomm_) ;

     auto it=overlapedComm_.find(sourceContext) ;
     int overlap=0 ;
     if (it!=overlapedComm_.end())
     {
       get<0>(it->second)=true ;
       overlap=1 ;
     }
     int nOverlap ;  
     MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ;
     int commSize ;
     MPI_Comm_size(contextComm_,&commSize ) ;

    if (nOverlap==commSize)
    {
      cout<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ;
      interCommClient=get<2>(it->second) ;
      interCommServer=get<1>(it->second) ;
      context_ -> createClientInterComm(interCommClient, interCommServer ) ;
      clientsInterComm_.push_back(interCommClient) ;
      clientsInterComm_.push_back(interCommServer) ;
    }
    else if (nOverlap==0)
    { 
      cout<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ;
      MPI_Intercomm_create(contextComm_, 0, xiosComm_, remoteLeader, 3141, &interCommServer) ;
      MPI_Comm_dup(interCommServer,&interCommClient) ;
      context_ -> createClientInterComm(interCommClient,interCommServer) ;
      clientsInterComm_.push_back(interCommClient) ;
      clientsInterComm_.push_back(interCommServer) ;
    }
    else
    {
      cout<<"CServerContext::createIntercomm : partial overlap ==> not managed"<<endl ;
    }
   
  }

  void CServerContext::finalizeSignal(void)
  {
    finalizeSignal_=true ;
  }
   

}