Changeset 2580 for XIOS3


Ignore:
Timestamp:
09/26/23 17:24:35 (17 months ago)
Author:
ymipsl
Message:

Tracking unfree MPI windows and communicators.

YM

Location:
XIOS3/trunk/src
Files:
20 edited

Legend:

Unmodified
Added
Removed

  • XIOS3/trunk/src/client.cpp

    r2576 r2580  
    106106 
    107107          MPI_Comm_split(globalComm, color, commRank, &clientComm) ; 
     108          CXios::getMpiGarbageCollector().registerCommunicator(clientComm) ; 
    108109        } 
    109110        else 
    … …  
    115116      { 
    116117        MPI_Comm_dup(localComm,&clientComm) ; 
     118        CXios::getMpiGarbageCollector().registerCommunicator(clientComm) ; 
    117119        MPI_Comm_dup(localComm,&intraComm_) ; 
     120        CXios::getMpiGarbageCollector().registerCommunicator(intraComm_) ; 
    118121 
    119122        if (CXios::usingServer) 
    … …  
    175178      MPI_Comm_rank(CXios::getXiosComm(), &commRank) ; 
    176179      MPI_Comm_split(CXios::getXiosComm(),false,commRank, &clientsComm_) ; 
     180      CXios::getMpiGarbageCollector().registerCommunicator(clientsComm_) ; 
    177181       
    178182      // is using server or not ? 
    … …  
    277281        {   
    278282          MPI_Intercomm_create(intraComm, 0, globalComm, serverRank, 3141, &interComm); 
     283          CXios::getMpiGarbageCollector().registerCommunicator(interComm) ; 
    279284          MPI_Comm_free(&intraComm) ; 
    280285          MPI_Intercomm_merge(interComm,high, &intraComm ) ; 
    … …  
    341346      CXios::setXiosComm(xiosGlobalComm) ; 
    342347 
    343       MPI_Comm commUnfree ; 
    344       MPI_Comm_dup(clientComm, &commUnfree ) ; 
    345   
    346348    } 
    347349 
    … …  
    519521      finalizePoolRessource() ; 
    520522      CContext::removeAllContexts() ; // free memory for related context  
    521  
    522523      CXios::getMpiGarbageCollector().release() ; // release unfree MPI ressources 
     524      MPI_Comm xiosComm=CXios::getXiosComm() ; 
     525      MPI_Comm_free(&xiosComm) ; 
    523526      CCommTrack::dumpComm() ; 
    524527      if (!is_MPI_Initialized) 

  • XIOS3/trunk/src/cxios.hpp

    r2547 r2580  
    125125     static CPoolRessource*     getPoolRessource(void) ; 
    126126      
    127      static MPI_Comm getGlobalComm(void) { return globalComm ;} 
    128      static MPI_Comm getXiosComm(void) { return xiosComm ;} 
     127     static MPI_Comm& getGlobalComm(void) { return globalComm ;} 
     128     static MPI_Comm& getXiosComm(void) { return xiosComm ;} 
    129129     static void setXiosComm(MPI_Comm comm) { xiosComm=comm ;} 
    130130     static CRegistry* getGlobalRegistry(void) { return globalRegistry ;} 

  • XIOS3/trunk/src/manager/contexts_manager.cpp

    r2517 r2580  
    2626 
    2727    MPI_Comm_rank(xiosComm_, &commRank) ; 
    28     winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     28    winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_,"CContextsManager::winNotify_") ; 
    2929    winNotify_->updateToExclusiveWindow(commRank, this, &CContextsManager::notificationsDumpOut) ; 
    3030    
    3131 
    32     winContexts_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     32    winContexts_ = new CWindowManager(xiosComm_, maxBufferSize_,"CContextsManager::winContexts_") ; 
    3333    winContexts_->updateToExclusiveWindow(commRank, this, &CContextsManager::contextsDumpOut) ; 
    3434   

  • XIOS3/trunk/src/manager/coupler_manager.cpp

    r2517 r2580  
    1919    MPI_Comm_rank(xiosComm_, &commRank) ; 
    2020     
    21     winRegistredCoupling_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
    22     winNextCoupling_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     21    winRegistredCoupling_ = new CWindowManager(xiosComm_, maxBufferSize_,"CCouplerManager::winRegistredCoupling_") ; 
     22    winNextCoupling_ = new CWindowManager(xiosComm_, maxBufferSize_,"CCouplerManager::winNextCoupling_") ; 
    2323    if (commRank==managerGlobalLeader_) 
    2424    { 

  • XIOS3/trunk/src/manager/pool_ressource.cpp

    r2571 r2580  
    1616    int commRank, commSize ; 
    1717    MPI_Comm_dup(poolComm, &poolComm_) ; 
    18     winNotify_ = new CWindowManager(poolComm_, maxBufferSize_) ; 
     18    CXios::getMpiGarbageCollector().registerCommunicator(poolComm_) ; 
     19    winNotify_ = new CWindowManager(poolComm_, maxBufferSize_,"CPoolRessource::winNotify_") ; 
    1920    MPI_Comm_rank(poolComm, &commRank) ; 
    2021    MPI_Comm_size(poolComm, &commSize) ; 
    … …  
    404405        MPI_Comm newServiceComm ; 
    405406        MPI_Comm_dup(serviceComm, &newServiceComm) ; 
     407        CXios::getMpiGarbageCollector().registerCommunicator(newServiceComm) ; 
    406408        int nbPartitions = service.second->getNbPartitions() ; 
    407409        int partitionId = service.second->getPartitionId() ; 

  • XIOS3/trunk/src/manager/ressources_manager.cpp

    r2562 r2580  
    2222    if (commRank==0 && isXiosServer) MPI_Comm_rank(xiosComm_, &commRank) ;  
    2323    else commRank=0 ; 
    24     tokenManager_ = new CTokenManager(xiosComm_,commRank) ; 
     24    //tokenManager_ = new CTokenManager(xiosComm_,commRank) ; 
    2525 
    2626    MPI_Allreduce(&commRank, &managerGlobalLeader_, 1, MPI_INT, MPI_SUM, xiosComm_) ; 
    2727 
    2828    MPI_Comm_rank(xiosComm_, &commRank) ; 
    29     winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
    30     
    31  
    32     winRessources_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     29    winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_,"CRessourcesManager::winNotify_") ; 
     30    
     31 
     32    winRessources_ = new CWindowManager(xiosComm_, maxBufferSize_,"CRessourcesManager::winRessources_") ; 
    3333    winRessources_->lockWindow(commRank,0) ; 
    3434    serverLeader_=-1 ; 
    … …  
    4343    delete winNotify_ ; 
    4444    delete winRessources_ ; 
    45     delete tokenManager_ ; 
     45    //delete tokenManager_ ; 
    4646  }  
    4747 

  • XIOS3/trunk/src/manager/ressources_manager.hpp

    r2523 r2580  
    5959    void registerPoolClient(const std::string& poolId,int size,int leader) ; 
    6060    void registerPoolServer(const std::string& poolId,int size,int leader) ; 
    61     CTokenManager* getTokenManager(void) {return tokenManager_ ;}  
     61    //CTokenManager* getTokenManager(void) {return tokenManager_ ;}  
    6262 
    6363    int managerGlobalLeader_ ; 
    … …  
    6666 
    6767    CWindowManager* winNotify_ ; 
    68     CTokenManager* tokenManager_ ; 
     68    //CTokenManager* tokenManager_ ; 
    6969 
    7070    const size_t maxBufferSize_=1024*1024 ; 

  • XIOS3/trunk/src/manager/server_context.cpp

    r2547 r2580  
    2424 
    2525    MPI_Comm_dup(contextComm, &contextComm_) ; 
     26    CXios::getMpiGarbageCollector().registerCommunicator(contextComm_) ; 
    2627    xiosComm_=CXios::getXiosComm() ; 
    2728   
    … …  
    2930    MPI_Comm_rank(contextComm_,&localRank) ; 
    3031  
    31     winNotify_ = new CWindowManager(contextComm_, maxBufferSize_) ; 
     32    winNotify_ = new CWindowManager(contextComm_, maxBufferSize_,"CServerContext::winNotify_") ; 
    3233    MPI_Barrier(contextComm_) ; 
    3334     
    … …  
    5758  CServerContext::~CServerContext() 
    5859  { 
     60    delete winNotify_ ; 
    5961    cout<<"Server Context destructor"<<endl; 
    6062  }  
    … …  
    126128      {  
    127129        MPI_Intercomm_create(intraComm, 0, xiosComm_, contextLeader, 3141, &interCommClient) ; 
     130        CXios::getMpiGarbageCollector().registerCommunicator(interCommClient) ; 
    128131        MPI_Comm_dup(interCommClient, &interCommServer) ; 
     132        CXios::getMpiGarbageCollector().registerCommunicator(interCommServer) ; 
    129133        MPI_Comm_free(&newInterCommClient) ; 
    130134        MPI_Comm_free(&newInterCommServer) ; 
    … …  
    315319      info(10)<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ; 
    316320      MPI_Intercomm_create(contextComm_, 0, xiosComm_, remoteLeader, 3141, &interCommServer) ; 
     321      CXios::getMpiGarbageCollector().registerCommunicator(interCommServer) ; 
    317322      MPI_Comm_dup(interCommServer,&interCommClient) ; 
     323      CXios::getMpiGarbageCollector().registerCommunicator(interCommClient) ; 
    318324      context_ -> createClientInterComm(interCommClient,interCommServer) ; 
    319325      clientsInterComm_.push_back(interCommClient) ; 
    … …  
    329335  void CServerContext::freeComm(void) 
    330336  { 
    331     delete winNotify_ ; 
    332     winNotify_=nullptr ; 
    333     MPI_Comm_free(&contextComm_) ; 
     337    //delete winNotify_ ; 
     338    //winNotify_=nullptr ; 
     339    //MPI_Comm_free(&contextComm_) ; 
    334340    // don't forget intercomm -> later 
    335341  } 

  • XIOS3/trunk/src/manager/servers_ressource.cpp

    r2547 r2580  
    2323 
    2424    MPI_Comm_dup(serverComm, &serverComm_) ; 
     25    CXios::getMpiGarbageCollector().registerCommunicator(serverComm_) ;  
    2526    MPI_Comm xiosComm=CXios::getXiosComm() ; 
    2627   
    … …  
    2930    MPI_Comm_rank(serverComm_,&localRank) ; 
    3031     
    31     winNotify_ = new CWindowManager(serverComm_, maxBufferSize_) ; 
     32    winNotify_ = new CWindowManager(serverComm_, maxBufferSize_,"CServersRessource::winNotify_") ; 
    3233    MPI_Barrier(serverComm_) ; 
    3334    if (localRank==localLeader_)  
    … …  
    4243 
    4344    MPI_Comm_dup(serverComm_, &freeRessourcesComm_) ;  
     45    CXios::getMpiGarbageCollector().registerCommunicator(freeRessourcesComm_) ; 
    4446    eventScheduler_ = make_shared<CEventScheduler>(freeRessourcesComm_) ; 
    4547    freeRessourceEventScheduler_ = eventScheduler_ ; 

  • XIOS3/trunk/src/manager/services.cpp

    r2547 r2580  
    2222 
    2323    MPI_Comm_dup(serviceComm, &serviceComm_) ; 
     24    CXios::getMpiGarbageCollector().registerCommunicator(serviceComm_) ; 
    2425    MPI_Comm globalComm_=CXios::getXiosComm() ; 
    2526   
    … …  
    2728    MPI_Comm_rank(serviceComm_,&localRank) ; 
    2829     
    29     winNotify_ = new CWindowManager(serviceComm_, maxBufferSize_) ; 
     30    winNotify_ = new CWindowManager(serviceComm_, maxBufferSize_,"CService::winNotify_") ; 
    3031    winNotify_->updateToExclusiveWindow(localRank, this, &CService::createContextDumpOut) ; 
    3132    MPI_Barrier(serviceComm_) ; 

  • XIOS3/trunk/src/manager/services_manager.cpp

    r2523 r2580  
    3030 
    3131    MPI_Comm_rank(xiosComm_, &commRank) ; 
    32     winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     32    winNotify_ = new CWindowManager(xiosComm_, maxBufferSize_,"CServicesManager::winNotify_") ; 
    3333    winNotify_->updateToExclusiveWindow(commRank, this, &CServicesManager::notificationsDumpOut) ; 
    3434 
    35     winServices_ = new CWindowManager(xiosComm_, maxBufferSize_) ; 
     35    winServices_ = new CWindowManager(xiosComm_, maxBufferSize_,"CServicesManager::winServices_") ; 
    3636    winServices_->updateToExclusiveWindow(commRank, this, &CServicesManager::servicesDumpOut) ; 
    3737    

  • XIOS3/trunk/src/manager/window_base.cpp

    r2570 r2580  
    44namespace xios 
    55{ 
    6     CWindowBase::CWindowBase(MPI_Comm winComm, size_t bufferSize) 
     6    CWindowBase::CWindowBase(MPI_Comm winComm, size_t bufferSize, const string name)  : name_(name) 
    77    { 
    88      bufferSize_ = bufferSize ; 
    … …  
    1313      lock=0 ; 
    1414      MPI_Win_lock_all(0, window_) ; 
     15      info(100)<<"CWindowBase constructor : "<<name_<<endl ; 
    1516      MPI_Barrier(winComm) ; 
    1617    } 

  • XIOS3/trunk/src/manager/window_base.hpp

    r2570 r2580  
    33 
    44#include <map> 
     5#include <string> 
     6 
    57#include "exception.hpp" 
    68#include "mpi.hpp" 
     9#include <string> 
    710 
    811namespace xios 
    … …  
    2124      const double maxLatency_ = 1e-3 ; // 1ms latency maximum 
    2225      MPI_Win window_ ; 
     26      std::string name_ ; 
    2327 
    2428    public : 
    2529 
    26     CWindowBase(MPI_Comm winComm, size_t bufferSize); 
     30    CWindowBase(MPI_Comm winComm, size_t bufferSize, const string name); 
    2731 
    2832    bool tryLockExclusive(int rank) 
    … …  
    143147    { 
    144148      MPI_Win_unlock_all(window_); 
     149      info(100)<<"CWindowBase destructor : "<<name_<<endl ; 
    145150    } 
    146151 

  • XIOS3/trunk/src/manager/window_manager.hpp

    r2517 r2580  
    3131    public : 
    3232 
    33     CWindowManager(MPI_Comm winComm, size_t bufferSize) : CWindowBase(winComm, bufferSize + OFFSET_BUFFER_SIZE) 
     33    CWindowManager(MPI_Comm winComm, size_t bufferSize, const string name) : CWindowBase(winComm, bufferSize + OFFSET_BUFFER_SIZE, name) 
    3434    { 
    3535      int lock=0 ; 

  • XIOS3/trunk/src/mpi_garbage_collector.hpp

    r2569 r2580  
    22#define __MPI_GARBAGE_COLLECTOR_HPP__ 
    33#include "mpi.hpp" 
     4#include "backtrace.hpp" 
    45namespace xios 
    56{ 
    … …  
    2021    public: 
    2122 
    22       void registerCommunicator(MPI_Comm& comm, std::string str="") { stack_.push_back(SType{SType::COMM, comm, MPI_WIN_NULL, str}) ;} 
    23       void registerWindow(MPI_Win& win, std::string str="") { stack_.push_back(SType{SType::WIN, MPI_COMM_NULL, win, str}) ;} 
     23     void registerCommunicator(MPI_Comm& comm, std::string str) { stack_.push_front(SType{SType::COMM, comm, MPI_WIN_NULL, str}) ;} 
     24     void registerCommunicator(MPI_Comm& comm) { stack_.push_front(SType{SType::COMM, comm, MPI_WIN_NULL, MemTrack::backTrace(2)}) ;} 
     25     void registerWindow(MPI_Win& win, std::string str) { stack_.push_front(SType{SType::WIN, MPI_COMM_NULL, win, str}) ;} 
     26     void registerWindow(MPI_Win& win) { stack_.push_front(SType{SType::WIN, MPI_COMM_NULL, win, MemTrack::backTrace(2)}) ;} 
    2427      void release(void) 
    2528      { 

  • XIOS3/trunk/src/mpi_tools.cpp

    r2578 r2580  
    2222  { 
    2323    auto it = commTrack_.find(comm) ; 
    24     if (it == commTrack_.end())  ERROR("CCommtrack::releaseComm", << "Communicator not allocated : " << endl) 
     24    if (it == commTrack_.end())  info(100)<<"WARNING : CCommtrack::releaseComm => Communicator not allocated !" << endl ; 
    2525    else commTrack_.erase(it) ; 
    2626  } 

  • XIOS3/trunk/src/node/context.cpp

    r2564 r2580  
    599599      intraCommClient=intraComm_ ; 
    600600      MPI_Comm_dup(intraComm_, &intraCommServer) ; 
     601      CXios::getMpiGarbageCollector().registerCommunicator(intraCommServer) ; 
    601602 
    602603      CContextClient* client = CContextClient::getNew(this, intraCommClient, interCommClient) ; 

  • XIOS3/trunk/src/server.cpp

    r2576 r2580  
    102102 
    103103        MPI_Comm_split(globalComm, color, commRank, &serverComm) ; 
     104        CXios::getMpiGarbageCollector().registerCommunicator(serverComm) ; 
     105 
    104106      } 
    105107      else // using OASIS 
    … …  
    118120      } 
    119121      MPI_Comm_dup(serverComm, &intraComm_); 
     122      CXios::getMpiGarbageCollector().registerCommunicator(intraComm_) ; 
    120123       
    121124      CTimer::get("XIOS").resume() ; 
    … …  
    165168      MPI_Comm_rank(CXios::getXiosComm(), &commRank) ; 
    166169      MPI_Comm_split(CXios::getXiosComm(),true,commRank,&serversComm_) ; 
     170      CXios::getMpiGarbageCollector().registerCommunicator(serversComm_) ; 
    167171       
    168172      CXios::setUsingServer() ; 
    … …  
    443447      {   
    444448        MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[i], 3141, &interComm); 
     449        CXios::getMpiGarbageCollector().registerCommunicator(interComm) ; 
    445450        interCommLeft.push_back(interComm) ; 
    446451        MPI_Comm_free(&intraComm) ; 
    … …  
    598603       
    599604      CContext::removeAllContexts() ; // free memory for related context 
    600            
     605       
    601606      CXios::getMpiGarbageCollector().release() ; // release unfree MPI ressources 
    602  
     607      MPI_Comm xiosComm=CXios::getXiosComm() ; 
     608      MPI_Comm_free(&xiosComm) ; 
    603609      CMemChecker::logMem( "CServer::finalize", true ); 
    604610       

  • XIOS3/trunk/src/transport/p2p_context_client.cpp

    r2556 r2580  
    3232 
    3333      MPI_Intercomm_merge(interComm_,false, &interCommMerged_) ; 
     34      CXios::getMpiGarbageCollector().registerCommunicator(interCommMerged_) ; 
    3435       
    3536      MPI_Comm_split(intraComm_,clientRank,clientRank, &commSelf_) ; // for windows 
     37      CXios::getMpiGarbageCollector().registerCommunicator(commSelf_) ; 
    3638      eventScheduler_ = parent->getEventScheduler() ;   
    3739      timeLine = 1; 

  • XIOS3/trunk/src/transport/p2p_context_server.cpp

    r2566 r2580  
    3737    
    3838    MPI_Comm_dup(intraComm, &processEventBarrier_) ; 
    39   
     39    CXios::getMpiGarbageCollector().registerCommunicator(processEventBarrier_) ; 
     40     
    4041    currentTimeLine=1; 
    4142    scheduled=false; 
    … …  
    4344 
    4445    MPI_Intercomm_merge(interComm_,true,&interCommMerged_) ; 
     46    CXios::getMpiGarbageCollector().registerCommunicator(interCommMerged_) ; 
    4547    MPI_Comm_split(intraComm_, intraCommRank, intraCommRank, &commSelf_) ; // for windows 
     48    CXios::getMpiGarbageCollector().registerCommunicator(commSelf_) ; 
    4649     
    4750    itLastTimeLine=lastTimeLine.begin() ; 
Note: See TracChangeset for help on using the changeset viewer.