Changeset 2522 for XIOS3/trunk/src


Ignore:
Timestamp:
06/23/23 14:35:34 (12 months ago)
Author:
ymipsl
Message:

Improvment of event scheduler. Now a hierachical approach make possible event scheduling accross different process groups, if a parent group of process is totally overlapping a child group of process.
YM

Location:
XIOS3/trunk/src
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • XIOS3/trunk/src/event_scheduler.cpp

    r2518 r2522  
    99  
    1010  CEventScheduler::CEventScheduler(const MPI_Comm& comm)  
     11  { 
     12     schedulerLevel_=0 ; 
     13     parentScheduler_.reset(); 
     14     childScheduler_.reset(); 
     15     initialize(comm) ; 
     16  } 
     17   
     18  CEventScheduler::CEventScheduler(const MPI_Comm& comm, size_t schedulerLevel)  
     19  { 
     20     schedulerLevel_=schedulerLevel ; 
     21     parentScheduler_.reset(); 
     22     childScheduler_.reset(); 
     23     initialize(comm) ; 
     24  } 
     25   
     26  void CEventScheduler::initialize(const MPI_Comm& comm)  
    1127  { 
    1228    MPI_Comm_dup(comm, &communicator_) ; 
    … …  
    7288    while (!pendingSentParentRequest_.empty() || !pendingRecvParentRequest_.empty() || !pendingRecvChildRequest_.empty() ||  !pendingSentChildRequest_.empty()) 
    7389    { 
    74       checkEvent() ; 
     90      checkEvent_() ; 
    7591    }  
    7692  }  
    7793 
     94  void CEventScheduler::splitScheduler(const MPI_Comm& splittedComm, shared_ptr<CEventScheduler>& parent, shared_ptr<CEventScheduler>& child) 
     95  { 
     96    int color ; 
     97    MPI_Comm newComm ; 
     98    child = make_shared<CEventScheduler>(splittedComm, schedulerLevel_+ 1) ; 
     99    if (child->isRoot()) color=1 ; 
     100    else color=0 ; 
     101    MPI_Comm_split(communicator_, color, mpiRank_, &newComm) ; 
     102 
     103    parent = make_shared<CEventScheduler>(newComm , schedulerLevel_) ; 
     104    child->setParentScheduler(parent) ; 
     105    parent->setChildScheduler(child) ; 
     106    if (parentScheduler_)  
     107    { 
     108      parentScheduler_->setChildScheduler(parent) ; 
     109      parent->setParentScheduler(parentScheduler_) ; 
     110    } 
     111 
     112  } 
     113 
    78114  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId) 
    79115  { 
    80     registerEvent(timeLine, contextHashId, level_) ; 
    81     checkEvent() ; 
    82   } 
    83    
    84   void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) 
     116    getBaseScheduler()->registerEvent(timeLine, contextHashId, schedulerLevel_) ; 
     117    checkEvent_() ; 
     118  } 
     119   
     120  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel) 
     121  { 
     122    registerEvent(timeLine, contextHashId, schedulerLevel, level_) ; 
     123    checkEvent_() ; 
     124  } 
     125 
     126  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev) 
    85127  { 
    86128        
    … …  
    89131    sentRequest->buffer[0]=timeLine ; 
    90132    sentRequest->buffer[1]=contextHashId ; 
    91     sentRequest->buffer[2]=lev-1 ; 
     133    sentRequest->buffer[2]=schedulerLevel ; 
     134    sentRequest->buffer[3]=lev-1 ; 
    92135 
    93136    pendingSentParentRequest_.push(sentRequest) ; 
    94     MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ; 
     137//    info(100)<<"CEventScheduler::registerEvent => send event to parent "<<parent_[lev]<<" of level" <<lev-1<<endl ; 
     138    MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ; 
    95139    traceOn() ; 
    96140  }  
    97141 
    98   bool CEventScheduler::queryEvent(const size_t timeLine, const size_t contextHashId) 
    99   { 
    100     checkEvent() ; 
     142   
     143  bool CEventScheduler::queryEvent_(const size_t timeLine, const size_t contextHashId) 
     144  { 
     145    checkEvent_() ; 
     146 
    101147    if (! eventStack_.empty() && eventStack_.front().first==timeLine && eventStack_.front().second==contextHashId) 
    102148    { 
    103       //eventStack_.pop() ; 
    104149      return true ; 
    105150    } 
    … …  
    107152  }  
    108153  
    109   void CEventScheduler::checkEvent(void) 
    110   { 
     154  void CEventScheduler::checkEvent_(void) 
     155  { 
     156    
     157    if (parentScheduler_) parentScheduler_->checkEvent_() ; 
    111158    traceOff() ; 
    112159    checkChildRequest() ; 
    … …  
    143190      { 
    144191        recvRequest=new SPendingRequest ; 
    145         MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ; 
     192        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ; 
    146193        pendingRecvParentRequest_.push(recvRequest) ; 
    147194      } 
    … …  
    154201      recvRequest=pendingRecvParentRequest_.front() ; 
    155202      MPI_Test( &(recvRequest->request), &completed, &status) ; 
     203 
    156204      if (completed)  
    157205      { 
    158206        size_t timeLine=recvRequest->buffer[0] ; 
    159207        size_t hashId=recvRequest->buffer[1] ; 
    160         size_t lev=recvRequest->buffer[2] ; 
     208        size_t schedulerLevel=recvRequest->buffer[2] ; 
     209        size_t lev=recvRequest->buffer[3] ; 
    161210        delete recvRequest ; 
    162211        pendingRecvParentRequest_.pop() ;        
    163   
    164         if (lev==level_) eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ; 
    165         else  bcastEvent(timeLine, hashId, lev) ; 
     212         
     213//        info(100)<<"CEventScheduler::checkParentRequest => receive event from parent "<< status.MPI_SOURCE<<"at level"<< lev<< endl ; 
     214         
     215        if (lev==level_)  
     216        { 
     217          if (childScheduler_) 
     218          { 
     219//            info(100)<<"CEventScheduler::checkParentRequest => bcast event to child scheduler "<<endl; 
     220            childScheduler_->bcastEvent(timeLine, hashId, schedulerLevel, 0) ; 
     221          } 
     222          else 
     223          {  
     224//            info(100)<<"CEventScheduler::checkParentRequest => put event to stack : timeLine : "<<timeLine<<"  hashId : "<<hashId<<endl; 
     225            eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ; 
     226          } 
     227        } 
     228        else   
     229        { 
     230//          info(100)<<"CEventScheduler::checkParentRequest => bcast event to child process "<<endl; 
     231          bcastEvent(timeLine, hashId, schedulerLevel, lev) ; 
     232        } 
    166233      } 
    167234    }    
    … …  
    185252      { 
    186253        recvRequest=new SPendingRequest ; 
    187         MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ; 
     254        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ; 
    188255        pendingRecvChildRequest_.push_back(recvRequest) ; 
    189256      } 
    … …  
    199266        size_t timeLine=(*it)->buffer[0] ; 
    200267        size_t hashId=(*it)->buffer[1] ; 
    201         size_t lev=(*it)->buffer[2] ; 
    202          
    203         SEvent event={timeLine,hashId,lev} ; 
     268        size_t schedulerLevel=(*it)->buffer[2] ; 
     269        size_t lev=(*it)->buffer[3] ; 
     270         
     271//        info(100)<<"CEventScheduler::checkChildRequest => received event from child "<<status.MPI_SOURCE<<" at level "<<lev<<endl; 
     272 
     273        SEvent event={timeLine, hashId, schedulerLevel, lev} ; 
    204274        delete *it ; // free mem 
    205275        it=pendingRecvChildRequest_.erase(it) ; // get out of the list 
    … …  
    216286          if (lev==0) 
    217287          { 
    218             bcastEvent(timeLine,hashId,lev) ; 
     288            if (schedulerLevel==schedulerLevel_)  
     289            {   
     290//              info(100)<<"CEventScheduler::checkChildRequest => bcastEvent to child"<<endl ; 
     291              bcastEvent(timeLine, hashId, schedulerLevel, lev) ; 
     292            } 
     293            else  
     294            {  
     295//              info(100)<<"CEventScheduler::checkChildRequest => register event to parent scheduler"<<endl ;  
     296              parentScheduler_->registerEvent(timeLine, hashId, schedulerLevel) ; 
     297            } 
    219298            recvEvent_.erase(itEvent) ; 
    220299          } 
    221300          else 
    222301          { 
    223             registerEvent( timeLine,hashId,lev) ; 
     302//            info(100)<<"CEventScheduler::checkChildRequest => register event to parent process"<<endl ;  
     303            registerEvent( timeLine,hashId, schedulerLevel, lev) ; 
    224304            recvEvent_.erase(itEvent) ; 
    225305          } 
    … …  
    245325  } 
    246326   
    247   void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) 
     327  void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev) 
    248328  { 
    249329    SPendingRequest* sentRequest ; 
    … …  
    255335      sentRequest->buffer[0]=timeLine ; 
    256336      sentRequest->buffer[1]=contextHashId ; 
    257       sentRequest->buffer[2]=lev+1 ; 
    258       MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ; 
     337      sentRequest->buffer[2]=schedulerLevel ; 
     338      sentRequest->buffer[3]=lev+1 ; 
     339      MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ; 
    259340      pendingSentChildRequest_.push_back(sentRequest) ; 
    260341    } 

  • XIOS3/trunk/src/event_scheduler.hpp

    r2518 r2522  
    2727        */ 
    2828       CEventScheduler(const MPI_Comm& comm) ; 
    29  
     29       CEventScheduler(const MPI_Comm& comm, size_t schedulerLevel) ; 
    3030 
    3131       //! Destructor 
    … …  
    4040        */ 
    4141       void registerEvent(const size_t timeLine, const size_t contextHashId) ; 
     42        
     43       private: 
     44       CEventScheduler* getBaseScheduler(void) { if (childScheduler_== nullptr) return this; else return childScheduler_->getBaseScheduler();} 
    4245 
    43  
    44  
     46       public: 
    4547       //! public interface for query if the event defined by timeLine and hashId is sheduled next 
    4648       /*! 
    … …  
    5153        *  If the event is scheduled next, it is remove from the `eventStack` queue list   
    5254        */     
    53        bool queryEvent(const size_t timeLine, const size_t contextHashId) ; 
    54        void popEvent() { eventStack_.pop() ; } 
    55  
     55       bool queryEvent(const size_t timeLine, const size_t contextHashId) { return getBaseScheduler()->queryEvent_(timeLine, contextHashId); } 
     56       bool queryEvent_(const size_t timeLine, const size_t contextHashId) ; 
     57       void popEvent() { getBaseScheduler()->popEvent_() ; } 
     58       void popEvent_() { eventStack_.pop() ; } 
     59       bool isRoot(void) { return parent_[0]==mpiRank_ ;} 
     60       void setParentScheduler(shared_ptr<CEventScheduler> parentScheduler) { parentScheduler_ = parentScheduler ;} 
     61       void setChildScheduler(shared_ptr<CEventScheduler> childScheduler) { childScheduler_ = childScheduler ;} 
     62       void splitScheduler(const MPI_Comm& splittedComm, shared_ptr<CEventScheduler>& parent, shared_ptr<CEventScheduler>& child) ; 
    5663 
    5764       //! Public interface to give the hand to the instance to check pending or incoming message. 
    … …  
    5966        * Must be called periodicaly. Call `checkParentRequest` and `checkChildRequest` private method. 
    6067        */ 
    61        void checkEvent(void) ; 
     68       void checkEvent(void) { getBaseScheduler()->checkEvent_(); }  
     69       void checkEvent_(void) ; 
    6270 
    6371       private: 
    64  
    65  
     72         void initialize(const MPI_Comm& comm) ; 
     73        
    6674       //! Send an event to the parent of level `lev+1` 
    6775       /*! 
    … …  
    7179        *  The event is sent by an asynchrounous MPI_ISend 
    7280        */ 
    73        void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) ; 
     81       
     82       void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel) ; 
     83       void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev) ; 
    7484 
    7585 
    … …  
    108118        * Asynchronus MPI_ISend is used. 
    109119        */ 
    110        void bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) ; 
     120       void bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel ,const size_t lev) ; 
    111121        
    112122 
    … …  
    118128         size_t timeLine ; /*!< Time line id of the event in the context */ 
    119129         size_t hashId ; /*!< hassh id of the context */ 
     130         size_t schedulerLevel ; /*!< hierarchical level of scherduler */ 
    120131         size_t level ;  /*!<hierarchical level of the communication*/ 
    121132 
    … …  
    127138         bool operator==(const SEvent& e) const 
    128139         {  
    129            if (timeLine == e.timeLine && hashId == e.hashId && level==e.level) return true ; 
     140           if (timeLine == e.timeLine && hashId == e.hashId && level==e.level && schedulerLevel==e.schedulerLevel) return true ; 
    130141           else return false ; 
    131142         } ; 
    … …  
    142153           if (timeLine < e.timeLine) return true ; 
    143154           else if (timeLine == e.timeLine && hashId < e.hashId) return true ; 
    144            else if (timeLine == e.timeLine && hashId == e.hashId && level<e.level) return true ; 
     155           else if (timeLine == e.timeLine && hashId == e.hashId && schedulerLevel<e.schedulerLevel) return true ; 
     156           else if (timeLine == e.timeLine && hashId == e.hashId && schedulerLevel==e.schedulerLevel && level<e.level) return true ; 
    145157           else return false ; 
    146158         } ; 
    147159       } ;        
    148          
    149  
     160        
    150161       //! Pending request struture. It keep send or receive buffer from asynchronous communication while the request is not complete. 
    151162       struct SPendingRequest 
    152163       { 
    153          size_t buffer[3] ;      /*!< communication buffer : timeLine, hashId, level */ 
     164         size_t buffer[4] ;      /*!< communication buffer : timeLine, hashId, level */ 
    154165         MPI_Request request ;   /*!< pending MPI request */  
    155166       } ; 
    … …  
    171182       vector<vector<int> >  child_ ; /*!< List of child rank for each level */ 
    172183       vector<int> nbChild_ ;         /*!< Number of child for each level */     
     184        
     185       shared_ptr<CEventScheduler> parentScheduler_ ; 
     186       shared_ptr<CEventScheduler> childScheduler_ ; 
     187       bool hasParentScheduler_=false ; 
     188       size_t schedulerLevel_ ; 
    173189 
    174190    } ; 
Note: See TracChangeset for help on using the changeset viewer.