Changeset 2522 for XIOS3/trunk/src/event_scheduler.cpp

Timestamp:

06/23/23 14:35:34 (12 months ago)

Author:

ymipsl

Message:

Improvment of event scheduler. Now a hierachical approach make possible event scheduling accross different process groups, if a parent group of process is totally overlapping a child group of process.
YM

File:

• XIOS3/trunk/src/event_scheduler.cpp (modified) (11 diffs)

XIOS3/trunk/src/event_scheduler.cpp

@@ -9 +9 @@
  
   CEventScheduler::CEventScheduler(const MPI_Comm& comm) 
+  {
+     schedulerLevel_=0 ;
+     parentScheduler_.reset();
+     childScheduler_.reset();
+     initialize(comm) ;
+  }
+  
+  CEventScheduler::CEventScheduler(const MPI_Comm& comm, size_t schedulerLevel) 
+  {
+     schedulerLevel_=schedulerLevel ;
+     parentScheduler_.reset();
+     childScheduler_.reset();
+     initialize(comm) ;
+  }
+  
+  void CEventScheduler::initialize(const MPI_Comm& comm) 
   {
     MPI_Comm_dup(comm, &communicator_) ;
@@ -72 +88 @@
     while (!pendingSentParentRequest_.empty() || !pendingRecvParentRequest_.empty() || !pendingRecvChildRequest_.empty() ||  !pendingSentChildRequest_.empty())
     {
-      checkEvent() ;
+      checkEvent_() ;
     } 
   } 
 
+  void CEventScheduler::splitScheduler(const MPI_Comm& splittedComm, shared_ptr<CEventScheduler>& parent, shared_ptr<CEventScheduler>& child)
+  {
+    int color ;
+    MPI_Comm newComm ;
+    child = make_shared<CEventScheduler>(splittedComm, schedulerLevel_+ 1) ;
+    if (child->isRoot()) color=1 ;
+    else color=0 ;
+    MPI_Comm_split(communicator_, color, mpiRank_, &newComm) ;
+
+    parent = make_shared<CEventScheduler>(newComm , schedulerLevel_) ;
+    child->setParentScheduler(parent) ;
+    parent->setChildScheduler(child) ;
+    if (parentScheduler_) 
+    {
+      parentScheduler_->setChildScheduler(parent) ;
+      parent->setParentScheduler(parentScheduler_) ;
+    }
+
+  }
+
   void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId)
   {
-    registerEvent(timeLine, contextHashId, level_) ;
-    checkEvent() ;
-  }
-  
-  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t lev)
+    getBaseScheduler()->registerEvent(timeLine, contextHashId, schedulerLevel_) ;
+    checkEvent_() ;
+  }
+  
+  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel)
+  {
+    registerEvent(timeLine, contextHashId, schedulerLevel, level_) ;
+    checkEvent_() ;
+  }
+
+  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev)
   {
        
@@ -89 +131 @@
     sentRequest->buffer[0]=timeLine ;
     sentRequest->buffer[1]=contextHashId ;
-    sentRequest->buffer[2]=lev-1 ;
+    sentRequest->buffer[2]=schedulerLevel ;
+    sentRequest->buffer[3]=lev-1 ;
 
     pendingSentParentRequest_.push(sentRequest) ;
-    MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ;
+//    info(100)<<"CEventScheduler::registerEvent => send event to parent "<<parent_[lev]<<" of level" <<lev-1<<endl ;
+    MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ;
     traceOn() ;
   } 
 
-  bool CEventScheduler::queryEvent(const size_t timeLine, const size_t contextHashId)
-  {
-    checkEvent() ;
+  
+  bool CEventScheduler::queryEvent_(const size_t timeLine, const size_t contextHashId)
+  {
+    checkEvent_() ;
+
     if (! eventStack_.empty() && eventStack_.front().first==timeLine && eventStack_.front().second==contextHashId)
     {
-      //eventStack_.pop() ;
       return true ;
     }
@@ -107 +152 @@
   } 
  
-  void CEventScheduler::checkEvent(void)
-  {
+  void CEventScheduler::checkEvent_(void)
+  {
+   
+    if (parentScheduler_) parentScheduler_->checkEvent_() ;
     traceOff() ;
     checkChildRequest() ;
@@ -143 +190 @@
       {
         recvRequest=new SPendingRequest ;
-        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ;
+        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ;
         pendingRecvParentRequest_.push(recvRequest) ;
       }
@@ -154 +201 @@
       recvRequest=pendingRecvParentRequest_.front() ;
       MPI_Test( &(recvRequest->request), &completed, &status) ;
+
       if (completed) 
       {
         size_t timeLine=recvRequest->buffer[0] ;
         size_t hashId=recvRequest->buffer[1] ;
-        size_t lev=recvRequest->buffer[2] ;
+        size_t schedulerLevel=recvRequest->buffer[2] ;
+        size_t lev=recvRequest->buffer[3] ;
         delete recvRequest ;
         pendingRecvParentRequest_.pop() ;       
- 
-        if (lev==level_) eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ;
-        else  bcastEvent(timeLine, hashId, lev) ;
+        
+//        info(100)<<"CEventScheduler::checkParentRequest => receive event from parent "<< status.MPI_SOURCE<<"at level"<< lev<< endl ;
+        
+        if (lev==level_) 
+        {
+          if (childScheduler_)
+          {
+//            info(100)<<"CEventScheduler::checkParentRequest => bcast event to child scheduler "<<endl;
+            childScheduler_->bcastEvent(timeLine, hashId, schedulerLevel, 0) ;
+          }
+          else
+          { 
+//            info(100)<<"CEventScheduler::checkParentRequest => put event to stack : timeLine : "<<timeLine<<"  hashId : "<<hashId<<endl;
+            eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ;
+          }
+        }
+        else  
+        {
+//          info(100)<<"CEventScheduler::checkParentRequest => bcast event to child process "<<endl;
+          bcastEvent(timeLine, hashId, schedulerLevel, lev) ;
+        }
       }
     }   
@@ -185 +252 @@
       {
         recvRequest=new SPendingRequest ;
-        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ;
+        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ;
         pendingRecvChildRequest_.push_back(recvRequest) ;
       }
@@ -199 +266 @@
         size_t timeLine=(*it)->buffer[0] ;
         size_t hashId=(*it)->buffer[1] ;
-        size_t lev=(*it)->buffer[2] ;
-        
-        SEvent event={timeLine,hashId,lev} ;
+        size_t schedulerLevel=(*it)->buffer[2] ;
+        size_t lev=(*it)->buffer[3] ;
+        
+//        info(100)<<"CEventScheduler::checkChildRequest => received event from child "<<status.MPI_SOURCE<<" at level "<<lev<<endl;
+
+        SEvent event={timeLine, hashId, schedulerLevel, lev} ;
         delete *it ; // free mem
         it=pendingRecvChildRequest_.erase(it) ; // get out of the list
@@ -216 +286 @@
           if (lev==0)
           {
-            bcastEvent(timeLine,hashId,lev) ;
+            if (schedulerLevel==schedulerLevel_) 
+            {  
+//              info(100)<<"CEventScheduler::checkChildRequest => bcastEvent to child"<<endl ;
+              bcastEvent(timeLine, hashId, schedulerLevel, lev) ;
+            }
+            else 
+            { 
+//              info(100)<<"CEventScheduler::checkChildRequest => register event to parent scheduler"<<endl ; 
+              parentScheduler_->registerEvent(timeLine, hashId, schedulerLevel) ;
+            }
             recvEvent_.erase(itEvent) ;
           }
           else
           {
-            registerEvent( timeLine,hashId,lev) ;
+//            info(100)<<"CEventScheduler::checkChildRequest => register event to parent process"<<endl ; 
+            registerEvent( timeLine,hashId, schedulerLevel, lev) ;
             recvEvent_.erase(itEvent) ;
           }
@@ -245 +325 @@
   }
   
-  void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t lev)
+  void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev)
   {
     SPendingRequest* sentRequest ;
@@ -255 +335 @@
       sentRequest->buffer[0]=timeLine ;
       sentRequest->buffer[1]=contextHashId ;
-      sentRequest->buffer[2]=lev+1 ;
-      MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ;
+      sentRequest->buffer[2]=schedulerLevel ;
+      sentRequest->buffer[3]=lev+1 ;
+      MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ;
       pendingSentChildRequest_.push_back(sentRequest) ;
     }