Index: XIOS3/trunk/src/event_scheduler.cpp
===================================================================
--- XIOS3/trunk/src/event_scheduler.cpp	(revision 2520)
+++ XIOS3/trunk/src/event_scheduler.cpp	(revision 2522)
@@ -9,4 +9,20 @@
  
   CEventScheduler::CEventScheduler(const MPI_Comm& comm) 
+  {
+     schedulerLevel_=0 ;
+     parentScheduler_.reset();
+     childScheduler_.reset();
+     initialize(comm) ;
+  }
+  
+  CEventScheduler::CEventScheduler(const MPI_Comm& comm, size_t schedulerLevel) 
+  {
+     schedulerLevel_=schedulerLevel ;
+     parentScheduler_.reset();
+     childScheduler_.reset();
+     initialize(comm) ;
+  }
+  
+  void CEventScheduler::initialize(const MPI_Comm& comm) 
   {
     MPI_Comm_dup(comm, &communicator_) ;
@@ -72,15 +88,41 @@
     while (!pendingSentParentRequest_.empty() || !pendingRecvParentRequest_.empty() || !pendingRecvChildRequest_.empty() ||  !pendingSentChildRequest_.empty())
     {
-      checkEvent() ;
+      checkEvent_() ;
     } 
   } 
 
+  void CEventScheduler::splitScheduler(const MPI_Comm& splittedComm, shared_ptr<CEventScheduler>& parent, shared_ptr<CEventScheduler>& child)
+  {
+    int color ;
+    MPI_Comm newComm ;
+    child = make_shared<CEventScheduler>(splittedComm, schedulerLevel_+ 1) ;
+    if (child->isRoot()) color=1 ;
+    else color=0 ;
+    MPI_Comm_split(communicator_, color, mpiRank_, &newComm) ;
+
+    parent = make_shared<CEventScheduler>(newComm , schedulerLevel_) ;
+    child->setParentScheduler(parent) ;
+    parent->setChildScheduler(child) ;
+    if (parentScheduler_) 
+    {
+      parentScheduler_->setChildScheduler(parent) ;
+      parent->setParentScheduler(parentScheduler_) ;
+    }
+
+  }
+
   void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId)
   {
-    registerEvent(timeLine, contextHashId, level_) ;
-    checkEvent() ;
-  }
-  
-  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t lev)
+    getBaseScheduler()->registerEvent(timeLine, contextHashId, schedulerLevel_) ;
+    checkEvent_() ;
+  }
+  
+  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel)
+  {
+    registerEvent(timeLine, contextHashId, schedulerLevel, level_) ;
+    checkEvent_() ;
+  }
+
+  void CEventScheduler::registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev)
   {
        
@@ -89,17 +131,20 @@
     sentRequest->buffer[0]=timeLine ;
     sentRequest->buffer[1]=contextHashId ;
-    sentRequest->buffer[2]=lev-1 ;
+    sentRequest->buffer[2]=schedulerLevel ;
+    sentRequest->buffer[3]=lev-1 ;
 
     pendingSentParentRequest_.push(sentRequest) ;
-    MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ;
+//    info(100)<<"CEventScheduler::registerEvent => send event to parent "<<parent_[lev]<<" of level" <<lev-1<<endl ;
+    MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, parent_[lev], 0, communicator_, &sentRequest->request) ;
     traceOn() ;
   } 
 
-  bool CEventScheduler::queryEvent(const size_t timeLine, const size_t contextHashId)
-  {
-    checkEvent() ;
+  
+  bool CEventScheduler::queryEvent_(const size_t timeLine, const size_t contextHashId)
+  {
+    checkEvent_() ;
+
     if (! eventStack_.empty() && eventStack_.front().first==timeLine && eventStack_.front().second==contextHashId)
     {
-      //eventStack_.pop() ;
       return true ;
     }
@@ -107,6 +152,8 @@
   } 
  
-  void CEventScheduler::checkEvent(void)
-  {
+  void CEventScheduler::checkEvent_(void)
+  {
+   
+    if (parentScheduler_) parentScheduler_->checkEvent_() ;
     traceOff() ;
     checkChildRequest() ;
@@ -143,5 +190,5 @@
       {
         recvRequest=new SPendingRequest ;
-        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ;
+        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 1, communicator_, &(recvRequest->request)) ;
         pendingRecvParentRequest_.push(recvRequest) ;
       }
@@ -154,14 +201,34 @@
       recvRequest=pendingRecvParentRequest_.front() ;
       MPI_Test( &(recvRequest->request), &completed, &status) ;
+
       if (completed) 
       {
         size_t timeLine=recvRequest->buffer[0] ;
         size_t hashId=recvRequest->buffer[1] ;
-        size_t lev=recvRequest->buffer[2] ;
+        size_t schedulerLevel=recvRequest->buffer[2] ;
+        size_t lev=recvRequest->buffer[3] ;
         delete recvRequest ;
         pendingRecvParentRequest_.pop() ;       
- 
-        if (lev==level_) eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ;
-        else  bcastEvent(timeLine, hashId, lev) ;
+        
+//        info(100)<<"CEventScheduler::checkParentRequest => receive event from parent "<< status.MPI_SOURCE<<"at level"<< lev<< endl ;
+        
+        if (lev==level_) 
+        {
+          if (childScheduler_)
+          {
+//            info(100)<<"CEventScheduler::checkParentRequest => bcast event to child scheduler "<<endl;
+            childScheduler_->bcastEvent(timeLine, hashId, schedulerLevel, 0) ;
+          }
+          else
+          { 
+//            info(100)<<"CEventScheduler::checkParentRequest => put event to stack : timeLine : "<<timeLine<<"  hashId : "<<hashId<<endl;
+            eventStack_.push(pair<size_t,size_t>(timeLine,hashId)) ;
+          }
+        }
+        else  
+        {
+//          info(100)<<"CEventScheduler::checkParentRequest => bcast event to child process "<<endl;
+          bcastEvent(timeLine, hashId, schedulerLevel, lev) ;
+        }
       }
     }   
@@ -185,5 +252,5 @@
       {
         recvRequest=new SPendingRequest ;
-        MPI_Irecv(recvRequest->buffer, 3, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ;
+        MPI_Irecv(recvRequest->buffer, 4, MPI_UNSIGNED_LONG, MPI_ANY_SOURCE, 0, communicator_, &recvRequest->request) ;
         pendingRecvChildRequest_.push_back(recvRequest) ;
       }
@@ -199,7 +266,10 @@
         size_t timeLine=(*it)->buffer[0] ;
         size_t hashId=(*it)->buffer[1] ;
-        size_t lev=(*it)->buffer[2] ;
-        
-        SEvent event={timeLine,hashId,lev} ;
+        size_t schedulerLevel=(*it)->buffer[2] ;
+        size_t lev=(*it)->buffer[3] ;
+        
+//        info(100)<<"CEventScheduler::checkChildRequest => received event from child "<<status.MPI_SOURCE<<" at level "<<lev<<endl;
+
+        SEvent event={timeLine, hashId, schedulerLevel, lev} ;
         delete *it ; // free mem
         it=pendingRecvChildRequest_.erase(it) ; // get out of the list
@@ -216,10 +286,20 @@
           if (lev==0)
           {
-            bcastEvent(timeLine,hashId,lev) ;
+            if (schedulerLevel==schedulerLevel_) 
+            {  
+//              info(100)<<"CEventScheduler::checkChildRequest => bcastEvent to child"<<endl ;
+              bcastEvent(timeLine, hashId, schedulerLevel, lev) ;
+            }
+            else 
+            { 
+//              info(100)<<"CEventScheduler::checkChildRequest => register event to parent scheduler"<<endl ; 
+              parentScheduler_->registerEvent(timeLine, hashId, schedulerLevel) ;
+            }
             recvEvent_.erase(itEvent) ;
           }
           else
           {
-            registerEvent( timeLine,hashId,lev) ;
+//            info(100)<<"CEventScheduler::checkChildRequest => register event to parent process"<<endl ; 
+            registerEvent( timeLine,hashId, schedulerLevel, lev) ;
             recvEvent_.erase(itEvent) ;
           }
@@ -245,5 +325,5 @@
   }
   
-  void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t lev)
+  void CEventScheduler::bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev)
   {
     SPendingRequest* sentRequest ;
@@ -255,6 +335,7 @@
       sentRequest->buffer[0]=timeLine ;
       sentRequest->buffer[1]=contextHashId ;
-      sentRequest->buffer[2]=lev+1 ;
-      MPI_Isend(sentRequest->buffer,3, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ;
+      sentRequest->buffer[2]=schedulerLevel ;
+      sentRequest->buffer[3]=lev+1 ;
+      MPI_Isend(sentRequest->buffer,4, MPI_UNSIGNED_LONG, child_[lev][i], 1, communicator_, & sentRequest->request) ;
       pendingSentChildRequest_.push_back(sentRequest) ;
     }
Index: XIOS3/trunk/src/event_scheduler.hpp
===================================================================
--- XIOS3/trunk/src/event_scheduler.hpp	(revision 2520)
+++ XIOS3/trunk/src/event_scheduler.hpp	(revision 2522)
@@ -27,5 +27,5 @@
         */
        CEventScheduler(const MPI_Comm& comm) ;
-
+       CEventScheduler(const MPI_Comm& comm, size_t schedulerLevel) ;
 
        //! Destructor
@@ -40,7 +40,9 @@
         */
        void registerEvent(const size_t timeLine, const size_t contextHashId) ;
+       
+       private:
+       CEventScheduler* getBaseScheduler(void) { if (childScheduler_== nullptr) return this; else return childScheduler_->getBaseScheduler();}
 
-
-
+       public:
        //! public interface for query if the event defined by timeLine and hashId is sheduled next
        /*!
@@ -51,7 +53,12 @@
         *  If the event is scheduled next, it is remove from the `eventStack` queue list  
         */    
-       bool queryEvent(const size_t timeLine, const size_t contextHashId) ;
-       void popEvent() { eventStack_.pop() ; }
-
+       bool queryEvent(const size_t timeLine, const size_t contextHashId) { return getBaseScheduler()->queryEvent_(timeLine, contextHashId); }
+       bool queryEvent_(const size_t timeLine, const size_t contextHashId) ;
+       void popEvent() { getBaseScheduler()->popEvent_() ; }
+       void popEvent_() { eventStack_.pop() ; }
+       bool isRoot(void) { return parent_[0]==mpiRank_ ;}
+       void setParentScheduler(shared_ptr<CEventScheduler> parentScheduler) { parentScheduler_ = parentScheduler ;}
+       void setChildScheduler(shared_ptr<CEventScheduler> childScheduler) { childScheduler_ = childScheduler ;}
+       void splitScheduler(const MPI_Comm& splittedComm, shared_ptr<CEventScheduler>& parent, shared_ptr<CEventScheduler>& child) ;
 
        //! Public interface to give the hand to the instance to check pending or incoming message.
@@ -59,9 +66,10 @@
         * Must be called periodicaly. Call `checkParentRequest` and `checkChildRequest` private method.
         */
-       void checkEvent(void) ;
+       void checkEvent(void) { getBaseScheduler()->checkEvent_(); } 
+       void checkEvent_(void) ;
 
        private:
-
-
+         void initialize(const MPI_Comm& comm) ;
+       
        //! Send an event to the parent of level `lev+1`
        /*!
@@ -71,5 +79,7 @@
         *  The event is sent by an asynchrounous MPI_ISend
         */
-       void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) ;
+      
+       void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel) ;
+       void registerEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel, const size_t lev) ;
 
 
@@ -108,5 +118,5 @@
         * Asynchronus MPI_ISend is used.
         */
-       void bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t lev) ;
+       void bcastEvent(const size_t timeLine, const size_t contextHashId, const size_t schedulerLevel ,const size_t lev) ;
        
 
@@ -118,4 +128,5 @@
          size_t timeLine ; /*!< Time line id of the event in the context */
          size_t hashId ; /*!< hassh id of the context */
+         size_t schedulerLevel ; /*!< hierarchical level of scherduler */
          size_t level ;  /*!<hierarchical level of the communication*/
 
@@ -127,5 +138,5 @@
          bool operator==(const SEvent& e) const
          { 
-           if (timeLine == e.timeLine && hashId == e.hashId && level==e.level) return true ;
+           if (timeLine == e.timeLine && hashId == e.hashId && level==e.level && schedulerLevel==e.schedulerLevel) return true ;
            else return false ;
          } ;
@@ -142,14 +153,14 @@
            if (timeLine < e.timeLine) return true ;
            else if (timeLine == e.timeLine && hashId < e.hashId) return true ;
-           else if (timeLine == e.timeLine && hashId == e.hashId && level<e.level) return true ;
+           else if (timeLine == e.timeLine && hashId == e.hashId && schedulerLevel<e.schedulerLevel) return true ;
+           else if (timeLine == e.timeLine && hashId == e.hashId && schedulerLevel==e.schedulerLevel && level<e.level) return true ;
            else return false ;
          } ;
        } ;       
-        
-
+       
        //! Pending request struture. It keep send or receive buffer from asynchronous communication while the request is not complete.
        struct SPendingRequest
        {
-         size_t buffer[3] ;      /*!< communication buffer : timeLine, hashId, level */
+         size_t buffer[4] ;      /*!< communication buffer : timeLine, hashId, level */
          MPI_Request request ;   /*!< pending MPI request */ 
        } ;
@@ -171,4 +182,9 @@
        vector<vector<int> >  child_ ; /*!< List of child rank for each level */
        vector<int> nbChild_ ;         /*!< Number of child for each level */    
+       
+       shared_ptr<CEventScheduler> parentScheduler_ ;
+       shared_ptr<CEventScheduler> childScheduler_ ;
+       bool hasParentScheduler_=false ;
+       size_t schedulerLevel_ ;
 
     } ;
