Changeset 2672 for XIOS3/dev


Ignore:
Timestamp:
10/30/24 17:12:42 (3 months ago)
Author:
ymipsl
Message:

New mode "fugaku-like" for exclusive/shared windows lock.
YM

Location:
XIOS3/dev/XIOS_NOTIFICATIONS_MANAGER/src/manager
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • XIOS3/dev/XIOS_NOTIFICATIONS_MANAGER/src/manager/window_base.cpp

    r2580 r2672  
    1010      MPI_Win_allocate(windowSize_, 1, MPI_INFO_NULL, winComm, &winBuffer_, &window_) ; 
    1111      CXios::getMpiGarbageCollector().registerWindow(window_) ; 
    12       MPI_Aint& lock = *((MPI_Aint*)((char*)winBuffer_+OFFSET_LOCK)) ; 
    13       lock=0 ; 
     12      uint64_t* lock = (uint64_t*)((char*)winBuffer_+OFFSET_LOCK) ; 
     13      lock[0]=0 ; lock[1]=0 ; lock[2]=0 ; lock[3]=0 ;  
    1414      MPI_Win_lock_all(0, window_) ; 
    1515      info(100)<<"CWindowBase constructor : "<<name_<<endl ; 

  • XIOS3/dev/XIOS_NOTIFICATIONS_MANAGER/src/manager/window_base.hpp

    r2580 r2672  
    88#include "mpi.hpp" 
    99#include <string> 
     10#include <cstdint> 
    1011 
    1112namespace xios 
    … …  
    1819      void * winBuffer_ ;    
    1920      const MPI_Aint OFFSET_LOCK=0 ; 
    20       const int SIZE_LOCK=sizeof(long) ; 
     21      const int SIZE_LOCK=4*sizeof(uint64_t) ; 
    2122      const MPI_Aint OFFSET_BUFFER =  SIZE_LOCK ; 
    2223      MPI_Aint bufferSize_ ; 
    … …  
    3031    CWindowBase(MPI_Comm winComm, size_t bufferSize, const string name); 
    3132 
    32     bool tryLockExclusive(int rank) 
    33     { 
    34       long lock = 1; 
    35       long unlock = 0; 
    36       long state; 
    37  
    38       int flag ; 
    39       MPI_Compare_and_swap(&lock, &unlock, &state, MPI_LONG, rank, OFFSET_LOCK, window_) ; 
    40       MPI_Win_flush(rank, window_); 
    41  
    42       bool locked = (state == unlock) ; 
    43       return locked ; 
    44     } 
    45  
    46     bool tryLockShared(int rank, MPI_Op op) 
    47     { 
    48       long one = 0x100000000; 
    49       long res; 
    50  
    51       MPI_Fetch_and_op(&one, &res, MPI_LONG, rank, OFFSET_LOCK, op, window_); 
    52       MPI_Win_flush(rank, window_); 
    53        
    54       bool locked =  ! (res & 1) ; 
    55       return locked ; 
    56     } 
    57  
    5833    void unlockExclusive(int rank) 
    5934    { 
    60       int lock = 1; 
    61       int unlock = 0; 
    62       int state; 
    63  
     35      uint64_t inc[2] = {1 , 1} ; 
     36      uint64_t state[2] ; 
     37       
     38      MPI_Accumulate(inc, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_SUM, window_) ; 
    6439      MPI_Win_flush(rank, window_); 
    65       MPI_Compare_and_swap(&unlock, &lock, &state, MPI_INT, rank, OFFSET_LOCK, window_) ; 
    66       MPI_Win_flush(rank, window_); 
    67       if (lock != state) ERROR("CWindowBase::unlockWindowExclusive",<<"unlockWindow failed: bad state"<<endl) ;  
    6840    } 
    6941 
    7042    void unlockShared(int rank) 
    7143    { 
    72       int minusone = -1; 
    73       int res; 
    74       MPI_Fetch_and_op(&minusone, &res, MPI_INT, rank, OFFSET_LOCK+4, MPI_SUM, window_); 
     44      uint64_t inc[2] = {1 , 0} ; 
     45      uint64_t state[2] ; 
     46       
     47      MPI_Accumulate(inc, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_SUM, window_) ; 
    7548      MPI_Win_flush(rank, window_); 
    7649    } 
    … …  
    7851    void lockExclusive(int rank) 
    7952    { 
     53      uint64_t inc[2] = {1 , 1} ; 
     54      uint64_t state[2] ; 
     55      uint64_t res[2] ; 
     56      bool locked ; 
     57 
     58      locked=false;  
     59      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, state, 2, MPI_UINT64_T, rank, OFFSET_LOCK, 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     60      MPI_Win_flush(rank, window_); 
    8061      double time =  MPI_Wtime() ; 
    81       bool locked = tryLockExclusive(rank); 
     62      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     63      MPI_Win_flush(rank, window_); 
    8264      double lastTime = MPI_Wtime() ; 
    8365      double delta = lastTime-time ; 
    84        
     66      if (res[0]==state[0]) locked=true ; 
     67 
    8568      while (!locked) 
    8669      { 
    … …  
    8972        if (time >= lastTime+delta) 
    9073        {  
    91           locked = tryLockExclusive(rank); 
    92           delta=delta*2.; 
    93           lastTime = time ;       
     74          MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     75          MPI_Win_flush(rank, window_); 
     76          if (res[0]==state[0]) locked=true ; 
     77          else 
     78          { 
     79            delta=delta*2.; 
     80            lastTime = time ; 
     81          }       
    9482        } 
    9583      }   
    … …  
    9886    void lockShared(int rank) 
    9987    { 
     88      uint64_t inc[2] = {1 , 0} ; 
     89      uint64_t state[2] ; 
     90      uint64_t res[2] ; 
     91      bool locked ; 
     92 
     93      locked=false;  
     94      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, state, 2, MPI_UINT64_T, rank, OFFSET_LOCK, 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     95      MPI_Win_flush(rank, window_); 
    10096      double time =  MPI_Wtime() ; 
    101       bool locked = tryLockShared(rank, MPI_SUM); 
     97      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     98      MPI_Win_flush(rank, window_); 
    10299      double lastTime = MPI_Wtime() ; 
    103100      double delta = lastTime-time ; 
    104        
     101      if (res[1]==state[1]) locked=true ; 
     102 
    105103      while (!locked) 
    106104      { 
    … …  
    109107        if (time >= lastTime+delta) 
    110108        {  
    111           locked = tryLockShared(rank, MPI_NO_OP); 
    112           delta=delta*2.; 
    113           lastTime = time ;       
     109          MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     110          MPI_Win_flush(rank, window_); 
     111          if (res[1]==state[1]) locked=true ; 
     112          else 
     113          { 
     114            delta=delta*2.; 
     115            lastTime = time ; 
     116          }       
    114117        } 
    115118      }   

  • XIOS3/dev/XIOS_NOTIFICATIONS_MANAGER/src/manager/window_dynamic.hpp

    r2667 r2672  
    1717      std::map<size_t, void *> winBuffer_ ;    
    1818      const MPI_Aint OFFSET_LOCK=0 ; 
    19       const int SIZE_LOCK=sizeof(long) ; 
     19      const int SIZE_LOCK=4*sizeof(uint64_t) ; 
    2020      const MPI_Aint OFFSET_BUFFER =  SIZE_LOCK ; 
    2121      const MPI_Aint OFFSET_BUFFER_SIZE = SIZE_LOCK   ; 
    … …  
    3535      windowSize_[tag] = size+OFFSET_BUFFER_SIZE ; 
    3636      MPI_Alloc_mem(windowSize_[tag], MPI_INFO_NULL, &winBuffer_[tag]) ; 
    37       MPI_Aint& lock = *((MPI_Aint*)(static_cast<char*>(winBuffer_[tag])+OFFSET_LOCK)) ; 
    38       lock=0 ; 
     37      uint64_t* lock = (uint64_t*)((char*)(winBuffer_[tag])+OFFSET_LOCK) ; 
     38      lock[0]=0 ; lock[1]=0 ; lock[2]=0 ; lock[3]=0 ;  
    3939    }   
    4040 
    … …  
    7575    } 
    7676     
    77     bool tryLockExclusive(int rank, size_t tag) 
    78     { 
    79       long lock = 1; 
    80       long unlock = 0; 
    81       long state; 
    82  
    83       int flag ; 
    84       if (rank==winCommRank_) MPI_Win_sync(window_) ; 
    85       MPI_Win_flush(rank, window_); 
    86       MPI_Compare_and_swap(&lock, &unlock, &state, MPI_LONG, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK, window_) ; 
    87       MPI_Win_flush(rank, window_); 
    88       if (rank==winCommRank_) MPI_Win_sync(window_) ; 
    89 //      if (rank==winCommRank_) MPI_Win_sync(window_) ; 
    90       bool locked = (state == unlock) ; 
    91       return locked ; 
    92     } 
    93  
    94     bool tryLockShared(int rank, MPI_Op op, size_t tag) 
    95     { 
    96       long one = 0x100000000; 
    97       long res; 
    98  
    99       MPI_Fetch_and_op(&one, &res, MPI_LONG, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK, op, window_); 
    100       MPI_Win_flush(rank, window_); 
     77     
     78    void unlockExclusive(int rank, size_t tag) 
     79    { 
     80      uint64_t inc[2] = {1 , 1} ; 
     81      uint64_t state[2] ; 
    10182       
    102       bool locked =  ! (res & 1) ; 
    103       return locked ; 
    104     } 
    105  
    106     void unlockExclusive(int rank, size_t tag) 
    107     { 
    108       int lock = 1; 
    109       int unlock = 0; 
    110       int state; 
     83      MPI_Accumulate(inc, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     84      MPI_Win_flush(rank, window_); 
     85    } 
     86 
     87    void unlockShared(int rank, size_t tag) 
     88    { 
     89      uint64_t inc[2] = {1 , 0} ; 
     90      uint64_t state[2] ; 
    11191       
    112       if (rank==winCommRank_) MPI_Win_sync(window_) ; 
    113       MPI_Win_flush(rank, window_); 
    114       MPI_Compare_and_swap(&unlock, &lock, &state, MPI_INT, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK, window_) ; 
    115       MPI_Win_flush(rank, window_); 
    116       //      if (rank==winCommRank_) MPI_Win_sync(window_) ; 
    117       if (lock != state)  
    118       { 
    119         info(100)<<"Bad State : "<<((long*)winBuffer_[tag])[0]<<endl ; 
    120         ERROR("CWindowBase::unlockWindowExclusive",<<"unlockWindow failed: bad state"<<endl) ;  
    121       } 
    122     } 
    123  
    124     void unlockShared(int rank, size_t tag) 
    125     { 
    126       int minusone = -1; 
    127       int res; 
    128       MPI_Fetch_and_op(&minusone, &res, MPI_INT, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+4, MPI_SUM, window_); 
    129       MPI_Win_flush(rank, window_); 
    130     } 
     92      MPI_Accumulate(inc, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     93      MPI_Win_flush(rank, window_); 
     94    } 
     95 
    13196 
    13297    void lockExclusive(int rank, size_t tag) 
    13398    { 
     99      uint64_t inc[2] = {1 , 1} ; 
     100      uint64_t state[2] ; 
     101      uint64_t res[2] ; 
     102      bool locked ; 
     103 
     104      locked=false;  
     105      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, state, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+ OFFSET_LOCK, 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     106      MPI_Win_flush(rank, window_); 
    134107      double time =  MPI_Wtime() ; 
    135       bool locked = tryLockExclusive(rank, tag); 
     108      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     109      MPI_Win_flush(rank, window_); 
    136110      double lastTime = MPI_Wtime() ; 
    137111      double delta = lastTime-time ; 
    138        
     112      if (res[0]==state[0]) locked=true ; 
     113 
    139114      while (!locked) 
    140115      { 
    … …  
    143118        if (time >= lastTime+delta) 
    144119        {  
    145           locked = tryLockExclusive(rank, tag); 
    146           delta=delta*2.; 
    147           lastTime = time ;       
     120          MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     121          MPI_Win_flush(rank, window_); 
     122          if (res[0]==state[0]) locked=true ; 
     123          else 
     124          { 
     125            delta=delta*2.; 
     126            lastTime = time ; 
     127          }       
    148128        } 
    149129      }   
    … …  
    152132    void lockShared(int rank, size_t tag) 
    153133    { 
     134      uint64_t inc[2] = {1 , 0} ; 
     135      uint64_t state[2] ; 
     136      uint64_t res[2] ; 
     137      bool locked ; 
     138 
     139      locked=false;  
     140      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, state, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK, 2, MPI_UINT64_T, MPI_SUM, window_) ; 
     141      MPI_Win_flush(rank, window_); 
    154142      double time =  MPI_Wtime() ; 
    155       bool locked = tryLockShared(rank, MPI_SUM, tag); 
     143      MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     144      MPI_Win_flush(rank, window_); 
    156145      double lastTime = MPI_Wtime() ; 
    157146      double delta = lastTime-time ; 
    158        
     147      if (res[1]==state[1]) locked=true ; 
     148 
    159149      while (!locked) 
    160150      { 
    … …  
    163153        if (time >= lastTime+delta) 
    164154        {  
    165           locked = tryLockShared(rank, MPI_NO_OP, tag); 
    166           delta=delta*2.; 
    167           lastTime = time ;       
     155          MPI_Get_accumulate(inc, 2, MPI_UINT64_T, res, 2, MPI_UINT64_T, rank, winBufferAddress_[{rank,tag}]+OFFSET_LOCK+2*sizeof(uint64_t), 2, MPI_UINT64_T, MPI_NO_OP, window_) ; 
     156          MPI_Win_flush(rank, window_); 
     157          if (res[1]==state[1]) locked=true ; 
     158          else 
     159          { 
     160            delta=delta*2.; 
     161            lastTime = time ; 
     162          }       
    168163        } 
    169164      }   
    170165    } 
    171      
     166 
     167    
    172168    void attach(MPI_Aint size, size_t tag)  
    173169    { 
    … …  
    175171      windowSize_[tag] = size+OFFSET_BUFFER_SIZE ; 
    176172      MPI_Alloc_mem(windowSize_[tag], MPI_INFO_NULL, &winBuffer_[tag]) ; 
    177       MPI_Aint& lock = *((MPI_Aint*)(static_cast<char*>(winBuffer_[tag])+OFFSET_LOCK)) ; 
    178       lock=0 ; 
     173      uint64_t* lock = (uint64_t*)((char*)(winBuffer_[tag])+OFFSET_LOCK) ; 
     174      lock[0]=0 ; lock[1]=0 ; lock[2]=0 ; lock[3]=0 ;  
     175 
    179176      info(100)<<"Attach Buffer  =>  "<<winBuffer_[tag]<<endl ; 
    180177      MPI_Win_attach(window_, winBuffer_[tag], windowSize_[tag]) ; 

  • XIOS3/dev/XIOS_NOTIFICATIONS_MANAGER/src/manager/window_dynamic_view.hpp

    r2667 r2672  
    5858    } 
    5959     
    60     bool tryLockExclusive(int rank) 
    61     { 
    62       return winDynamic_->tryLockExclusive(rank, tag_) ; 
    63     } 
    64  
    65     bool tryLockShared(int rank, MPI_Op op) 
    66     { 
    67       return winDynamic_->tryLockShared(rank, op, tag_) ; 
    68     } 
    69  
    7060    void unlockExclusive(int rank) 
    7161    { 
Note: See TracChangeset for help on using the changeset viewer.