[1761] | 1 | #include "server_context.hpp" |
---|
| 2 | #include "contexts_manager.hpp" |
---|
| 3 | #include "cxios.hpp" |
---|
| 4 | #include "mpi.hpp" |
---|
| 5 | #include "context.hpp" |
---|
| 6 | #include "register_context_info.hpp" |
---|
| 7 | #include "services.hpp" |
---|
| 8 | |
---|
| 9 | |
---|
| 10 | namespace xios |
---|
| 11 | { |
---|
| 12 | using namespace std ; |
---|
| 13 | |
---|
| 14 | map<string, tuple<bool,MPI_Comm,MPI_Comm> > CServerContext::overlapedComm_ ; |
---|
| 15 | |
---|
| 16 | CServerContext::CServerContext(CService* parentService, MPI_Comm contextComm, const std::string& poolId, const std::string& serviceId, |
---|
[1764] | 17 | const int& partitionId, const std::string& contextId) : finalizeSignal_(false), parentService_(parentService), |
---|
| 18 | hasNotification_(false) |
---|
[1761] | 19 | { |
---|
| 20 | int localRank, globalRank, commSize ; |
---|
| 21 | |
---|
| 22 | MPI_Comm_dup(contextComm, &contextComm_) ; |
---|
| 23 | xiosComm_=CXios::getXiosComm() ; |
---|
| 24 | |
---|
| 25 | MPI_Comm_rank(xiosComm_,&globalRank) ; |
---|
| 26 | MPI_Comm_rank(contextComm_,&localRank) ; |
---|
| 27 | |
---|
| 28 | winNotify_ = new CWindowManager(contextComm_, maxBufferSize_) ; |
---|
| 29 | |
---|
| 30 | int type; |
---|
| 31 | if (localRank==localLeader_) |
---|
| 32 | { |
---|
| 33 | globalLeader_=globalRank ; |
---|
| 34 | MPI_Comm_rank(contextComm_,&commSize) ; |
---|
| 35 | |
---|
| 36 | CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ; |
---|
| 37 | SRegisterContextInfo contextInfo = {poolId, serviceId, partitionId, type, contextId, commSize, globalLeader_} ; |
---|
| 38 | name_ = CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ; |
---|
| 39 | CXios::getContextsManager()->registerContext(name_, contextInfo) ; |
---|
| 40 | } |
---|
| 41 | MPI_Bcast(&type, 1, MPI_INT, localLeader_,contextComm_) ; |
---|
| 42 | name_ = CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ; |
---|
| 43 | context_=CContext::create(name_); |
---|
| 44 | |
---|
| 45 | context_->init(this, contextComm, type) ; |
---|
| 46 | |
---|
| 47 | info(10)<<"Context "<< CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId)<<" created, on local rank "<<localRank |
---|
| 48 | <<" and global rank "<<globalRank<<endl ; |
---|
| 49 | } |
---|
| 50 | |
---|
[1764] | 51 | CServerContext::~CServerContext() |
---|
| 52 | { |
---|
| 53 | |
---|
| 54 | } |
---|
| 55 | |
---|
[1761] | 56 | bool CServerContext::createIntercomm(const string& poolId, const string& serviceId, const int& partitionId, const string& contextId, |
---|
| 57 | const MPI_Comm& intraComm, MPI_Comm& interCommClient, MPI_Comm& interCommServer, bool wait) |
---|
| 58 | { |
---|
| 59 | int intraCommRank ; |
---|
| 60 | MPI_Comm_rank(intraComm, &intraCommRank) ; |
---|
| 61 | int contextLeader ; |
---|
| 62 | |
---|
| 63 | bool ok ; |
---|
| 64 | int type ; |
---|
| 65 | MPI_Comm newInterCommClient, newInterCommServer ; |
---|
| 66 | MPI_Comm_dup(contextComm_,&newInterCommClient) ; |
---|
| 67 | MPI_Comm_dup(contextComm_,&newInterCommServer) ; |
---|
| 68 | overlapedComm_[name_]=tuple<bool, MPI_Comm, MPI_Comm>(false, newInterCommClient, newInterCommServer) ; |
---|
| 69 | MPI_Barrier(contextComm_) ; |
---|
| 70 | |
---|
| 71 | if (intraCommRank==0) |
---|
| 72 | { |
---|
| 73 | ok=CXios::getContextsManager()->createServerContextIntercomm(poolId, serviceId, partitionId, contextId, name_, wait) ; |
---|
| 74 | if (ok) |
---|
| 75 | { |
---|
| 76 | CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ; |
---|
| 77 | string name=CXios::getContextsManager()->getServerContextName(poolId, serviceId, partitionId, type, contextId) ; |
---|
| 78 | CXios::getContextsManager()->getContextLeader(name, contextLeader) ; |
---|
| 79 | } |
---|
| 80 | } |
---|
| 81 | |
---|
[1764] | 82 | MPI_Request req ; |
---|
| 83 | MPI_Status status ; |
---|
| 84 | MPI_Ibarrier(intraComm,&req) ; |
---|
| 85 | |
---|
| 86 | int flag=false ; |
---|
| 87 | while(!flag) |
---|
[1761] | 88 | { |
---|
[1764] | 89 | CXios::getDaemonsManager()->servicesEventLoop() ; |
---|
| 90 | MPI_Test(&req,&flag,&status) ; |
---|
| 91 | } |
---|
[1765] | 92 | |
---|
[1761] | 93 | MPI_Bcast(&ok, 1, MPI_INT, 0, intraComm) ; |
---|
| 94 | |
---|
| 95 | if (ok) |
---|
| 96 | { |
---|
[1764] | 97 | int globalRank ; |
---|
| 98 | MPI_Comm_rank(xiosComm_,&globalRank) ; |
---|
| 99 | MPI_Bcast(&contextLeader, 1, MPI_INT, 0, intraComm) ; |
---|
| 100 | |
---|
| 101 | int overlap, nOverlap ; |
---|
| 102 | if (contextLeader==globalRank) overlap=1 ; |
---|
| 103 | else overlap=0 ; |
---|
| 104 | MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ; |
---|
| 105 | /* |
---|
[1761] | 106 | int overlap ; |
---|
| 107 | if (get<0>(overlapedComm_[name_])) overlap=1 ; |
---|
| 108 | else overlap=0 ; |
---|
| 109 | |
---|
| 110 | int nOverlap ; |
---|
| 111 | MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ; |
---|
| 112 | int commSize ; |
---|
| 113 | MPI_Comm_size(contextComm_,&commSize ) ; |
---|
[1764] | 114 | */ |
---|
| 115 | if (nOverlap> 0 ) |
---|
[1761] | 116 | { |
---|
[1764] | 117 | while (get<0>(overlapedComm_[name_])==false) CXios::getDaemonsManager()->servicesEventLoop() ; |
---|
| 118 | isAttachedMode_=true ; |
---|
[1761] | 119 | cout<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ; |
---|
| 120 | interCommClient=newInterCommClient ; |
---|
| 121 | interCommServer=newInterCommServer ; |
---|
| 122 | } |
---|
| 123 | else if (nOverlap==0) |
---|
| 124 | { |
---|
| 125 | cout<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ; |
---|
[1764] | 126 | isAttachedMode_=false ; |
---|
[1761] | 127 | MPI_Intercomm_create(intraComm, 0, xiosComm_, contextLeader, 3141, &interCommClient) ; |
---|
| 128 | MPI_Comm_dup(interCommClient, &interCommServer) ; |
---|
| 129 | MPI_Comm_free(&newInterCommClient) ; |
---|
| 130 | MPI_Comm_free(&newInterCommServer) ; |
---|
| 131 | } |
---|
| 132 | else |
---|
| 133 | { |
---|
| 134 | cout<<"CServerContext::createIntercomm : partial overlap ==> not managed"<<endl ; |
---|
| 135 | } |
---|
| 136 | } |
---|
| 137 | overlapedComm_.erase(name_) ; |
---|
| 138 | return ok ; |
---|
| 139 | } |
---|
| 140 | |
---|
| 141 | |
---|
| 142 | void CServerContext::createIntercomm(int remoteLeader, const string& sourceContext) |
---|
| 143 | { |
---|
| 144 | int commSize ; |
---|
| 145 | MPI_Comm_size(contextComm_,&commSize) ; |
---|
| 146 | for(int rank=0; rank<commSize; rank++) |
---|
| 147 | { |
---|
[1764] | 148 | notifyOutType_=NOTIFY_CREATE_INTERCOMM ; |
---|
| 149 | notifyOutCreateIntercomm_ = make_tuple(remoteLeader, sourceContext) ; |
---|
[1761] | 150 | sendNotification(rank) ; |
---|
| 151 | } |
---|
| 152 | } |
---|
| 153 | |
---|
| 154 | void CServerContext::sendNotification(int rank) |
---|
| 155 | { |
---|
| 156 | winNotify_->lockWindow(rank,0) ; |
---|
| 157 | winNotify_->pushToWindow(rank, this, &CServerContext::notificationsDumpOut) ; |
---|
| 158 | winNotify_->unlockWindow(rank,0) ; |
---|
| 159 | } |
---|
| 160 | |
---|
| 161 | |
---|
| 162 | void CServerContext::notificationsDumpOut(CBufferOut& buffer) |
---|
| 163 | { |
---|
| 164 | |
---|
| 165 | buffer.realloc(maxBufferSize_) ; |
---|
| 166 | |
---|
[1764] | 167 | if (notifyOutType_==NOTIFY_CREATE_INTERCOMM) |
---|
[1761] | 168 | { |
---|
[1764] | 169 | auto& arg=notifyOutCreateIntercomm_ ; |
---|
| 170 | buffer << notifyOutType_ << std::get<0>(arg)<<std::get<1>(arg) ; |
---|
[1761] | 171 | } |
---|
| 172 | } |
---|
| 173 | |
---|
| 174 | void CServerContext::notificationsDumpIn(CBufferIn& buffer) |
---|
| 175 | { |
---|
[1764] | 176 | if (buffer.bufferSize() == 0) notifyInType_= NOTIFY_NOTHING ; |
---|
[1761] | 177 | else |
---|
| 178 | { |
---|
[1764] | 179 | buffer>>notifyInType_; |
---|
| 180 | if (notifyInType_==NOTIFY_CREATE_INTERCOMM) |
---|
[1761] | 181 | { |
---|
[1764] | 182 | auto& arg=notifyInCreateIntercomm_ ; |
---|
[1761] | 183 | buffer >> std::get<0>(arg)>> std::get<1>(arg) ; |
---|
| 184 | } |
---|
| 185 | } |
---|
| 186 | } |
---|
| 187 | |
---|
| 188 | void CServerContext::checkNotifications(void) |
---|
| 189 | { |
---|
[1764] | 190 | if (!hasNotification_) |
---|
| 191 | { |
---|
| 192 | int commRank ; |
---|
| 193 | MPI_Comm_rank(contextComm_, &commRank) ; |
---|
| 194 | winNotify_->lockWindow(commRank,0) ; |
---|
| 195 | winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ; |
---|
| 196 | winNotify_->unlockWindow(commRank,0) ; |
---|
| 197 | |
---|
| 198 | if (notifyInType_!= NOTIFY_NOTHING) |
---|
| 199 | { |
---|
| 200 | hasNotification_=true ; |
---|
| 201 | auto eventScheduler=parentService_->getEventScheduler() ; |
---|
| 202 | std::hash<string> hashString ; |
---|
| 203 | size_t hashId = hashString(name_) ; |
---|
| 204 | size_t currentTimeLine=0 ; |
---|
| 205 | eventScheduler->registerEvent(currentTimeLine,hashId); |
---|
| 206 | } |
---|
| 207 | } |
---|
| 208 | |
---|
| 209 | if (hasNotification_) |
---|
| 210 | { |
---|
| 211 | auto eventScheduler=parentService_->getEventScheduler() ; |
---|
| 212 | std::hash<string> hashString ; |
---|
| 213 | size_t hashId = hashString(name_) ; |
---|
| 214 | size_t currentTimeLine=0 ; |
---|
| 215 | if (eventScheduler->queryEvent(currentTimeLine,hashId)) |
---|
| 216 | { |
---|
| 217 | if (notifyInType_==NOTIFY_CREATE_INTERCOMM) createIntercomm() ; |
---|
| 218 | hasNotification_=false ; |
---|
| 219 | } |
---|
| 220 | } |
---|
[1761] | 221 | } |
---|
| 222 | |
---|
[1764] | 223 | bool CServerContext::eventLoop(bool serviceOnly) |
---|
[1761] | 224 | { |
---|
| 225 | bool finished=false ; |
---|
[1764] | 226 | if (winNotify_!=nullptr) checkNotifications() ; |
---|
| 227 | if (!serviceOnly && context_!=nullptr) |
---|
[1761] | 228 | { |
---|
| 229 | if (context_->eventLoop()) |
---|
| 230 | { |
---|
| 231 | context_=nullptr ; |
---|
| 232 | // destroy context ??? --> later |
---|
| 233 | } |
---|
| 234 | } |
---|
| 235 | |
---|
| 236 | if (context_==nullptr && finalizeSignal_) finished=true ; |
---|
| 237 | return finished ; |
---|
| 238 | } |
---|
| 239 | |
---|
| 240 | void CServerContext::createIntercomm(void) |
---|
| 241 | { |
---|
| 242 | MPI_Comm interCommServer, interCommClient ; |
---|
[1764] | 243 | auto& arg=notifyInCreateIntercomm_ ; |
---|
| 244 | int remoteLeader=get<0>(arg) ; |
---|
| 245 | string sourceContext=get<1>(arg) ; |
---|
[1761] | 246 | |
---|
| 247 | auto it=overlapedComm_.find(sourceContext) ; |
---|
| 248 | int overlap=0 ; |
---|
| 249 | if (it!=overlapedComm_.end()) |
---|
| 250 | { |
---|
| 251 | get<0>(it->second)=true ; |
---|
| 252 | overlap=1 ; |
---|
| 253 | } |
---|
| 254 | int nOverlap ; |
---|
| 255 | MPI_Allreduce(&overlap, &nOverlap, 1, MPI_INT, MPI_SUM, contextComm_) ; |
---|
| 256 | int commSize ; |
---|
| 257 | MPI_Comm_size(contextComm_,&commSize ) ; |
---|
| 258 | |
---|
| 259 | if (nOverlap==commSize) |
---|
| 260 | { |
---|
[1764] | 261 | info(10)<<"CServerContext::createIntercomm : total overlap ==> context in attached mode"<<endl ; |
---|
| 262 | isAttachedMode_=true ; |
---|
[1761] | 263 | interCommClient=get<2>(it->second) ; |
---|
| 264 | interCommServer=get<1>(it->second) ; |
---|
| 265 | context_ -> createClientInterComm(interCommClient, interCommServer ) ; |
---|
| 266 | clientsInterComm_.push_back(interCommClient) ; |
---|
| 267 | clientsInterComm_.push_back(interCommServer) ; |
---|
| 268 | } |
---|
| 269 | else if (nOverlap==0) |
---|
| 270 | { |
---|
[1764] | 271 | info(10)<<"CServerContext::createIntercomm : No overlap ==> context in server mode"<<endl ; |
---|
| 272 | isAttachedMode_=false ; |
---|
[1761] | 273 | MPI_Intercomm_create(contextComm_, 0, xiosComm_, remoteLeader, 3141, &interCommServer) ; |
---|
| 274 | MPI_Comm_dup(interCommServer,&interCommClient) ; |
---|
| 275 | context_ -> createClientInterComm(interCommClient,interCommServer) ; |
---|
| 276 | clientsInterComm_.push_back(interCommClient) ; |
---|
| 277 | clientsInterComm_.push_back(interCommServer) ; |
---|
| 278 | } |
---|
| 279 | else |
---|
| 280 | { |
---|
[1764] | 281 | ERROR("void CServerContext::createIntercomm(void)",<<"CServerContext::createIntercomm : partial overlap ==> not managed") ; |
---|
[1761] | 282 | } |
---|
| 283 | |
---|
| 284 | } |
---|
| 285 | |
---|
[1764] | 286 | void CServerContext::freeComm(void) |
---|
| 287 | { |
---|
| 288 | delete winNotify_ ; |
---|
| 289 | winNotify_=nullptr ; |
---|
| 290 | MPI_Comm_free(&contextComm_) ; |
---|
| 291 | // don't forget intercomm -> later |
---|
| 292 | } |
---|
| 293 | |
---|
[1761] | 294 | void CServerContext::finalizeSignal(void) |
---|
| 295 | { |
---|
| 296 | finalizeSignal_=true ; |
---|
| 297 | } |
---|
| 298 | |
---|
| 299 | } |
---|