[688] | 1 | #include <cassert> |
---|
| 2 | #include "node.hpp" |
---|
| 3 | #include "timerRemap.hpp" |
---|
| 4 | #include "circle.hpp" |
---|
| 5 | #include "meshutil.hpp" |
---|
| 6 | #include "polyg.hpp" |
---|
| 7 | #include "gridRemap.hpp" |
---|
| 8 | #include "intersect.hpp" |
---|
| 9 | #include "errhandle.hpp" |
---|
| 10 | #include "mpi_routing.hpp" |
---|
| 11 | #include "misc.hpp" |
---|
| 12 | |
---|
| 13 | #include "parallel_tree.hpp" |
---|
| 14 | |
---|
| 15 | namespace sphereRemap { |
---|
| 16 | |
---|
| 17 | static const int assignLevel = 2; |
---|
| 18 | |
---|
| 19 | // only the circle is packed, rest of node will be initialized on unpacking |
---|
| 20 | static void packNode(Node& node, char *buffer, int& index) |
---|
| 21 | { |
---|
| 22 | if (buffer == NULL) // compute size only |
---|
| 23 | index += 4 * sizeof(double); |
---|
| 24 | else |
---|
| 25 | { |
---|
| 26 | *(Coord *)(&buffer[index]) = node.centre; |
---|
| 27 | index += sizeof(Coord); |
---|
| 28 | |
---|
| 29 | *(double *)(&buffer[index]) = node.radius; |
---|
| 30 | index += sizeof(double); |
---|
| 31 | } |
---|
| 32 | } |
---|
| 33 | |
---|
| 34 | static void unpackNode(Node& node, char *buffer, int& index) |
---|
| 35 | { |
---|
| 36 | Coord centre; |
---|
| 37 | double r; |
---|
| 38 | |
---|
| 39 | centre = *(Coord *)(&buffer[index]); |
---|
| 40 | index += sizeof(Coord); |
---|
| 41 | |
---|
| 42 | r = *(double *)(&buffer[index]); |
---|
| 43 | index += sizeof(double); |
---|
| 44 | |
---|
| 45 | node.centre = centre; |
---|
| 46 | node.radius = r; |
---|
| 47 | } |
---|
| 48 | |
---|
| 49 | |
---|
| 50 | static void packElement(Elt *ptElement, char *buffer, int& index) |
---|
| 51 | { |
---|
| 52 | if (buffer == NULL) index += packedPolygonSize(*ptElement); |
---|
| 53 | else packPolygon(*ptElement, buffer, index); |
---|
| 54 | } |
---|
| 55 | |
---|
| 56 | static void unpackElement(Elt *ptElement, char *buffer, int& index) |
---|
| 57 | { |
---|
| 58 | unpackPolygon(*ptElement, buffer, index); |
---|
| 59 | } |
---|
| 60 | |
---|
| 61 | static void packVector(const vector<int>& vec, char *buf, int& pos) |
---|
| 62 | { |
---|
| 63 | if (buf == NULL) |
---|
| 64 | pos += sizeof(int) + vec.size()*sizeof(int); |
---|
| 65 | else |
---|
| 66 | { |
---|
| 67 | *((int *) &(buf[pos])) = vec.size(); |
---|
| 68 | pos += sizeof(int); |
---|
| 69 | for (int i = 0; i < vec.size(); i++) |
---|
| 70 | { |
---|
| 71 | *((int *) &(buf[pos])) = vec[i]; |
---|
| 72 | pos += sizeof(int); |
---|
| 73 | } |
---|
| 74 | } |
---|
| 75 | } |
---|
| 76 | |
---|
| 77 | static void unpackVector(vector<int>& vec, const char *buf, int& pos) |
---|
| 78 | { |
---|
| 79 | vec.resize(*((int *) &(buf[pos]))); |
---|
| 80 | pos += sizeof(int); |
---|
| 81 | for (int i = 0; i < vec.size(); i++) |
---|
| 82 | { |
---|
| 83 | vec[i] = *((int *) &(buf[pos])); |
---|
| 84 | pos += sizeof(int); |
---|
| 85 | } |
---|
| 86 | } |
---|
| 87 | |
---|
| 88 | static void assignRoute(CSampleTree& tree, const CCascadeLevel& cl) // newroot <- root |
---|
| 89 | { |
---|
| 90 | vector<int> routeRank(cl.group_size); |
---|
| 91 | for (int i = 0; i < cl.group_size; i++) |
---|
| 92 | routeRank[i] = i; //(cl.group_size + cl.polour() - i) % cl.group_size; |
---|
| 93 | std::vector<int>::iterator rank = routeRank.begin(); |
---|
| 94 | tree.root->assignRoute(rank, assignLevel); |
---|
| 95 | } |
---|
| 96 | |
---|
| 97 | static void transferRoutedNodes(CMPIRouting& MPIRoute, /*const*/ vector<Node>& nodeSend, const vector<int>& route, vector<Node>& nodeRecv) |
---|
| 98 | { |
---|
| 99 | /* `route` knows what goes where */ |
---|
| 100 | MPIRoute.init(route); |
---|
| 101 | int nRecv = MPIRoute.getTotalSourceElement(); |
---|
| 102 | nodeRecv.resize(nRecv); |
---|
| 103 | MPIRoute.transferToTarget(&nodeSend[0], &nodeRecv[0], packNode, unpackNode); |
---|
| 104 | } |
---|
| 105 | |
---|
| 106 | static void transferRoutedIntersections(CMPIRouting& MPIRoute, const vector<Node>& nodeSend, const vector<vector<int> >& route, vector<Node>& nodeRecv) |
---|
| 107 | { |
---|
| 108 | // `route` knows what goes where |
---|
| 109 | MPIRoute.init(route); |
---|
| 110 | int nRecv = MPIRoute.getTotalSourceElement(); |
---|
| 111 | nodeRecv.resize(nRecv); |
---|
| 112 | MPIRoute.transferToTarget((Node * /*mpi wants non-const*/) &nodeSend[0], &nodeRecv[0], packNode, unpackNode); |
---|
| 113 | //cout << MPIRoute.mpiRank << " ROUTE " << nRecv << ": " << nodeSend.size() << " " << nodeRecv.size() << " "; |
---|
| 114 | } |
---|
| 115 | |
---|
[923] | 116 | //CParallelTree::CParallelTree(MPI_Comm comm) : communicator(comm), cascade(MIN_NODE_SZ*MIN_NODE_SZ, comm) |
---|
[1639] | 117 | CParallelTree::CParallelTree(MPI_Comm comm) : communicator(comm), cascade(MAX_NODE_SZ*MAX_NODE_SZ*2, comm) |
---|
[688] | 118 | { |
---|
| 119 | treeCascade.reserve(cascade.num_levels); |
---|
| 120 | for (int lev = 0; lev < cascade.num_levels; lev++) |
---|
| 121 | treeCascade.push_back(CSampleTree(cascade.level[lev].group_size, assignLevel)); |
---|
| 122 | } |
---|
| 123 | |
---|
| 124 | void CParallelTree::buildSampleTreeCascade(vector<Node>& sampleNodes /*route field will be modified*/, int level) |
---|
| 125 | { |
---|
| 126 | buildSampleTree(treeCascade[level], sampleNodes, cascade.level[level]); |
---|
| 127 | assignRoute(treeCascade[level] /*out*/, cascade.level[level] /*in*/); |
---|
| 128 | |
---|
| 129 | if (level+1 < cascade.num_levels) |
---|
| 130 | { |
---|
| 131 | vector<int> route(sampleNodes.size()); |
---|
| 132 | treeCascade[level].routeNodes(route, sampleNodes, assignLevel); |
---|
| 133 | |
---|
| 134 | vector<Node> routedNodes; |
---|
| 135 | CMPIRouting mpiRoute(cascade.level[level].pg_comm); |
---|
| 136 | transferRoutedNodes(mpiRoute, sampleNodes, route, routedNodes); |
---|
| 137 | buildSampleTreeCascade(routedNodes, level+1); |
---|
| 138 | } |
---|
| 139 | } |
---|
| 140 | |
---|
| 141 | void buildSampleTree(CSampleTree& tree, const vector<Node>& node, const CCascadeLevel& comm) |
---|
| 142 | /* |
---|
| 143 | In the beginning all the sample elements are distributed |
---|
| 144 | -> communicate to make available at each rank |
---|
| 145 | so that each rank can build the same sample tree |
---|
| 146 | */ |
---|
| 147 | { |
---|
| 148 | int n = node.size(); // number of samples intially on this rank (local) |
---|
| 149 | |
---|
| 150 | int blocSize = comm.group_size * ipow(MAX_NODE_SZ, assignLevel); |
---|
| 151 | |
---|
| 152 | int nrecv; // global number of samples THIS WILL BE THE NUMBER OF LEAFS IN THE SAMPLE TREE |
---|
[1639] | 153 | MPI_Allreduce(&n, &nrecv, 1, MPI_INT, MPI_SUM, comm.comm); // => size of sample tree does not depend on keepNodes! |
---|
[688] | 154 | double ratio = blocSize / (1.0 * nrecv); |
---|
| 155 | int nsend = ratio * n + 1; // nsend = n_local_samples / n_global_samples * blocksize + 1 = blocksize/comm.size |
---|
| 156 | if (nsend > n) nsend = n; |
---|
| 157 | |
---|
| 158 | int *counts = new int[comm.size]; |
---|
[1639] | 159 | MPI_Allgather(&nsend, 1, MPI_INT, counts, 1, MPI_INT, comm.comm); |
---|
[688] | 160 | |
---|
| 161 | nrecv = 0; |
---|
| 162 | int *displs = new int[comm.size]; |
---|
| 163 | for (int i = 0; i < comm.size; i++) |
---|
| 164 | { |
---|
| 165 | displs[i] = 4 * nrecv; |
---|
| 166 | nrecv += counts[i]; |
---|
| 167 | counts[i] = 4 * counts[i]; |
---|
| 168 | } |
---|
| 169 | |
---|
| 170 | /* pack circles around sample elements */ |
---|
| 171 | double *sendBuffer = new double[nsend*4]; |
---|
| 172 | int index = 0; |
---|
| 173 | vector<int> randomArray(n); |
---|
| 174 | randomizeArray(randomArray); |
---|
| 175 | for (int i = 0; i < nsend; i++) |
---|
| 176 | { |
---|
| 177 | const Node& no = node[randomArray[i]]; |
---|
| 178 | *((Coord *) (sendBuffer + index)) = no.centre; |
---|
| 179 | index += sizeof(Coord)/sizeof(*sendBuffer); |
---|
| 180 | sendBuffer[index++] = no.radius; |
---|
| 181 | } |
---|
| 182 | |
---|
| 183 | /* each process needs the sample elements from all processes */ |
---|
| 184 | double *recvBuffer = new double[nrecv*4]; |
---|
[1639] | 185 | MPI_Allgatherv(sendBuffer, 4 * nsend, MPI_DOUBLE, recvBuffer, counts, displs, MPI_DOUBLE, comm.comm); |
---|
[688] | 186 | delete[] sendBuffer; |
---|
| 187 | delete[] counts; |
---|
| 188 | delete[] displs; |
---|
| 189 | |
---|
| 190 | /* unpack */ |
---|
[923] | 191 | /* |
---|
[688] | 192 | randomArray.resize(nrecv); |
---|
| 193 | randomizeArray(randomArray); |
---|
| 194 | tree.leafs.resize(nrecv); |
---|
| 195 | index = 0; |
---|
[923] | 196 | |
---|
| 197 | |
---|
| 198 | for (int i = 0; i < nrecv; i++) |
---|
[688] | 199 | { |
---|
| 200 | Coord x = *(Coord *)(&recvBuffer[index]); |
---|
| 201 | index += sizeof(Coord)/sizeof(*recvBuffer); |
---|
| 202 | double radius = recvBuffer[index++]; |
---|
| 203 | tree.leafs[randomArray[i]].centre = x; |
---|
| 204 | tree.leafs[randomArray[i]].radius = radius; |
---|
| 205 | |
---|
| 206 | } |
---|
[923] | 207 | */ |
---|
[688] | 208 | |
---|
[923] | 209 | randomArray.resize(blocSize); |
---|
| 210 | randomizeArray(randomArray); |
---|
| 211 | tree.leafs.resize(blocSize); |
---|
| 212 | index = 0; |
---|
| 213 | |
---|
| 214 | size_t s=(sizeof(Coord)/sizeof(*recvBuffer)+1)*nrecv ; |
---|
| 215 | |
---|
| 216 | for (int i = 0; i < blocSize; i++) |
---|
| 217 | { |
---|
| 218 | Coord x = *(Coord *)(&recvBuffer[index%s]); |
---|
| 219 | index += sizeof(Coord)/sizeof(*recvBuffer); |
---|
| 220 | double radius = recvBuffer[index%s]; |
---|
| 221 | index++ ; |
---|
| 222 | tree.leafs[randomArray[i]].centre = x; |
---|
| 223 | tree.leafs[randomArray[i]].radius = radius; |
---|
| 224 | |
---|
| 225 | } |
---|
| 226 | |
---|
| 227 | |
---|
[688] | 228 | delete [] recvBuffer; |
---|
| 229 | |
---|
| 230 | CTimer::get("buildSampleTree(local)").resume(); |
---|
| 231 | tree.build(tree.leafs); |
---|
[923] | 232 | // cout << "SampleTree build : assign Level " << assignLevel << " nb Nodes : " << tree.levelSize[assignLevel] << endl; |
---|
[688] | 233 | CTimer::get("buildSampleTree(local)").suspend(); |
---|
| 234 | CTimer::get("buildSampleTree(local)").print(); |
---|
| 235 | |
---|
| 236 | /* End gracefully if sample tree could not be constructed with desired number of nodes on assignLevel */ |
---|
| 237 | int allok, ok = (tree.levelSize[assignLevel] == comm.group_size); |
---|
| 238 | if (!ok) |
---|
| 239 | { |
---|
[923] | 240 | cerr << comm.rank << ": PROBLEM: (node assign)" << tree.levelSize[assignLevel] << " != " << comm.group_size << " (keepNodes)" |
---|
| 241 | << " node size : "<<node.size()<<" bloc size : "<<blocSize<<" total number of leaf : "<<tree.leafs.size()<<endl ; |
---|
[688] | 242 | /* |
---|
[1639] | 243 | MPI_Allreduce(&ok, &allok, 1, MPI_INT, MPI_PROD, communicator); |
---|
[688] | 244 | if (!allok) { |
---|
| 245 | MPI_Finalize(); |
---|
| 246 | exit(1); |
---|
| 247 | } |
---|
| 248 | */ |
---|
[1639] | 249 | MPI_Abort(MPI_COMM_WORLD,-1) ; |
---|
[688] | 250 | } |
---|
[923] | 251 | /* |
---|
[688] | 252 | cout<<"*******************************************"<<endl ; |
---|
| 253 | cout<<"******* Sample Tree output *********"<<endl ; |
---|
| 254 | cout<<"*******************************************"<<endl ; |
---|
| 255 | tree.output(cout,1) ; |
---|
| 256 | |
---|
| 257 | cout<<"*******************************************"<<endl ; |
---|
[923] | 258 | */ |
---|
[688] | 259 | |
---|
| 260 | assert(tree.root->incluCheck() == 0); |
---|
| 261 | } |
---|
| 262 | |
---|
| 263 | |
---|
| 264 | void CParallelTree::buildLocalTree(const vector<Node>& node, const vector<int>& route) |
---|
| 265 | { |
---|
| 266 | CMPIRouting MPIRoute(communicator); |
---|
[1639] | 267 | MPI_Barrier(communicator); |
---|
[688] | 268 | CTimer::get("buildLocalTree(initRoute)").resume(); |
---|
| 269 | MPIRoute.init(route); |
---|
| 270 | CTimer::get("buildLocalTree(initRoute)").suspend(); |
---|
| 271 | CTimer::get("buildLocalTree(initRoute)").print(); |
---|
| 272 | |
---|
| 273 | nbLocalElements = MPIRoute.getTotalSourceElement(); |
---|
| 274 | localElements = new Elt[nbLocalElements]; |
---|
| 275 | |
---|
| 276 | vector<Elt*> ptElement(node.size()); |
---|
| 277 | for (int i = 0; i < node.size(); i++) |
---|
| 278 | ptElement[i] = (Elt *) (node[i].data); |
---|
| 279 | |
---|
| 280 | vector<Elt*> ptLocalElement(nbLocalElements); |
---|
| 281 | for (int i = 0; i < nbLocalElements; i++) |
---|
| 282 | ptLocalElement[i] = &localElements[i]; |
---|
| 283 | |
---|
| 284 | CTimer::get("buildLocalTree(transfer)").resume(); |
---|
| 285 | MPIRoute.transferToTarget(&ptElement[0], &ptLocalElement[0], packElement, unpackElement); |
---|
| 286 | CTimer::get("buildLocalTree(transfer)").suspend(); |
---|
| 287 | CTimer::get("buildLocalTree(transfer)").print(); |
---|
| 288 | |
---|
| 289 | CTimer::get("buildLocalTree(local)").resume(); |
---|
| 290 | |
---|
| 291 | int mpiRank; |
---|
[1639] | 292 | MPI_Comm_rank(communicator, &mpiRank); |
---|
[688] | 293 | localTree.leafs.reserve(nbLocalElements); |
---|
| 294 | for (int i = 0; i < nbLocalElements; i++) |
---|
| 295 | { |
---|
| 296 | Elt& elt = localElements[i]; |
---|
| 297 | elt.id.ind = i; |
---|
| 298 | elt.id.rank = mpiRank; |
---|
| 299 | localTree.leafs.push_back(Node(elt.x, cptRadius(elt), &localElements[i])); |
---|
| 300 | } |
---|
| 301 | localTree.build(localTree.leafs); |
---|
| 302 | |
---|
| 303 | cptAllEltsGeom(localElements, nbLocalElements, srcGrid.pole); |
---|
| 304 | CTimer::get("buildLocalTree(local)").suspend(); |
---|
| 305 | CTimer::get("buildLocalTree(local)").print(); |
---|
| 306 | } |
---|
| 307 | |
---|
| 308 | void CParallelTree::build(vector<Node>& node, vector<Node>& node2) |
---|
| 309 | { |
---|
| 310 | |
---|
| 311 | int assignLevel = 2; |
---|
| 312 | int nbSampleNodes = 2*ipow(MAX_NODE_SZ + 1, assignLevel); |
---|
| 313 | |
---|
[852] | 314 | |
---|
| 315 | long int nb1, nb2, nb, nbTot ; |
---|
[898] | 316 | nb1=node.size() ; nb2=node2.size() ; |
---|
[852] | 317 | nb=nb1+nb2 ; |
---|
[1639] | 318 | MPI_Allreduce(&nb, &nbTot, 1, MPI_LONG, MPI_SUM, communicator) ; |
---|
[852] | 319 | int commSize ; |
---|
[1639] | 320 | MPI_Comm_size(communicator,&commSize) ; |
---|
[852] | 321 | |
---|
[688] | 322 | // make multiple of two |
---|
| 323 | nbSampleNodes /= 2; |
---|
| 324 | nbSampleNodes *= 2; |
---|
[898] | 325 | // assert( nbTot > nbSampleNodes*commSize) ; |
---|
[852] | 326 | |
---|
[853] | 327 | int nbSampleNodes1 = nbSampleNodes * (nb1*commSize)/(1.*nbTot) ; |
---|
| 328 | int nbSampleNodes2 = nbSampleNodes * (nb2*commSize)/(1.*nbTot) ; |
---|
[852] | 329 | |
---|
[688] | 330 | |
---|
[812] | 331 | // assert(node.size() > nbSampleNodes); |
---|
| 332 | // assert(node2.size() > nbSampleNodes); |
---|
[852] | 333 | // assert(node.size() + node2.size() > nbSampleNodes); |
---|
| 334 | vector<Node> sampleNodes; sampleNodes.reserve(nbSampleNodes1+nbSampleNodes2); |
---|
[688] | 335 | |
---|
| 336 | vector<int> randomArray1(node.size()); |
---|
| 337 | randomizeArray(randomArray1); |
---|
| 338 | vector<int> randomArray2(node2.size()); |
---|
| 339 | randomizeArray(randomArray2); |
---|
| 340 | |
---|
[852] | 341 | /* |
---|
[812] | 342 | int s1,s2 ; |
---|
| 343 | |
---|
| 344 | if (node.size()< nbSampleNodes/2) |
---|
| 345 | { |
---|
| 346 | s1 = node.size() ; |
---|
| 347 | s2 = nbSampleNodes-s1 ; |
---|
| 348 | } |
---|
| 349 | else if (node2.size()< nbSampleNodes/2) |
---|
| 350 | { |
---|
| 351 | s2 = node.size() ; |
---|
| 352 | s1 = nbSampleNodes-s2 ; |
---|
| 353 | } |
---|
| 354 | else |
---|
| 355 | { |
---|
| 356 | s1=nbSampleNodes/2 ; |
---|
| 357 | s2=nbSampleNodes/2 ; |
---|
| 358 | } |
---|
[852] | 359 | */ |
---|
[898] | 360 | for (int i = 0; i <nbSampleNodes1; i++) sampleNodes.push_back(Node(node[randomArray1[i%nb1]].centre, node[randomArray1[i%nb1]].radius, NULL)); |
---|
| 361 | for (int i = 0; i <nbSampleNodes2; i++) sampleNodes.push_back(Node(node2[randomArray2[i%nb2]].centre, node2[randomArray2[i%nb2]].radius, NULL)); |
---|
[812] | 362 | |
---|
| 363 | /* |
---|
| 364 | for (int i = 0; i < nbSampleNodes/2; i++) |
---|
[688] | 365 | { |
---|
[812] | 366 | sampleNodes.push_back(Node(node[randomArray1[i]].centre, node[randomArray1[i]].radius, NULL)); |
---|
| 367 | sampleNodes.push_back(Node(node2[randomArray2[i]].centre, node2[randomArray2[i]].radius, NULL)); |
---|
[688] | 368 | } |
---|
[812] | 369 | */ |
---|
[688] | 370 | CTimer::get("buildParallelSampleTree").resume(); |
---|
| 371 | //sampleTree.buildParallelSampleTree(sampleNodes, cascade); |
---|
| 372 | buildSampleTreeCascade(sampleNodes); |
---|
| 373 | CTimer::get("buildParallelSampleTree").suspend(); |
---|
| 374 | CTimer::get("buildParallelSampleTree").print(); |
---|
| 375 | |
---|
| 376 | //route source mesh |
---|
| 377 | CTimer::get("parallelRouteNode").resume(); |
---|
| 378 | vector<int> route(node.size()); |
---|
| 379 | routeNodes(route /*out*/, node); |
---|
| 380 | CTimer::get("parallelRouteNode").suspend(); |
---|
| 381 | CTimer::get("parallelRouteNode").print(); |
---|
| 382 | |
---|
| 383 | CTimer::get("buildLocalTree").resume(); |
---|
| 384 | buildLocalTree(node, route); |
---|
| 385 | CTimer::get("buildLocalTree").suspend(); |
---|
| 386 | CTimer::get("buildLocalTree").print(); |
---|
| 387 | |
---|
| 388 | CTimer::get("buildRouteTree").resume(); |
---|
| 389 | /* update circles of tree cascade so it can be used for routing */ |
---|
| 390 | updateCirclesForRouting(localTree.root->centre, localTree.root->radius); |
---|
| 391 | CTimer::get("buildRouteTree").suspend(); |
---|
| 392 | CTimer::get("buildRouteTree").print(); |
---|
| 393 | } |
---|
| 394 | |
---|
| 395 | void CParallelTree::routeNodes(vector<int>& route, vector<Node>& nodes /*route field used*/, int level) |
---|
| 396 | { |
---|
| 397 | treeCascade[level].routeNodes(route /*assign*/, nodes, assignLevel); |
---|
| 398 | |
---|
| 399 | if (level+1 < cascade.num_levels) |
---|
| 400 | { |
---|
| 401 | vector<Node> routedNodes; |
---|
| 402 | CMPIRouting MPIRoute(cascade.level[level].pg_comm); |
---|
| 403 | transferRoutedNodes(MPIRoute, nodes, route /*use*/, routedNodes); |
---|
| 404 | vector<int> globalRank(routedNodes.size()); |
---|
| 405 | routeNodes(globalRank, routedNodes, level + 1); |
---|
| 406 | MPIRoute.transferFromSource(&route[0] /*override*/, &globalRank[0]); |
---|
| 407 | } |
---|
| 408 | else |
---|
| 409 | { |
---|
| 410 | CMPIRouting MPIRoute(cascade.level[level].comm); // or use pg_comm, on last cascade level identical |
---|
| 411 | MPIRoute.init(route); |
---|
| 412 | int nbRecvNode = MPIRoute.getTotalSourceElement(); |
---|
| 413 | vector<int> globalRank(nbRecvNode); |
---|
| 414 | for (int i = 0; i < globalRank.size(); i++) |
---|
| 415 | globalRank[i] = cascade.level[0].rank; |
---|
| 416 | MPIRoute.transferFromSource(&route[0] /*override*/, &globalRank[0]); |
---|
| 417 | } |
---|
| 418 | } |
---|
| 419 | |
---|
| 420 | /* assume `to` to be empty vector at entry */ |
---|
| 421 | void linearize(const vector<vector<int> >& from, vector<int>& to) |
---|
| 422 | { |
---|
| 423 | int cnt = 0; |
---|
| 424 | for (int i = 0; i < from.size(); ++i) |
---|
| 425 | cnt += from[i].size(); |
---|
| 426 | to.resize(cnt); |
---|
| 427 | vector<int>::iterator dest = to.begin(); |
---|
| 428 | for (int i = 0; i < from.size(); ++i) |
---|
| 429 | dest = copy(from[i].begin(), from[i].end(), dest); |
---|
| 430 | } |
---|
| 431 | |
---|
| 432 | /* at entry `to` must already have it's final shape and only values are overriden */ |
---|
| 433 | void delinearize(const vector<int>& from, vector<vector<int> >& to) |
---|
| 434 | { |
---|
| 435 | vector<int>::const_iterator end, src = from.begin(); |
---|
| 436 | for (int i = 0; i < to.size(); ++i, src=end) |
---|
| 437 | copy(src, end = src + to[i].size(), to[i].begin()); |
---|
| 438 | } |
---|
| 439 | |
---|
| 440 | void CParallelTree::routeIntersections(vector<vector<int> >& routes, vector<Node>& nodes, int level) |
---|
| 441 | { |
---|
| 442 | treeCascade[level].routeIntersections(routes /*assign*/, nodes); |
---|
| 443 | |
---|
| 444 | if (level+1 < cascade.num_levels) |
---|
| 445 | { |
---|
| 446 | vector<Node> routedNodes; |
---|
| 447 | CMPIRouting MPIRoute(cascade.level[level].pg_comm); |
---|
| 448 | |
---|
| 449 | vector<int> flattenedRoutes1; |
---|
| 450 | linearize(routes, flattenedRoutes1); |
---|
| 451 | vector<Node> double_nodes(flattenedRoutes1.size()); |
---|
| 452 | int j = 0; |
---|
| 453 | for (int i = 0; i < routes.size(); ++i) |
---|
| 454 | for (int k = 0; k < routes[i].size(); ++k, ++j) |
---|
| 455 | double_nodes[j] = nodes[i]; |
---|
| 456 | transferRoutedNodes(MPIRoute, double_nodes, flattenedRoutes1 /*use*/, routedNodes); |
---|
| 457 | vector<vector<int> > globalRanks(routedNodes.size()); |
---|
| 458 | routeIntersections(globalRanks /*out*/, routedNodes /*in*/, level + 1); |
---|
| 459 | vector<vector<int> > flattenedRoutes(flattenedRoutes1.size()); |
---|
| 460 | // transferFromSource expects sizes (nbSendNode=flattenedRoutes, nbRecvNode=routedNodes.size()) |
---|
| 461 | MPIRoute.transferFromSource(&flattenedRoutes[0], &globalRanks[0], packVector, unpackVector); |
---|
| 462 | for (int i = 0, j = 0; i < routes.size(); ++i) |
---|
| 463 | { |
---|
| 464 | int old_size = routes[i].size(); |
---|
| 465 | routes[i].resize(0); |
---|
| 466 | for (int k = 0; k < old_size; ++k, ++j) |
---|
| 467 | for (int l = 0; l < flattenedRoutes[j].size(); ++l) |
---|
| 468 | routes[i].push_back(flattenedRoutes[j][l]); |
---|
| 469 | } |
---|
| 470 | assert(j == flattenedRoutes1.size()); |
---|
| 471 | |
---|
| 472 | } |
---|
| 473 | else |
---|
| 474 | { |
---|
| 475 | CMPIRouting MPIRoute(cascade.level[level].comm); |
---|
| 476 | MPIRoute.init(routes); |
---|
| 477 | int nbRecvNode = MPIRoute.getTotalSourceElement(); |
---|
| 478 | vector<int> globalRanks(nbRecvNode, cascade.level[0].rank); |
---|
| 479 | vector<int> flattened_routes; |
---|
| 480 | linearize(routes, flattened_routes); |
---|
| 481 | MPIRoute.transferFromSource(&flattened_routes[0], &globalRanks[0]); |
---|
| 482 | delinearize(flattened_routes, routes); |
---|
| 483 | } |
---|
| 484 | if (level!=level) |
---|
| 485 | { |
---|
| 486 | for (int i = 0; i < routes.size(); ++i) |
---|
| 487 | for (int k = 0; k < routes[i].size(); ++k) |
---|
| 488 | if (routes[i][k] == cascade.level[0].rank) routes[i].erase(routes[i].begin() + (k--)); |
---|
| 489 | } |
---|
| 490 | } |
---|
| 491 | |
---|
| 492 | void CParallelTree::updateCirclesForRouting(Coord rootCentre, double rootRadius, int level) |
---|
| 493 | { |
---|
| 494 | if (level + 1 < cascade.num_levels) // children in the cascade have to update first |
---|
| 495 | { |
---|
| 496 | updateCirclesForRouting(rootCentre, rootRadius, level + 1); |
---|
| 497 | rootCentre = treeCascade[level+1].root->centre; |
---|
| 498 | rootRadius = treeCascade[level+1].root->radius; |
---|
| 499 | } |
---|
| 500 | |
---|
| 501 | // gather circles on this level of the cascade |
---|
| 502 | int pg_size; |
---|
[1639] | 503 | MPI_Comm_size(cascade.level[level].pg_comm, &pg_size); |
---|
[688] | 504 | vector<Coord> allRootCentres(pg_size); |
---|
| 505 | vector<double> allRootRadia(pg_size); |
---|
[1639] | 506 | MPI_Allgather(&rootCentre, 3, MPI_DOUBLE, &allRootCentres[0], 3, MPI_DOUBLE, cascade.level[level].pg_comm); |
---|
| 507 | MPI_Allgather(&rootRadius, 1, MPI_DOUBLE, &allRootRadia[0], 1, MPI_DOUBLE, cascade.level[level].pg_comm); |
---|
[688] | 508 | |
---|
| 509 | // now allRootsRadia and allRootCentres must be inserted into second levels of us and propagated to root |
---|
| 510 | treeCascade[level].root->assignCircleAndPropagateUp(&allRootCentres[0], &allRootRadia[0], assignLevel); |
---|
| 511 | } |
---|
| 512 | |
---|
| 513 | CParallelTree::~CParallelTree() |
---|
| 514 | { |
---|
| 515 | delete [] localElements; |
---|
| 516 | } |
---|
| 517 | |
---|
| 518 | } |
---|