1 | #include <cassert> |
---|
2 | #include "node.hpp" |
---|
3 | #include "timerRemap.hpp" |
---|
4 | #include "circle.hpp" |
---|
5 | #include "meshutil.hpp" |
---|
6 | #include "polyg.hpp" |
---|
7 | #include "gridRemap.hpp" |
---|
8 | #include "intersect.hpp" |
---|
9 | #include "errhandle.hpp" |
---|
10 | #include "mpi_routing.hpp" |
---|
11 | #include "misc.hpp" |
---|
12 | |
---|
13 | #include "parallel_tree.hpp" |
---|
14 | |
---|
15 | namespace sphereRemap { |
---|
16 | |
---|
17 | static const int assignLevel = 2; |
---|
18 | |
---|
19 | // only the circle is packed, rest of node will be initialized on unpacking |
---|
20 | static void packNode(Node& node, char *buffer, int& index) |
---|
21 | { |
---|
22 | if (buffer == NULL) // compute size only |
---|
23 | index += 4 * sizeof(double); |
---|
24 | else |
---|
25 | { |
---|
26 | *(Coord *)(&buffer[index]) = node.centre; |
---|
27 | index += sizeof(Coord); |
---|
28 | |
---|
29 | *(double *)(&buffer[index]) = node.radius; |
---|
30 | index += sizeof(double); |
---|
31 | } |
---|
32 | } |
---|
33 | |
---|
34 | static void unpackNode(Node& node, char *buffer, int& index) |
---|
35 | { |
---|
36 | Coord centre; |
---|
37 | double r; |
---|
38 | |
---|
39 | centre = *(Coord *)(&buffer[index]); |
---|
40 | index += sizeof(Coord); |
---|
41 | |
---|
42 | r = *(double *)(&buffer[index]); |
---|
43 | index += sizeof(double); |
---|
44 | |
---|
45 | node.centre = centre; |
---|
46 | node.radius = r; |
---|
47 | } |
---|
48 | |
---|
49 | |
---|
50 | static void packElement(Elt *ptElement, char *buffer, int& index) |
---|
51 | { |
---|
52 | if (buffer == NULL) index += packedPolygonSize(*ptElement); |
---|
53 | else packPolygon(*ptElement, buffer, index); |
---|
54 | } |
---|
55 | |
---|
56 | static void unpackElement(Elt *ptElement, char *buffer, int& index) |
---|
57 | { |
---|
58 | unpackPolygon(*ptElement, buffer, index); |
---|
59 | } |
---|
60 | |
---|
61 | static void packVector(const vector<int>& vec, char *buf, int& pos) |
---|
62 | { |
---|
63 | if (buf == NULL) |
---|
64 | pos += sizeof(int) + vec.size()*sizeof(int); |
---|
65 | else |
---|
66 | { |
---|
67 | *((int *) &(buf[pos])) = vec.size(); |
---|
68 | pos += sizeof(int); |
---|
69 | for (int i = 0; i < vec.size(); i++) |
---|
70 | { |
---|
71 | *((int *) &(buf[pos])) = vec[i]; |
---|
72 | pos += sizeof(int); |
---|
73 | } |
---|
74 | } |
---|
75 | } |
---|
76 | |
---|
77 | static void unpackVector(vector<int>& vec, const char *buf, int& pos) |
---|
78 | { |
---|
79 | vec.resize(*((int *) &(buf[pos]))); |
---|
80 | pos += sizeof(int); |
---|
81 | for (int i = 0; i < vec.size(); i++) |
---|
82 | { |
---|
83 | vec[i] = *((int *) &(buf[pos])); |
---|
84 | pos += sizeof(int); |
---|
85 | } |
---|
86 | } |
---|
87 | |
---|
88 | static void assignRoute(CSampleTree& tree, const CCascadeLevel& cl) // newroot <- root |
---|
89 | { |
---|
90 | vector<int> routeRank(cl.group_size); |
---|
91 | for (int i = 0; i < cl.group_size; i++) |
---|
92 | routeRank[i] = i; //(cl.group_size + cl.polour() - i) % cl.group_size; |
---|
93 | std::vector<int>::iterator rank = routeRank.begin(); |
---|
94 | tree.root->assignRoute(rank, assignLevel); |
---|
95 | } |
---|
96 | |
---|
97 | static void transferRoutedNodes(CMPIRouting& MPIRoute, /*const*/ vector<Node>& nodeSend, const vector<int>& route, vector<Node>& nodeRecv) |
---|
98 | { |
---|
99 | /* `route` knows what goes where */ |
---|
100 | MPIRoute.init(route); |
---|
101 | int nRecv = MPIRoute.getTotalSourceElement(); |
---|
102 | nodeRecv.resize(nRecv); |
---|
103 | MPIRoute.transferToTarget(&nodeSend[0], &nodeRecv[0], packNode, unpackNode); |
---|
104 | } |
---|
105 | |
---|
106 | static void transferRoutedIntersections(CMPIRouting& MPIRoute, const vector<Node>& nodeSend, const vector<vector<int> >& route, vector<Node>& nodeRecv) |
---|
107 | { |
---|
108 | // `route` knows what goes where |
---|
109 | MPIRoute.init(route); |
---|
110 | int nRecv = MPIRoute.getTotalSourceElement(); |
---|
111 | nodeRecv.resize(nRecv); |
---|
112 | MPIRoute.transferToTarget((Node * /*mpi wants non-const*/) &nodeSend[0], &nodeRecv[0], packNode, unpackNode); |
---|
113 | //cout << MPIRoute.mpiRank << " ROUTE " << nRecv << ": " << nodeSend.size() << " " << nodeRecv.size() << " "; |
---|
114 | } |
---|
115 | |
---|
116 | CParallelTree::CParallelTree(MPI_Comm comm) : communicator(comm), cascade(MIN_NODE_SZ*MIN_NODE_SZ, comm) |
---|
117 | { |
---|
118 | treeCascade.reserve(cascade.num_levels); |
---|
119 | for (int lev = 0; lev < cascade.num_levels; lev++) |
---|
120 | treeCascade.push_back(CSampleTree(cascade.level[lev].group_size, assignLevel)); |
---|
121 | } |
---|
122 | |
---|
123 | void CParallelTree::buildSampleTreeCascade(vector<Node>& sampleNodes /*route field will be modified*/, int level) |
---|
124 | { |
---|
125 | buildSampleTree(treeCascade[level], sampleNodes, cascade.level[level]); |
---|
126 | assignRoute(treeCascade[level] /*out*/, cascade.level[level] /*in*/); |
---|
127 | |
---|
128 | if (level+1 < cascade.num_levels) |
---|
129 | { |
---|
130 | vector<int> route(sampleNodes.size()); |
---|
131 | treeCascade[level].routeNodes(route, sampleNodes, assignLevel); |
---|
132 | |
---|
133 | vector<Node> routedNodes; |
---|
134 | CMPIRouting mpiRoute(cascade.level[level].pg_comm); |
---|
135 | transferRoutedNodes(mpiRoute, sampleNodes, route, routedNodes); |
---|
136 | buildSampleTreeCascade(routedNodes, level+1); |
---|
137 | } |
---|
138 | } |
---|
139 | |
---|
140 | void buildSampleTree(CSampleTree& tree, const vector<Node>& node, const CCascadeLevel& comm) |
---|
141 | /* |
---|
142 | In the beginning all the sample elements are distributed |
---|
143 | -> communicate to make available at each rank |
---|
144 | so that each rank can build the same sample tree |
---|
145 | */ |
---|
146 | { |
---|
147 | int n = node.size(); // number of samples intially on this rank (local) |
---|
148 | |
---|
149 | int blocSize = comm.group_size * ipow(MAX_NODE_SZ, assignLevel); |
---|
150 | |
---|
151 | int nrecv; // global number of samples THIS WILL BE THE NUMBER OF LEAFS IN THE SAMPLE TREE |
---|
152 | MPI_Allreduce(&n, &nrecv, 1, MPI_INT, MPI_SUM, comm.comm); // => size of sample tree does not depend on keepNodes! |
---|
153 | double ratio = blocSize / (1.0 * nrecv); |
---|
154 | int nsend = ratio * n + 1; // nsend = n_local_samples / n_global_samples * blocksize + 1 = blocksize/comm.size |
---|
155 | if (nsend > n) nsend = n; |
---|
156 | |
---|
157 | int *counts = new int[comm.size]; |
---|
158 | MPI_Allgather(&nsend, 1, MPI_INT, counts, 1, MPI_INT, comm.comm); |
---|
159 | |
---|
160 | nrecv = 0; |
---|
161 | int *displs = new int[comm.size]; |
---|
162 | for (int i = 0; i < comm.size; i++) |
---|
163 | { |
---|
164 | displs[i] = 4 * nrecv; |
---|
165 | nrecv += counts[i]; |
---|
166 | counts[i] = 4 * counts[i]; |
---|
167 | } |
---|
168 | |
---|
169 | /* pack circles around sample elements */ |
---|
170 | double *sendBuffer = new double[nsend*4]; |
---|
171 | int index = 0; |
---|
172 | vector<int> randomArray(n); |
---|
173 | randomizeArray(randomArray); |
---|
174 | for (int i = 0; i < nsend; i++) |
---|
175 | { |
---|
176 | const Node& no = node[randomArray[i]]; |
---|
177 | *((Coord *) (sendBuffer + index)) = no.centre; |
---|
178 | index += sizeof(Coord)/sizeof(*sendBuffer); |
---|
179 | sendBuffer[index++] = no.radius; |
---|
180 | } |
---|
181 | |
---|
182 | /* each process needs the sample elements from all processes */ |
---|
183 | double *recvBuffer = new double[nrecv*4]; |
---|
184 | MPI_Allgatherv(sendBuffer, 4 * nsend, MPI_DOUBLE, recvBuffer, counts, displs, MPI_DOUBLE, comm.comm); |
---|
185 | delete[] sendBuffer; |
---|
186 | delete[] counts; |
---|
187 | delete[] displs; |
---|
188 | |
---|
189 | /* unpack */ |
---|
190 | randomArray.resize(nrecv); |
---|
191 | randomizeArray(randomArray); |
---|
192 | tree.leafs.resize(nrecv); |
---|
193 | index = 0; |
---|
194 | for (int i = 0; i < nrecv; i++) |
---|
195 | { |
---|
196 | Coord x = *(Coord *)(&recvBuffer[index]); |
---|
197 | index += sizeof(Coord)/sizeof(*recvBuffer); |
---|
198 | double radius = recvBuffer[index++]; |
---|
199 | tree.leafs[randomArray[i]].centre = x; |
---|
200 | tree.leafs[randomArray[i]].radius = radius; |
---|
201 | |
---|
202 | } |
---|
203 | |
---|
204 | delete [] recvBuffer; |
---|
205 | |
---|
206 | CTimer::get("buildSampleTree(local)").resume(); |
---|
207 | tree.build(tree.leafs); |
---|
208 | cout << "SampleTree build : assign Level " << assignLevel << " nb Nodes : " << tree.levelSize[assignLevel] << endl; |
---|
209 | CTimer::get("buildSampleTree(local)").suspend(); |
---|
210 | CTimer::get("buildSampleTree(local)").print(); |
---|
211 | |
---|
212 | /* End gracefully if sample tree could not be constructed with desired number of nodes on assignLevel */ |
---|
213 | int allok, ok = (tree.levelSize[assignLevel] == comm.group_size); |
---|
214 | if (!ok) |
---|
215 | { |
---|
216 | cerr << comm.rank << ": PROBLEM: (node assign)" << tree.levelSize[assignLevel] << " != " << comm.group_size << " (keepNodes)" << endl; |
---|
217 | /* |
---|
218 | MPI_Allreduce(&ok, &allok, 1, MPI_INT, MPI_PROD, communicator); |
---|
219 | if (!allok) { |
---|
220 | MPI_Finalize(); |
---|
221 | exit(1); |
---|
222 | } |
---|
223 | */ |
---|
224 | MPI_Abort(MPI_COMM_WORLD,-1) ; |
---|
225 | } |
---|
226 | cout<<"*******************************************"<<endl ; |
---|
227 | cout<<"******* Sample Tree output *********"<<endl ; |
---|
228 | cout<<"*******************************************"<<endl ; |
---|
229 | tree.output(cout,1) ; |
---|
230 | |
---|
231 | cout<<"*******************************************"<<endl ; |
---|
232 | |
---|
233 | |
---|
234 | assert(tree.root->incluCheck() == 0); |
---|
235 | } |
---|
236 | |
---|
237 | |
---|
238 | void CParallelTree::buildLocalTree(const vector<Node>& node, const vector<int>& route) |
---|
239 | { |
---|
240 | CMPIRouting MPIRoute(communicator); |
---|
241 | MPI_Barrier(communicator); |
---|
242 | CTimer::get("buildLocalTree(initRoute)").resume(); |
---|
243 | MPIRoute.init(route); |
---|
244 | CTimer::get("buildLocalTree(initRoute)").suspend(); |
---|
245 | CTimer::get("buildLocalTree(initRoute)").print(); |
---|
246 | |
---|
247 | nbLocalElements = MPIRoute.getTotalSourceElement(); |
---|
248 | localElements = new Elt[nbLocalElements]; |
---|
249 | |
---|
250 | vector<Elt*> ptElement(node.size()); |
---|
251 | for (int i = 0; i < node.size(); i++) |
---|
252 | ptElement[i] = (Elt *) (node[i].data); |
---|
253 | |
---|
254 | vector<Elt*> ptLocalElement(nbLocalElements); |
---|
255 | for (int i = 0; i < nbLocalElements; i++) |
---|
256 | ptLocalElement[i] = &localElements[i]; |
---|
257 | |
---|
258 | CTimer::get("buildLocalTree(transfer)").resume(); |
---|
259 | MPIRoute.transferToTarget(&ptElement[0], &ptLocalElement[0], packElement, unpackElement); |
---|
260 | CTimer::get("buildLocalTree(transfer)").suspend(); |
---|
261 | CTimer::get("buildLocalTree(transfer)").print(); |
---|
262 | |
---|
263 | CTimer::get("buildLocalTree(local)").resume(); |
---|
264 | |
---|
265 | int mpiRank; |
---|
266 | MPI_Comm_rank(communicator, &mpiRank); |
---|
267 | localTree.leafs.reserve(nbLocalElements); |
---|
268 | for (int i = 0; i < nbLocalElements; i++) |
---|
269 | { |
---|
270 | Elt& elt = localElements[i]; |
---|
271 | elt.id.ind = i; |
---|
272 | elt.id.rank = mpiRank; |
---|
273 | localTree.leafs.push_back(Node(elt.x, cptRadius(elt), &localElements[i])); |
---|
274 | } |
---|
275 | localTree.build(localTree.leafs); |
---|
276 | |
---|
277 | cptAllEltsGeom(localElements, nbLocalElements, srcGrid.pole); |
---|
278 | CTimer::get("buildLocalTree(local)").suspend(); |
---|
279 | CTimer::get("buildLocalTree(local)").print(); |
---|
280 | } |
---|
281 | |
---|
282 | void CParallelTree::build(vector<Node>& node, vector<Node>& node2) |
---|
283 | { |
---|
284 | |
---|
285 | int assignLevel = 2; |
---|
286 | int nbSampleNodes = 2*ipow(MAX_NODE_SZ + 1, assignLevel); |
---|
287 | |
---|
288 | // make multiple of two |
---|
289 | nbSampleNodes /= 2; |
---|
290 | nbSampleNodes *= 2; |
---|
291 | |
---|
292 | assert(node.size() > nbSampleNodes); |
---|
293 | assert(node2.size() > nbSampleNodes); |
---|
294 | vector<Node> sampleNodes; sampleNodes.reserve(nbSampleNodes); |
---|
295 | |
---|
296 | vector<int> randomArray1(node.size()); |
---|
297 | randomizeArray(randomArray1); |
---|
298 | vector<int> randomArray2(node2.size()); |
---|
299 | randomizeArray(randomArray2); |
---|
300 | |
---|
301 | for (int i = 0; i < nbSampleNodes/2; i++) |
---|
302 | { |
---|
303 | sampleNodes.push_back(Node(node[randomArray1[i]].centre, node[randomArray1[i]].radius, NULL)); |
---|
304 | sampleNodes.push_back(Node(node2[randomArray2[i]].centre, node2[randomArray2[i]].radius, NULL)); |
---|
305 | } |
---|
306 | |
---|
307 | CTimer::get("buildParallelSampleTree").resume(); |
---|
308 | //sampleTree.buildParallelSampleTree(sampleNodes, cascade); |
---|
309 | buildSampleTreeCascade(sampleNodes); |
---|
310 | CTimer::get("buildParallelSampleTree").suspend(); |
---|
311 | CTimer::get("buildParallelSampleTree").print(); |
---|
312 | |
---|
313 | //route source mesh |
---|
314 | CTimer::get("parallelRouteNode").resume(); |
---|
315 | vector<int> route(node.size()); |
---|
316 | routeNodes(route /*out*/, node); |
---|
317 | CTimer::get("parallelRouteNode").suspend(); |
---|
318 | CTimer::get("parallelRouteNode").print(); |
---|
319 | |
---|
320 | CTimer::get("buildLocalTree").resume(); |
---|
321 | buildLocalTree(node, route); |
---|
322 | CTimer::get("buildLocalTree").suspend(); |
---|
323 | CTimer::get("buildLocalTree").print(); |
---|
324 | |
---|
325 | CTimer::get("buildRouteTree").resume(); |
---|
326 | /* update circles of tree cascade so it can be used for routing */ |
---|
327 | updateCirclesForRouting(localTree.root->centre, localTree.root->radius); |
---|
328 | CTimer::get("buildRouteTree").suspend(); |
---|
329 | CTimer::get("buildRouteTree").print(); |
---|
330 | } |
---|
331 | |
---|
332 | void CParallelTree::routeNodes(vector<int>& route, vector<Node>& nodes /*route field used*/, int level) |
---|
333 | { |
---|
334 | treeCascade[level].routeNodes(route /*assign*/, nodes, assignLevel); |
---|
335 | |
---|
336 | if (level+1 < cascade.num_levels) |
---|
337 | { |
---|
338 | vector<Node> routedNodes; |
---|
339 | CMPIRouting MPIRoute(cascade.level[level].pg_comm); |
---|
340 | transferRoutedNodes(MPIRoute, nodes, route /*use*/, routedNodes); |
---|
341 | vector<int> globalRank(routedNodes.size()); |
---|
342 | routeNodes(globalRank, routedNodes, level + 1); |
---|
343 | MPIRoute.transferFromSource(&route[0] /*override*/, &globalRank[0]); |
---|
344 | } |
---|
345 | else |
---|
346 | { |
---|
347 | CMPIRouting MPIRoute(cascade.level[level].comm); // or use pg_comm, on last cascade level identical |
---|
348 | MPIRoute.init(route); |
---|
349 | int nbRecvNode = MPIRoute.getTotalSourceElement(); |
---|
350 | vector<int> globalRank(nbRecvNode); |
---|
351 | for (int i = 0; i < globalRank.size(); i++) |
---|
352 | globalRank[i] = cascade.level[0].rank; |
---|
353 | MPIRoute.transferFromSource(&route[0] /*override*/, &globalRank[0]); |
---|
354 | } |
---|
355 | } |
---|
356 | |
---|
357 | /* assume `to` to be empty vector at entry */ |
---|
358 | void linearize(const vector<vector<int> >& from, vector<int>& to) |
---|
359 | { |
---|
360 | int cnt = 0; |
---|
361 | for (int i = 0; i < from.size(); ++i) |
---|
362 | cnt += from[i].size(); |
---|
363 | to.resize(cnt); |
---|
364 | vector<int>::iterator dest = to.begin(); |
---|
365 | for (int i = 0; i < from.size(); ++i) |
---|
366 | dest = copy(from[i].begin(), from[i].end(), dest); |
---|
367 | } |
---|
368 | |
---|
369 | /* at entry `to` must already have it's final shape and only values are overriden */ |
---|
370 | void delinearize(const vector<int>& from, vector<vector<int> >& to) |
---|
371 | { |
---|
372 | vector<int>::const_iterator end, src = from.begin(); |
---|
373 | for (int i = 0; i < to.size(); ++i, src=end) |
---|
374 | copy(src, end = src + to[i].size(), to[i].begin()); |
---|
375 | } |
---|
376 | |
---|
377 | void CParallelTree::routeIntersections(vector<vector<int> >& routes, vector<Node>& nodes, int level) |
---|
378 | { |
---|
379 | treeCascade[level].routeIntersections(routes /*assign*/, nodes); |
---|
380 | |
---|
381 | if (level+1 < cascade.num_levels) |
---|
382 | { |
---|
383 | vector<Node> routedNodes; |
---|
384 | CMPIRouting MPIRoute(cascade.level[level].pg_comm); |
---|
385 | |
---|
386 | vector<int> flattenedRoutes1; |
---|
387 | linearize(routes, flattenedRoutes1); |
---|
388 | vector<Node> double_nodes(flattenedRoutes1.size()); |
---|
389 | int j = 0; |
---|
390 | for (int i = 0; i < routes.size(); ++i) |
---|
391 | for (int k = 0; k < routes[i].size(); ++k, ++j) |
---|
392 | double_nodes[j] = nodes[i]; |
---|
393 | transferRoutedNodes(MPIRoute, double_nodes, flattenedRoutes1 /*use*/, routedNodes); |
---|
394 | vector<vector<int> > globalRanks(routedNodes.size()); |
---|
395 | routeIntersections(globalRanks /*out*/, routedNodes /*in*/, level + 1); |
---|
396 | vector<vector<int> > flattenedRoutes(flattenedRoutes1.size()); |
---|
397 | // transferFromSource expects sizes (nbSendNode=flattenedRoutes, nbRecvNode=routedNodes.size()) |
---|
398 | MPIRoute.transferFromSource(&flattenedRoutes[0], &globalRanks[0], packVector, unpackVector); |
---|
399 | for (int i = 0, j = 0; i < routes.size(); ++i) |
---|
400 | { |
---|
401 | int old_size = routes[i].size(); |
---|
402 | routes[i].resize(0); |
---|
403 | for (int k = 0; k < old_size; ++k, ++j) |
---|
404 | for (int l = 0; l < flattenedRoutes[j].size(); ++l) |
---|
405 | routes[i].push_back(flattenedRoutes[j][l]); |
---|
406 | } |
---|
407 | assert(j == flattenedRoutes1.size()); |
---|
408 | |
---|
409 | } |
---|
410 | else |
---|
411 | { |
---|
412 | CMPIRouting MPIRoute(cascade.level[level].comm); |
---|
413 | MPIRoute.init(routes); |
---|
414 | int nbRecvNode = MPIRoute.getTotalSourceElement(); |
---|
415 | vector<int> globalRanks(nbRecvNode, cascade.level[0].rank); |
---|
416 | vector<int> flattened_routes; |
---|
417 | linearize(routes, flattened_routes); |
---|
418 | MPIRoute.transferFromSource(&flattened_routes[0], &globalRanks[0]); |
---|
419 | delinearize(flattened_routes, routes); |
---|
420 | } |
---|
421 | if (level!=level) |
---|
422 | { |
---|
423 | for (int i = 0; i < routes.size(); ++i) |
---|
424 | for (int k = 0; k < routes[i].size(); ++k) |
---|
425 | if (routes[i][k] == cascade.level[0].rank) routes[i].erase(routes[i].begin() + (k--)); |
---|
426 | } |
---|
427 | } |
---|
428 | |
---|
429 | void CParallelTree::updateCirclesForRouting(Coord rootCentre, double rootRadius, int level) |
---|
430 | { |
---|
431 | if (level + 1 < cascade.num_levels) // children in the cascade have to update first |
---|
432 | { |
---|
433 | updateCirclesForRouting(rootCentre, rootRadius, level + 1); |
---|
434 | rootCentre = treeCascade[level+1].root->centre; |
---|
435 | rootRadius = treeCascade[level+1].root->radius; |
---|
436 | } |
---|
437 | |
---|
438 | // gather circles on this level of the cascade |
---|
439 | int pg_size; |
---|
440 | MPI_Comm_size(cascade.level[level].pg_comm, &pg_size); |
---|
441 | vector<Coord> allRootCentres(pg_size); |
---|
442 | vector<double> allRootRadia(pg_size); |
---|
443 | MPI_Allgather(&rootCentre, 3, MPI_DOUBLE, &allRootCentres[0], 3, MPI_DOUBLE, cascade.level[level].pg_comm); |
---|
444 | MPI_Allgather(&rootRadius, 1, MPI_DOUBLE, &allRootRadia[0], 1, MPI_DOUBLE, cascade.level[level].pg_comm); |
---|
445 | |
---|
446 | // now allRootsRadia and allRootCentres must be inserted into second levels of us and propagated to root |
---|
447 | treeCascade[level].root->assignCircleAndPropagateUp(&allRootCentres[0], &allRootRadia[0], assignLevel); |
---|
448 | } |
---|
449 | |
---|
450 | CParallelTree::~CParallelTree() |
---|
451 | { |
---|
452 | delete [] localElements; |
---|
453 | } |
---|
454 | |
---|
455 | } |
---|