1 | /********************************************************************* |
---|
2 | * Copyright 1993, UCAR/Unidata |
---|
3 | * See netcdf/COPYRIGHT file for copying and redistribution conditions. |
---|
4 | * $Header$ |
---|
5 | *********************************************************************/ |
---|
6 | #include "ncdap3.h" |
---|
7 | #include "dapodom.h" |
---|
8 | #include "dapdump.h" |
---|
9 | |
---|
10 | static int iscacheableconstraint(DCEconstraint* con); |
---|
11 | |
---|
12 | /* Return 1 if we can reuse cached data to address |
---|
13 | the current get_vara request; return 0 otherwise. |
---|
14 | Target is in the constrained tree space. |
---|
15 | Currently, if the target matches a cache that is not |
---|
16 | a whole variable, then match is false. |
---|
17 | */ |
---|
18 | int |
---|
19 | iscached(NCDAPCOMMON* nccomm, CDFnode* target, NCcachenode** cachenodep) |
---|
20 | { |
---|
21 | int i,j,found,index; |
---|
22 | NCcache* cache; |
---|
23 | NCcachenode* cachenode; |
---|
24 | |
---|
25 | found = 0; |
---|
26 | if(target == NULL) goto done; |
---|
27 | |
---|
28 | /* Match the target variable against the prefetch, if any */ |
---|
29 | /* Note that prefetches are always whole variable */ |
---|
30 | cache = nccomm->cdf.cache; |
---|
31 | cachenode = cache->prefetch; |
---|
32 | if(cachenode!= NULL) { |
---|
33 | for(found=0,i=0;i<nclistlength(cachenode->vars);i++) { |
---|
34 | CDFnode* var = (CDFnode*)nclistget(cachenode->vars,i); |
---|
35 | if(var == target) { |
---|
36 | if(cachenodep) *cachenodep = cachenode; |
---|
37 | found=1; |
---|
38 | goto done; |
---|
39 | } |
---|
40 | } |
---|
41 | } |
---|
42 | |
---|
43 | /*search other cache nodes starting at latest first */ |
---|
44 | index = 0; |
---|
45 | for(i=nclistlength(cache->nodes)-1;i>=0;i--) { |
---|
46 | cachenode = (NCcachenode*)nclistget(cache->nodes,i); |
---|
47 | /* We currently do not try to match constraints; |
---|
48 | If the cachenode is constrained by more than |
---|
49 | simple wholevariable projections, then skip it. |
---|
50 | */ |
---|
51 | if(!cachenode->wholevariable) continue; |
---|
52 | for(found=0,j=0;j<nclistlength(cachenode->vars);j++) { |
---|
53 | CDFnode* var = (CDFnode*)nclistget(cachenode->vars,j); |
---|
54 | if(var == target) {found=1;index=i;break;} |
---|
55 | } |
---|
56 | if(found) break; |
---|
57 | } |
---|
58 | |
---|
59 | if(found) { |
---|
60 | ASSERT((cachenode != NULL)); |
---|
61 | if(nclistlength(cache->nodes) > 1) { |
---|
62 | /* Manage the cache nodes as LRU */ |
---|
63 | nclistremove(cache->nodes,index); |
---|
64 | nclistpush(cache->nodes,(ncelem)cachenode); |
---|
65 | } |
---|
66 | if(cachenodep) *cachenodep = cachenode; |
---|
67 | } |
---|
68 | |
---|
69 | done: |
---|
70 | #ifdef DEBUG |
---|
71 | fprintf(stderr,"iscached: search: %s\n",makecdfpathstring3(target,".")); |
---|
72 | if(found) |
---|
73 | fprintf(stderr,"iscached: found: %s\n",dumpcachenode(cachenode)); |
---|
74 | else |
---|
75 | fprintf(stderr,"iscached: notfound\n"); |
---|
76 | #endif |
---|
77 | return found; |
---|
78 | } |
---|
79 | |
---|
80 | /* Compute the set of prefetched data. |
---|
81 | Notes: |
---|
82 | 1. Even if caching is off, we will |
---|
83 | still prefetch the small variables. |
---|
84 | 2. All prefetches are whole variable fetches. |
---|
85 | 3. If the data set is unconstrainable, we |
---|
86 | will prefetch the whole thing |
---|
87 | */ |
---|
88 | NCerror |
---|
89 | prefetchdata3(NCDAPCOMMON* nccomm) |
---|
90 | { |
---|
91 | int i,j; |
---|
92 | NCerror ncstat = NC_NOERR; |
---|
93 | NClist* allvars = nccomm->cdf.varnodes; |
---|
94 | DCEconstraint* urlconstraint = nccomm->oc.dapconstraint; |
---|
95 | NClist* vars = nclistnew(); |
---|
96 | NCcachenode* cache = NULL; |
---|
97 | DCEconstraint* newconstraint = NULL; |
---|
98 | int isnc4 = FLAGSET(nccomm->controls,NCF_NC4); |
---|
99 | |
---|
100 | |
---|
101 | if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { |
---|
102 | /* If we cannot constrain and caching is enabled, |
---|
103 | then pull in everything */ |
---|
104 | if(FLAGSET(nccomm->controls,NCF_CACHE)) { |
---|
105 | for(i=0;i<nclistlength(allvars);i++) { |
---|
106 | nclistpush(vars,nclistget(allvars,i)); |
---|
107 | } |
---|
108 | } else { /* do no prefetching */ |
---|
109 | nccomm->cdf.cache->prefetch = NULL; |
---|
110 | goto done; |
---|
111 | } |
---|
112 | } else { /* can do constraints */ |
---|
113 | /* pull in those variables of sufficiently small size */ |
---|
114 | for(i=0;i<nclistlength(allvars);i++) { |
---|
115 | CDFnode* var = (CDFnode*)nclistget(allvars,i); |
---|
116 | size_t nelems = 1; |
---|
117 | |
---|
118 | if(!isnc4) { |
---|
119 | /* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */ |
---|
120 | if(var->nctype == NC_Sequence || dapinsequence(var)) continue; |
---|
121 | } |
---|
122 | |
---|
123 | /* Compute the # of elements in the variable */ |
---|
124 | for(j=0;j<nclistlength(var->array.dimset0);j++) { |
---|
125 | CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j); |
---|
126 | nelems *= dim->dim.declsize; |
---|
127 | } |
---|
128 | if(SHOWFETCH) { |
---|
129 | nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems); |
---|
130 | } |
---|
131 | if(nelems <= nccomm->cdf.smallsizelimit) { |
---|
132 | nclistpush(vars,(ncelem)var); |
---|
133 | if(SHOWFETCH) { |
---|
134 | nclog(NCLOGDBG,"prefetch: %s",var->ncfullname); |
---|
135 | } |
---|
136 | } |
---|
137 | } |
---|
138 | } |
---|
139 | |
---|
140 | /* If there are no vars, then do nothing */ |
---|
141 | if(nclistlength(vars) == 0) { |
---|
142 | nccomm->cdf.cache->prefetch = NULL; |
---|
143 | goto done; |
---|
144 | } |
---|
145 | |
---|
146 | /* Create a single constraint consisting of the projections for the variables; |
---|
147 | each projection is whole variable. The selections are passed on as is. |
---|
148 | */ |
---|
149 | |
---|
150 | newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT); |
---|
151 | newconstraint->projections = nclistnew(); |
---|
152 | newconstraint->selections = dceclonelist(urlconstraint->selections); |
---|
153 | |
---|
154 | for(i=0;i<nclistlength(vars);i++) { |
---|
155 | CDFnode* var = (CDFnode*)nclistget(vars,i); |
---|
156 | DCEprojection* varprojection; |
---|
157 | /* convert var to a projection */ |
---|
158 | ncstat = dapvar2projection(var,&varprojection); |
---|
159 | if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} |
---|
160 | nclistpush(newconstraint->projections,(ncelem)varprojection); |
---|
161 | } |
---|
162 | if(SHOWFETCH) { |
---|
163 | char* s = dumpprojections(newconstraint->projections); |
---|
164 | LOG1(NCLOGNOTE,"prefetch.final: %s",s); |
---|
165 | nullfree(s); |
---|
166 | } |
---|
167 | ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4); |
---|
168 | newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */ |
---|
169 | if(ncstat) goto done; |
---|
170 | cache->wholevariable = 1; /* All prefetches are whole variable */ |
---|
171 | /* Make cache node be the prefetch node */ |
---|
172 | nccomm->cdf.cache->prefetch = cache; |
---|
173 | if(SHOWFETCH) { |
---|
174 | LOG0(NCLOGNOTE,"prefetch.complete"); |
---|
175 | } |
---|
176 | |
---|
177 | if(SHOWFETCH) { |
---|
178 | char* s = NULL; |
---|
179 | /* Log the set of prefetch variables */ |
---|
180 | NCbytes* buf = ncbytesnew(); |
---|
181 | ncbytescat(buf,"prefetch.vars: "); |
---|
182 | for(i=0;i<nclistlength(vars);i++) { |
---|
183 | CDFnode* var = (CDFnode*)nclistget(vars,i); |
---|
184 | ncbytescat(buf," "); |
---|
185 | s = makecdfpathstring3(var,"."); |
---|
186 | ncbytescat(buf,s); |
---|
187 | nullfree(s); |
---|
188 | } |
---|
189 | ncbytescat(buf,"\n"); |
---|
190 | nclog(NCLOGNOTE,"%s",ncbytescontents(buf)); |
---|
191 | ncbytesfree(buf); |
---|
192 | } |
---|
193 | |
---|
194 | done: |
---|
195 | nclistfree(vars); |
---|
196 | dcefree((DCEnode*)newconstraint); |
---|
197 | if(ncstat) freenccachenode(nccomm,cache); |
---|
198 | return THROW(ncstat); |
---|
199 | } |
---|
200 | |
---|
201 | NCerror |
---|
202 | buildcachenode34(NCDAPCOMMON* nccomm, |
---|
203 | DCEconstraint* constraint, |
---|
204 | NClist* varlist, |
---|
205 | NCcachenode** cachep, |
---|
206 | int isprefetch) |
---|
207 | { |
---|
208 | NCerror ncstat = NC_NOERR; |
---|
209 | OCerror ocstat = OC_NOERR; |
---|
210 | OCconnection conn = nccomm->oc.conn; |
---|
211 | OCobject ocroot = OCNULL; |
---|
212 | CDFnode* dxdroot = NULL; |
---|
213 | NCcachenode* cachenode = NULL; |
---|
214 | char* ce = NULL; |
---|
215 | |
---|
216 | ce = buildconstraintstring3(constraint); |
---|
217 | |
---|
218 | ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot); |
---|
219 | nullfree(ce); |
---|
220 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
221 | |
---|
222 | ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot); |
---|
223 | if(ncstat) {THROWCHK(ncstat); goto done;} |
---|
224 | |
---|
225 | /* regrid */ |
---|
226 | if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { |
---|
227 | ncstat = regrid3(dxdroot,nccomm->cdf.ddsroot,constraint->projections); |
---|
228 | if(ncstat) {THROWCHK(ncstat); goto done;} |
---|
229 | } |
---|
230 | |
---|
231 | /* create the cache node */ |
---|
232 | cachenode = createnccachenode(); |
---|
233 | cachenode->prefetch = isprefetch; |
---|
234 | cachenode->vars = nclistclone(varlist); |
---|
235 | cachenode->datadds = dxdroot; |
---|
236 | /* Give the constraint over to the cachenode */ |
---|
237 | cachenode->constraint = constraint; |
---|
238 | constraint = NULL; |
---|
239 | cachenode->wholevariable = iscacheableconstraint(cachenode->constraint); |
---|
240 | |
---|
241 | /* save the root content*/ |
---|
242 | cachenode->ocroot = ocroot; |
---|
243 | cachenode->content = oc_data_new(conn); |
---|
244 | ocstat = oc_data_root(conn,ocroot,cachenode->content); |
---|
245 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
246 | |
---|
247 | /* capture the packet size */ |
---|
248 | ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize); |
---|
249 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
250 | |
---|
251 | #ifdef DEBUG |
---|
252 | fprintf(stderr,"buildcachenode: new cache node: %s\n", |
---|
253 | dumpcachenode(cachenode)); |
---|
254 | #endif |
---|
255 | /* Insert into the cache. If not caching, then |
---|
256 | remove any previous cache node |
---|
257 | */ |
---|
258 | if(!isprefetch) { |
---|
259 | NCcache* cache = nccomm->cdf.cache; |
---|
260 | if(cache->nodes == NULL) cache->nodes = nclistnew(); |
---|
261 | /* remove cache nodes to get below the max cache size */ |
---|
262 | while(cache->cachesize + cachenode->xdrsize > cache->cachelimit |
---|
263 | && nclistlength(cache->nodes) > 0) { |
---|
264 | NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); |
---|
265 | #ifdef DEBUG |
---|
266 | fprintf(stderr,"buildcachenode: purge cache node: %s\n", |
---|
267 | dumpcachenode(cachenode)); |
---|
268 | #endif |
---|
269 | cache->cachesize -= node->xdrsize; |
---|
270 | freenccachenode(nccomm,node); |
---|
271 | } |
---|
272 | /* Remove cache nodes to get below the max cache count */ |
---|
273 | /* If not caching, then cachecount should be 0 */ |
---|
274 | while(nclistlength(cache->nodes) > cache->cachecount) { |
---|
275 | NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); |
---|
276 | #ifdef DEBUG |
---|
277 | fprintf(stderr,"buildcachenode: count purge cache node: %s\n", |
---|
278 | dumpcachenode(node)); |
---|
279 | #endif |
---|
280 | cache->cachesize -= node->xdrsize; |
---|
281 | freenccachenode(nccomm,node); |
---|
282 | } |
---|
283 | nclistpush(nccomm->cdf.cache->nodes,(ncelem)cachenode); |
---|
284 | cache->cachesize += cachenode->xdrsize; |
---|
285 | } |
---|
286 | |
---|
287 | #ifdef DEBUG |
---|
288 | fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode)); |
---|
289 | #endif |
---|
290 | |
---|
291 | done: |
---|
292 | if(constraint != NULL) dcefree((DCEnode*)constraint); |
---|
293 | if(cachep) *cachep = cachenode; |
---|
294 | if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); |
---|
295 | if(ncstat) { |
---|
296 | freecdfroot34(dxdroot); |
---|
297 | freenccachenode(nccomm,cachenode); |
---|
298 | } |
---|
299 | return THROW(ncstat); |
---|
300 | } |
---|
301 | |
---|
302 | NCcachenode* |
---|
303 | createnccachenode(void) |
---|
304 | { |
---|
305 | NCcachenode* mem = (NCcachenode*)calloc(1,sizeof(NCcachenode)); |
---|
306 | return mem; |
---|
307 | } |
---|
308 | |
---|
309 | void |
---|
310 | freenccachenode(NCDAPCOMMON* nccomm, NCcachenode* node) |
---|
311 | { |
---|
312 | if(node == NULL) return; |
---|
313 | oc_data_free(nccomm->oc.conn,node->content); |
---|
314 | dcefree((DCEnode*)node->constraint); |
---|
315 | freecdfroot34(node->datadds); |
---|
316 | nclistfree(node->vars); |
---|
317 | nullfree(node); |
---|
318 | } |
---|
319 | |
---|
320 | void |
---|
321 | freenccache(NCDAPCOMMON* nccomm, NCcache* cache) |
---|
322 | { |
---|
323 | int i; |
---|
324 | if(cache == NULL) return; |
---|
325 | freenccachenode(nccomm,cache->prefetch); |
---|
326 | for(i=0;i<nclistlength(cache->nodes);i++) { |
---|
327 | freenccachenode(nccomm,(NCcachenode*)nclistget(cache->nodes,i)); |
---|
328 | } |
---|
329 | nclistfree(cache->nodes); |
---|
330 | nullfree(cache); |
---|
331 | } |
---|
332 | |
---|
333 | NCcache* |
---|
334 | createnccache(void) |
---|
335 | { |
---|
336 | NCcache* c = (NCcache*)calloc(1,sizeof(NCcache)); |
---|
337 | c->cachelimit = DFALTCACHELIMIT; |
---|
338 | c->cachesize = 0; |
---|
339 | c->nodes = nclistnew(); |
---|
340 | c->cachecount = DFALTCACHECOUNT; |
---|
341 | return c; |
---|
342 | } |
---|
343 | |
---|
344 | static int |
---|
345 | iscacheableprojection(DCEprojection* proj) |
---|
346 | { |
---|
347 | int i,cacheable; |
---|
348 | if(proj->discrim != CES_VAR) return 0; |
---|
349 | cacheable = 1; /* assume so */ |
---|
350 | for(i=0;i<nclistlength(proj->var->segments);i++) { |
---|
351 | DCEsegment* segment = (DCEsegment*)nclistget(proj->var->segments,i); |
---|
352 | if(!iswholesegment(segment)) {cacheable = 0; break;} |
---|
353 | } |
---|
354 | return cacheable; |
---|
355 | } |
---|
356 | |
---|
357 | static int |
---|
358 | iscacheableconstraint(DCEconstraint* con) |
---|
359 | { |
---|
360 | int i; |
---|
361 | if(con == NULL) return 1; |
---|
362 | if(con->selections != NULL && nclistlength(con->selections) > 0) |
---|
363 | return 0; /* cant deal with selections */ |
---|
364 | for(i=0;i<nclistlength(con->projections);i++) { |
---|
365 | if(!iscacheableprojection((DCEprojection*)nclistget(con->projections,i))) |
---|
366 | return 0; |
---|
367 | } |
---|
368 | return 1; |
---|
369 | } |
---|