[409] | 1 | /********************************************************************* |
---|
| 2 | * Copyright 1993, UCAR/Unidata |
---|
| 3 | * See netcdf/COPYRIGHT file for copying and redistribution conditions. |
---|
| 4 | * $Header$ |
---|
| 5 | *********************************************************************/ |
---|
| 6 | #include "ncdap3.h" |
---|
| 7 | #include "dapodom.h" |
---|
| 8 | #include "dapdump.h" |
---|
| 9 | |
---|
| 10 | static int iscacheableconstraint(DCEconstraint* con); |
---|
| 11 | |
---|
| 12 | /* Return 1 if we can reuse cached data to address |
---|
| 13 | the current get_vara request; return 0 otherwise. |
---|
| 14 | Target is in the constrained tree space. |
---|
| 15 | Currently, if the target matches a cache that is not |
---|
| 16 | a whole variable, then match is false. |
---|
| 17 | */ |
---|
| 18 | int |
---|
| 19 | iscached(NCDAPCOMMON* nccomm, CDFnode* target, NCcachenode** cachenodep) |
---|
| 20 | { |
---|
| 21 | int i,j,found,index; |
---|
| 22 | NCcache* cache; |
---|
| 23 | NCcachenode* cachenode; |
---|
| 24 | |
---|
| 25 | found = 0; |
---|
| 26 | if(target == NULL) goto done; |
---|
| 27 | |
---|
| 28 | /* Match the target variable against the prefetch, if any */ |
---|
| 29 | /* Note that prefetches are always whole variable */ |
---|
| 30 | cache = nccomm->cdf.cache; |
---|
| 31 | cachenode = cache->prefetch; |
---|
| 32 | if(cachenode!= NULL) { |
---|
| 33 | for(found=0,i=0;i<nclistlength(cachenode->vars);i++) { |
---|
| 34 | CDFnode* var = (CDFnode*)nclistget(cachenode->vars,i); |
---|
| 35 | if(var == target) { |
---|
| 36 | if(cachenodep) *cachenodep = cachenode; |
---|
| 37 | found=1; |
---|
| 38 | goto done; |
---|
| 39 | } |
---|
| 40 | } |
---|
| 41 | } |
---|
| 42 | |
---|
| 43 | /*search other cache nodes starting at latest first */ |
---|
| 44 | index = 0; |
---|
| 45 | for(i=nclistlength(cache->nodes)-1;i>=0;i--) { |
---|
| 46 | cachenode = (NCcachenode*)nclistget(cache->nodes,i); |
---|
| 47 | /* We currently do not try to match constraints; |
---|
| 48 | If the cachenode is constrained by more than |
---|
| 49 | simple wholevariable projections, then skip it. |
---|
| 50 | */ |
---|
| 51 | if(!cachenode->wholevariable) continue; |
---|
| 52 | for(found=0,j=0;j<nclistlength(cachenode->vars);j++) { |
---|
| 53 | CDFnode* var = (CDFnode*)nclistget(cachenode->vars,j); |
---|
| 54 | if(var == target) {found=1;index=i;break;} |
---|
| 55 | } |
---|
| 56 | if(found) break; |
---|
| 57 | } |
---|
| 58 | |
---|
| 59 | if(found) { |
---|
| 60 | ASSERT((cachenode != NULL)); |
---|
| 61 | if(nclistlength(cache->nodes) > 1) { |
---|
| 62 | /* Manage the cache nodes as LRU */ |
---|
| 63 | nclistremove(cache->nodes,index); |
---|
| 64 | nclistpush(cache->nodes,(ncelem)cachenode); |
---|
| 65 | } |
---|
| 66 | if(cachenodep) *cachenodep = cachenode; |
---|
| 67 | } |
---|
| 68 | |
---|
| 69 | done: |
---|
| 70 | #ifdef DEBUG |
---|
| 71 | fprintf(stderr,"iscached: search: %s\n",makecdfpathstring3(target,".")); |
---|
| 72 | if(found) |
---|
| 73 | fprintf(stderr,"iscached: found: %s\n",dumpcachenode(cachenode)); |
---|
| 74 | else |
---|
| 75 | fprintf(stderr,"iscached: notfound\n"); |
---|
| 76 | #endif |
---|
| 77 | return found; |
---|
| 78 | } |
---|
| 79 | |
---|
| 80 | /* Compute the set of prefetched data. |
---|
| 81 | Notes: |
---|
| 82 | 1. Even if caching is off, we will |
---|
| 83 | still prefetch the small variables. |
---|
| 84 | 2. All prefetches are whole variable fetches. |
---|
| 85 | 3. If the data set is unconstrainable, we |
---|
| 86 | will prefetch the whole thing |
---|
| 87 | */ |
---|
| 88 | NCerror |
---|
| 89 | prefetchdata3(NCDAPCOMMON* nccomm) |
---|
| 90 | { |
---|
| 91 | int i,j; |
---|
| 92 | NCerror ncstat = NC_NOERR; |
---|
| 93 | NClist* allvars = nccomm->cdf.varnodes; |
---|
| 94 | DCEconstraint* urlconstraint = nccomm->oc.dapconstraint; |
---|
| 95 | NClist* vars = nclistnew(); |
---|
| 96 | NCcachenode* cache = NULL; |
---|
| 97 | DCEconstraint* newconstraint = NULL; |
---|
| 98 | int isnc4 = FLAGSET(nccomm->controls,NCF_NC4); |
---|
| 99 | |
---|
| 100 | |
---|
| 101 | if(FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { |
---|
| 102 | /* If we cannot constrain and caching is enabled, |
---|
| 103 | then pull in everything */ |
---|
| 104 | if(FLAGSET(nccomm->controls,NCF_CACHE)) { |
---|
| 105 | for(i=0;i<nclistlength(allvars);i++) { |
---|
| 106 | nclistpush(vars,nclistget(allvars,i)); |
---|
| 107 | } |
---|
| 108 | } else { /* do no prefetching */ |
---|
| 109 | nccomm->cdf.cache->prefetch = NULL; |
---|
| 110 | goto done; |
---|
| 111 | } |
---|
| 112 | } else { /* can do constraints */ |
---|
| 113 | /* pull in those variables of sufficiently small size */ |
---|
| 114 | for(i=0;i<nclistlength(allvars);i++) { |
---|
| 115 | CDFnode* var = (CDFnode*)nclistget(allvars,i); |
---|
| 116 | size_t nelems = 1; |
---|
| 117 | |
---|
| 118 | if(!isnc4) { |
---|
| 119 | /* If netcdf 3 and var is a sequence or under a sequence, then never prefetch */ |
---|
| 120 | if(var->nctype == NC_Sequence || dapinsequence(var)) continue; |
---|
| 121 | } |
---|
| 122 | |
---|
| 123 | /* Compute the # of elements in the variable */ |
---|
| 124 | for(j=0;j<nclistlength(var->array.dimset0);j++) { |
---|
| 125 | CDFnode* dim = (CDFnode*)nclistget(var->array.dimset0,j); |
---|
| 126 | nelems *= dim->dim.declsize; |
---|
| 127 | } |
---|
| 128 | if(SHOWFETCH) { |
---|
| 129 | nclog(NCLOGDBG,"prefetch: %s=%lu",var->ncfullname,(unsigned long)nelems); |
---|
| 130 | } |
---|
| 131 | if(nelems <= nccomm->cdf.smallsizelimit) { |
---|
| 132 | nclistpush(vars,(ncelem)var); |
---|
| 133 | if(SHOWFETCH) { |
---|
| 134 | nclog(NCLOGDBG,"prefetch: %s",var->ncfullname); |
---|
| 135 | } |
---|
| 136 | } |
---|
| 137 | } |
---|
| 138 | } |
---|
| 139 | |
---|
| 140 | /* If there are no vars, then do nothing */ |
---|
| 141 | if(nclistlength(vars) == 0) { |
---|
| 142 | nccomm->cdf.cache->prefetch = NULL; |
---|
| 143 | goto done; |
---|
| 144 | } |
---|
| 145 | |
---|
| 146 | /* Create a single constraint consisting of the projections for the variables; |
---|
| 147 | each projection is whole variable. The selections are passed on as is. |
---|
| 148 | */ |
---|
| 149 | |
---|
| 150 | newconstraint = (DCEconstraint*)dcecreate(CES_CONSTRAINT); |
---|
| 151 | newconstraint->projections = nclistnew(); |
---|
| 152 | newconstraint->selections = dceclonelist(urlconstraint->selections); |
---|
| 153 | |
---|
| 154 | for(i=0;i<nclistlength(vars);i++) { |
---|
| 155 | CDFnode* var = (CDFnode*)nclistget(vars,i); |
---|
| 156 | DCEprojection* varprojection; |
---|
| 157 | /* convert var to a projection */ |
---|
| 158 | ncstat = dapvar2projection(var,&varprojection); |
---|
| 159 | if(ncstat != NC_NOERR) {THROWCHK(ncstat); goto done;} |
---|
| 160 | nclistpush(newconstraint->projections,(ncelem)varprojection); |
---|
| 161 | } |
---|
| 162 | if(SHOWFETCH) { |
---|
| 163 | char* s = dumpprojections(newconstraint->projections); |
---|
| 164 | LOG1(NCLOGNOTE,"prefetch.final: %s",s); |
---|
| 165 | nullfree(s); |
---|
| 166 | } |
---|
| 167 | ncstat = buildcachenode34(nccomm,newconstraint,vars,&cache,!isnc4); |
---|
| 168 | newconstraint = NULL; /* buildcachenode34 takes control of newconstraint */ |
---|
| 169 | if(ncstat) goto done; |
---|
| 170 | cache->wholevariable = 1; /* All prefetches are whole variable */ |
---|
| 171 | /* Make cache node be the prefetch node */ |
---|
| 172 | nccomm->cdf.cache->prefetch = cache; |
---|
| 173 | if(SHOWFETCH) { |
---|
| 174 | LOG0(NCLOGNOTE,"prefetch.complete"); |
---|
| 175 | } |
---|
| 176 | |
---|
| 177 | if(SHOWFETCH) { |
---|
| 178 | char* s = NULL; |
---|
| 179 | /* Log the set of prefetch variables */ |
---|
| 180 | NCbytes* buf = ncbytesnew(); |
---|
| 181 | ncbytescat(buf,"prefetch.vars: "); |
---|
| 182 | for(i=0;i<nclistlength(vars);i++) { |
---|
| 183 | CDFnode* var = (CDFnode*)nclistget(vars,i); |
---|
| 184 | ncbytescat(buf," "); |
---|
| 185 | s = makecdfpathstring3(var,"."); |
---|
| 186 | ncbytescat(buf,s); |
---|
| 187 | nullfree(s); |
---|
| 188 | } |
---|
| 189 | ncbytescat(buf,"\n"); |
---|
| 190 | nclog(NCLOGNOTE,"%s",ncbytescontents(buf)); |
---|
| 191 | ncbytesfree(buf); |
---|
| 192 | } |
---|
| 193 | |
---|
| 194 | done: |
---|
| 195 | nclistfree(vars); |
---|
| 196 | dcefree((DCEnode*)newconstraint); |
---|
| 197 | if(ncstat) freenccachenode(nccomm,cache); |
---|
| 198 | return THROW(ncstat); |
---|
| 199 | } |
---|
| 200 | |
---|
| 201 | NCerror |
---|
| 202 | buildcachenode34(NCDAPCOMMON* nccomm, |
---|
| 203 | DCEconstraint* constraint, |
---|
| 204 | NClist* varlist, |
---|
| 205 | NCcachenode** cachep, |
---|
| 206 | int isprefetch) |
---|
| 207 | { |
---|
| 208 | NCerror ncstat = NC_NOERR; |
---|
| 209 | OCerror ocstat = OC_NOERR; |
---|
| 210 | OCconnection conn = nccomm->oc.conn; |
---|
| 211 | OCobject ocroot = OCNULL; |
---|
| 212 | CDFnode* dxdroot = NULL; |
---|
| 213 | NCcachenode* cachenode = NULL; |
---|
| 214 | char* ce = NULL; |
---|
| 215 | |
---|
| 216 | ce = buildconstraintstring3(constraint); |
---|
| 217 | |
---|
| 218 | ocstat = dap_fetch(nccomm,conn,ce,OCDATADDS,&ocroot); |
---|
| 219 | nullfree(ce); |
---|
| 220 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
| 221 | |
---|
| 222 | ncstat = buildcdftree34(nccomm,ocroot,OCDATA,&dxdroot); |
---|
| 223 | if(ncstat) {THROWCHK(ncstat); goto done;} |
---|
| 224 | |
---|
| 225 | /* regrid */ |
---|
| 226 | if(!FLAGSET(nccomm->controls,NCF_UNCONSTRAINABLE)) { |
---|
| 227 | ncstat = regrid3(dxdroot,nccomm->cdf.ddsroot,constraint->projections); |
---|
| 228 | if(ncstat) {THROWCHK(ncstat); goto done;} |
---|
| 229 | } |
---|
| 230 | |
---|
| 231 | /* create the cache node */ |
---|
| 232 | cachenode = createnccachenode(); |
---|
| 233 | cachenode->prefetch = isprefetch; |
---|
| 234 | cachenode->vars = nclistclone(varlist); |
---|
| 235 | cachenode->datadds = dxdroot; |
---|
| 236 | /* Give the constraint over to the cachenode */ |
---|
| 237 | cachenode->constraint = constraint; |
---|
| 238 | constraint = NULL; |
---|
| 239 | cachenode->wholevariable = iscacheableconstraint(cachenode->constraint); |
---|
| 240 | |
---|
| 241 | /* save the root content*/ |
---|
| 242 | cachenode->ocroot = ocroot; |
---|
| 243 | cachenode->content = oc_data_new(conn); |
---|
| 244 | ocstat = oc_data_root(conn,ocroot,cachenode->content); |
---|
| 245 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
| 246 | |
---|
| 247 | /* capture the packet size */ |
---|
| 248 | ocstat = oc_raw_xdrsize(conn,ocroot,&cachenode->xdrsize); |
---|
| 249 | if(ocstat) {THROWCHK(ocerrtoncerr(ocstat)); goto done;} |
---|
| 250 | |
---|
| 251 | #ifdef DEBUG |
---|
| 252 | fprintf(stderr,"buildcachenode: new cache node: %s\n", |
---|
| 253 | dumpcachenode(cachenode)); |
---|
| 254 | #endif |
---|
| 255 | /* Insert into the cache. If not caching, then |
---|
| 256 | remove any previous cache node |
---|
| 257 | */ |
---|
| 258 | if(!isprefetch) { |
---|
| 259 | NCcache* cache = nccomm->cdf.cache; |
---|
| 260 | if(cache->nodes == NULL) cache->nodes = nclistnew(); |
---|
| 261 | /* remove cache nodes to get below the max cache size */ |
---|
| 262 | while(cache->cachesize + cachenode->xdrsize > cache->cachelimit |
---|
| 263 | && nclistlength(cache->nodes) > 0) { |
---|
| 264 | NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); |
---|
| 265 | #ifdef DEBUG |
---|
| 266 | fprintf(stderr,"buildcachenode: purge cache node: %s\n", |
---|
| 267 | dumpcachenode(cachenode)); |
---|
| 268 | #endif |
---|
| 269 | cache->cachesize -= node->xdrsize; |
---|
| 270 | freenccachenode(nccomm,node); |
---|
| 271 | } |
---|
| 272 | /* Remove cache nodes to get below the max cache count */ |
---|
| 273 | /* If not caching, then cachecount should be 0 */ |
---|
| 274 | while(nclistlength(cache->nodes) > cache->cachecount) { |
---|
| 275 | NCcachenode* node = (NCcachenode*)nclistremove(cache->nodes,0); |
---|
| 276 | #ifdef DEBUG |
---|
| 277 | fprintf(stderr,"buildcachenode: count purge cache node: %s\n", |
---|
| 278 | dumpcachenode(node)); |
---|
| 279 | #endif |
---|
| 280 | cache->cachesize -= node->xdrsize; |
---|
| 281 | freenccachenode(nccomm,node); |
---|
| 282 | } |
---|
| 283 | nclistpush(nccomm->cdf.cache->nodes,(ncelem)cachenode); |
---|
| 284 | cache->cachesize += cachenode->xdrsize; |
---|
| 285 | } |
---|
| 286 | |
---|
| 287 | #ifdef DEBUG |
---|
| 288 | fprintf(stderr,"buildcachenode: %s\n",dumpcachenode(cachenode)); |
---|
| 289 | #endif |
---|
| 290 | |
---|
| 291 | done: |
---|
| 292 | if(constraint != NULL) dcefree((DCEnode*)constraint); |
---|
| 293 | if(cachep) *cachep = cachenode; |
---|
| 294 | if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); |
---|
| 295 | if(ncstat) { |
---|
| 296 | freecdfroot34(dxdroot); |
---|
| 297 | freenccachenode(nccomm,cachenode); |
---|
| 298 | } |
---|
| 299 | return THROW(ncstat); |
---|
| 300 | } |
---|
| 301 | |
---|
| 302 | NCcachenode* |
---|
| 303 | createnccachenode(void) |
---|
| 304 | { |
---|
| 305 | NCcachenode* mem = (NCcachenode*)calloc(1,sizeof(NCcachenode)); |
---|
| 306 | return mem; |
---|
| 307 | } |
---|
| 308 | |
---|
| 309 | void |
---|
| 310 | freenccachenode(NCDAPCOMMON* nccomm, NCcachenode* node) |
---|
| 311 | { |
---|
| 312 | if(node == NULL) return; |
---|
| 313 | oc_data_free(nccomm->oc.conn,node->content); |
---|
| 314 | dcefree((DCEnode*)node->constraint); |
---|
| 315 | freecdfroot34(node->datadds); |
---|
| 316 | nclistfree(node->vars); |
---|
| 317 | nullfree(node); |
---|
| 318 | } |
---|
| 319 | |
---|
| 320 | void |
---|
| 321 | freenccache(NCDAPCOMMON* nccomm, NCcache* cache) |
---|
| 322 | { |
---|
| 323 | int i; |
---|
| 324 | if(cache == NULL) return; |
---|
| 325 | freenccachenode(nccomm,cache->prefetch); |
---|
| 326 | for(i=0;i<nclistlength(cache->nodes);i++) { |
---|
| 327 | freenccachenode(nccomm,(NCcachenode*)nclistget(cache->nodes,i)); |
---|
| 328 | } |
---|
| 329 | nclistfree(cache->nodes); |
---|
| 330 | nullfree(cache); |
---|
| 331 | } |
---|
| 332 | |
---|
| 333 | NCcache* |
---|
| 334 | createnccache(void) |
---|
| 335 | { |
---|
| 336 | NCcache* c = (NCcache*)calloc(1,sizeof(NCcache)); |
---|
| 337 | c->cachelimit = DFALTCACHELIMIT; |
---|
| 338 | c->cachesize = 0; |
---|
| 339 | c->nodes = nclistnew(); |
---|
| 340 | c->cachecount = DFALTCACHECOUNT; |
---|
| 341 | return c; |
---|
| 342 | } |
---|
| 343 | |
---|
| 344 | static int |
---|
| 345 | iscacheableprojection(DCEprojection* proj) |
---|
| 346 | { |
---|
| 347 | int i,cacheable; |
---|
| 348 | if(proj->discrim != CES_VAR) return 0; |
---|
| 349 | cacheable = 1; /* assume so */ |
---|
| 350 | for(i=0;i<nclistlength(proj->var->segments);i++) { |
---|
| 351 | DCEsegment* segment = (DCEsegment*)nclistget(proj->var->segments,i); |
---|
| 352 | if(!iswholesegment(segment)) {cacheable = 0; break;} |
---|
| 353 | } |
---|
| 354 | return cacheable; |
---|
| 355 | } |
---|
| 356 | |
---|
| 357 | static int |
---|
| 358 | iscacheableconstraint(DCEconstraint* con) |
---|
| 359 | { |
---|
| 360 | int i; |
---|
| 361 | if(con == NULL) return 1; |
---|
| 362 | if(con->selections != NULL && nclistlength(con->selections) > 0) |
---|
| 363 | return 0; /* cant deal with selections */ |
---|
| 364 | for(i=0;i<nclistlength(con->projections);i++) { |
---|
| 365 | if(!iscacheableprojection((DCEprojection*)nclistget(con->projections,i))) |
---|
| 366 | return 0; |
---|
| 367 | } |
---|
| 368 | return 1; |
---|
| 369 | } |
---|