1 | #!/usr/bin/env python |
---|
2 | # -*- coding: utf-8 -*- |
---|
3 | |
---|
4 | # ==================================================================== # |
---|
5 | # Author: Sonia Labetoulle # |
---|
6 | # Contact: sonia.labetoulle _at_ ipsl.jussieu.fr # |
---|
7 | # Created: 2016 # |
---|
8 | # History: # |
---|
9 | # Modification: # |
---|
10 | # ==================================================================== # |
---|
11 | |
---|
12 | # This must come first |
---|
13 | from __future__ import print_function, unicode_literals, division |
---|
14 | |
---|
15 | # Standard library imports |
---|
16 | import os |
---|
17 | # import math |
---|
18 | from argparse import ArgumentParser |
---|
19 | import shutil |
---|
20 | import pprint |
---|
21 | import collections |
---|
22 | import ConfigParser as cp |
---|
23 | import datetime as dt |
---|
24 | import subprocess |
---|
25 | from distutils.dir_util import copy_tree |
---|
26 | |
---|
27 | # Application library imports |
---|
28 | import libconso_db as cdb |
---|
29 | # import libconso_cpt as ccpt |
---|
30 | import db_data |
---|
31 | |
---|
32 | pp = pprint.PrettyPrinter(indent=2) |
---|
33 | |
---|
34 | |
---|
35 | ####################################################################### |
---|
36 | def get_arguments(): |
---|
37 | parser = ArgumentParser() |
---|
38 | # parser.add_argument("project", action="store", |
---|
39 | # help="Project name") |
---|
40 | # parser.add_argument("center", action="store", |
---|
41 | # help="Center name (idris/tgcc)") |
---|
42 | |
---|
43 | parser.add_argument("-v", "--verbose", action="store_true", |
---|
44 | help="verbose mode") |
---|
45 | parser.add_argument("-d", "--dryrun", action="store_true", |
---|
46 | help="only print what is to be done") |
---|
47 | # parser.add_argument("-r", "--range", action="store", nargs=2, |
---|
48 | # help="date range: ssaammjj ssaammjj") |
---|
49 | return parser.parse_args() |
---|
50 | |
---|
51 | |
---|
52 | ######################################################################## |
---|
53 | def html_header(project, css_file): |
---|
54 | html = ( |
---|
55 | "<!DOCTYPE html>\n" |
---|
56 | "<html lang=\"fr-FR\">\n" |
---|
57 | "<head>\n" |
---|
58 | " <meta charset=\"UTF-8\">\n" |
---|
59 | " <meta name=\"description\" content=\"Suivi de consommation {pj}\">\n" |
---|
60 | " <meta name=\"author\" content=\"Sonia Labetoulle\">\n" |
---|
61 | " <title>Suivi de consommation {pj}</title>\n" |
---|
62 | " <link rel=\"stylesheet\" type=\"text/css\" " |
---|
63 | "href=\"{css}\">\n" |
---|
64 | "</head>\n" |
---|
65 | "\n" |
---|
66 | "<body>\n" |
---|
67 | " <div id=\"main\">\n" |
---|
68 | "\n".format(pj=project.upper(), css=css_file) |
---|
69 | ) |
---|
70 | |
---|
71 | return html |
---|
72 | |
---|
73 | |
---|
74 | ######################################################################## |
---|
75 | def html_footer(): |
---|
76 | html = ( |
---|
77 | "\n" |
---|
78 | " <div class=\"clear\"></div>\n" |
---|
79 | " </div>\n" |
---|
80 | "</body>\n" |
---|
81 | "</html>\n" |
---|
82 | ) |
---|
83 | |
---|
84 | return html |
---|
85 | |
---|
86 | |
---|
87 | ######################################################################## |
---|
88 | def html_main_index(): |
---|
89 | html = ( |
---|
90 | " <h1>Conso GENCI</h1>\n" |
---|
91 | ) |
---|
92 | |
---|
93 | return html |
---|
94 | |
---|
95 | |
---|
96 | ######################################################################## |
---|
97 | def html_main_project(centre, project, node_subp): |
---|
98 | |
---|
99 | html = ( |
---|
100 | " <h1>Conso {pj} ({ce})</h1>\n\n" |
---|
101 | .format(ce=centre.upper(), pj=project.upper()) |
---|
102 | ) |
---|
103 | |
---|
104 | for (node_type, sub_project) in node_subp: |
---|
105 | if sub_project: |
---|
106 | title = "Sous-imputation {sp} / {nd}".format( |
---|
107 | sp=sub_project if sub_project else project, |
---|
108 | nd=node_type.replace("/", "") |
---|
109 | ) |
---|
110 | else: |
---|
111 | title = "Conso globale {nd}".format( |
---|
112 | nd=node_type.replace("/", "") |
---|
113 | ) |
---|
114 | html = html + ( |
---|
115 | " <div class=\"img\">\n" |
---|
116 | " <h2>{tl}</h2>\n" |
---|
117 | " <a href=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\">\n" |
---|
118 | " <img class=\"data\"\n" |
---|
119 | " src=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\"\n" |
---|
120 | " alt=\"{tl}\" />\n" |
---|
121 | " </a><br />\n" |
---|
122 | " <div class=\"lien\">\n" |
---|
123 | " <a href=\"../pdf/bilan_{ce}_{pj}_{sp}_{nd}.pdf\">\n" |
---|
124 | " <!--<img class=\"icon\" src=\"utils/pdf_2.jpg\" />-->\n" |
---|
125 | " Fichier pdf\n" |
---|
126 | " </a>\n" |
---|
127 | " </div>\n" |
---|
128 | " </div>\n".format( |
---|
129 | tl=title, |
---|
130 | ce=centre, |
---|
131 | pj=project, |
---|
132 | sp=sub_project if sub_project else project, |
---|
133 | nd=node_type.replace("/", "") |
---|
134 | ) |
---|
135 | ) |
---|
136 | |
---|
137 | return html |
---|
138 | |
---|
139 | |
---|
140 | ######################################################################## |
---|
141 | def html_project(centre, project, node_subp, project_file): |
---|
142 | |
---|
143 | if args.verbose: |
---|
144 | print("Create {}".format(project_file)) |
---|
145 | |
---|
146 | header = html_header(project, os.path.join("..", css_file)) |
---|
147 | footer = html_footer() |
---|
148 | main = html_main_project(centre, project, node_subp) |
---|
149 | |
---|
150 | subdir = os.path.dirname(project_file) |
---|
151 | if not os.path.isdir(subdir): |
---|
152 | try: |
---|
153 | os.mkdir(subdir) |
---|
154 | except Exception as rc: |
---|
155 | raise("Could not create directory {}:\n{}".format(subdir, rc)) |
---|
156 | |
---|
157 | with open(project_file, "w") as fileout: |
---|
158 | try: |
---|
159 | fileout.write(header + main + footer) |
---|
160 | except Exception as rc: |
---|
161 | raise("Could not create directory {}:\n{}".format(project_file, rc)) |
---|
162 | |
---|
163 | |
---|
164 | ######################################################################## |
---|
165 | def config_write(centre, project): |
---|
166 | config_file = os.path.join( |
---|
167 | CARD_DIR, |
---|
168 | "config_{}_{}.ini".format(centre, project) |
---|
169 | ) |
---|
170 | |
---|
171 | if not os.path.isfile(config_file): |
---|
172 | if args.verbose: |
---|
173 | print("Create {}".format(config_file)) |
---|
174 | |
---|
175 | config = cp.ConfigParser(allow_no_value=True) |
---|
176 | config.optionxform = str |
---|
177 | |
---|
178 | config.add_section("projet") |
---|
179 | config.set("projet", "name", project) |
---|
180 | |
---|
181 | config.add_section("directories") |
---|
182 | # config.set("directories", "ROOT_DIR", "/home/igcmg/ConsoGENCI") |
---|
183 | config.set("directories", "ROOT_DIR", ROOT_DIR) |
---|
184 | config.set("directories", "DATA", "%(ROOT_DIR)s/data") |
---|
185 | config.set("directories", "PLOT", "%(ROOT_DIR)s/plot") |
---|
186 | config.set("directories", "DODS") |
---|
187 | |
---|
188 | config.add_section("dods") |
---|
189 | config.set("dods", "DIR", "/home/igcmg/dods/ConsoGENCI") |
---|
190 | config.set("dods", "USER", "igcmg") |
---|
191 | config.set("dods", "SERVER", "ciclad") |
---|
192 | |
---|
193 | with open(config_file, "w") as fileout: |
---|
194 | try: |
---|
195 | config.write(fileout) |
---|
196 | except Exception as rc: |
---|
197 | raise("Could not write {}:\n{}".format(config_file, rc)) |
---|
198 | |
---|
199 | |
---|
200 | ####################################################################### |
---|
201 | if __name__ == "__main__": |
---|
202 | |
---|
203 | date_deb = dt.datetime.now() |
---|
204 | |
---|
205 | # .. Initialization .. |
---|
206 | # ==================== |
---|
207 | # ... Command line arguments ... |
---|
208 | # ------------------------------ |
---|
209 | args = get_arguments() |
---|
210 | if args.verbose: |
---|
211 | print(args) |
---|
212 | |
---|
213 | # ... Files and directories ... |
---|
214 | # ----------------------------- |
---|
215 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
---|
216 | CARD_DIR = os.path.join(ROOT_DIR, "card") |
---|
217 | DATA_DIR = os.path.join(ROOT_DIR, "data") |
---|
218 | PLOT_DIR = os.path.join(ROOT_DIR, "plot") |
---|
219 | HTML_DIR = os.path.join(ROOT_DIR, "web") |
---|
220 | DODS_DIR = os.path.join( |
---|
221 | os.path.dirname(ROOT_DIR), |
---|
222 | "dods", |
---|
223 | "ConsoGENCI", |
---|
224 | ) |
---|
225 | SUBMIT_DIR = os.getcwd() |
---|
226 | index_file = os.path.join( |
---|
227 | HTML_DIR, |
---|
228 | "index_full.html" |
---|
229 | ) |
---|
230 | # css_file = os.path.join( |
---|
231 | # HTML_DIR, |
---|
232 | # "utils", |
---|
233 | # "ConsoGENCI.css" |
---|
234 | # ) |
---|
235 | css_file = os.path.join( |
---|
236 | "utils", |
---|
237 | "ConsoGENCI.css" |
---|
238 | ) |
---|
239 | |
---|
240 | # pattern = ccpt.cpt_pattern(center=args.center, project=args.project) |
---|
241 | # dirin = os.path.join(DATA_DIR, args.center, "tmp") |
---|
242 | # dirout = os.path.join(DATA_DIR, args.center, args.project) |
---|
243 | # filelist = ccpt.find_input_files(dirin, pattern, args.range) |
---|
244 | |
---|
245 | # if not filelist: |
---|
246 | # print("No input files found, exit") |
---|
247 | # exit(1) |
---|
248 | |
---|
249 | # if args.verbose: |
---|
250 | # print("SUBMIT_DIR:", SUBMIT_DIR) |
---|
251 | # print("DATA_DIR:", DATA_DIR) |
---|
252 | # print("dirin:", dirin) |
---|
253 | |
---|
254 | # .. Connection to database .. |
---|
255 | # ============================ |
---|
256 | if args.verbose: |
---|
257 | print("Connection to database") |
---|
258 | conn, cursor = cdb.connect_db( |
---|
259 | db_data.db_host, |
---|
260 | db_data.db_name, |
---|
261 | db_data.db_user, |
---|
262 | db_data.db_pwd, |
---|
263 | ) |
---|
264 | |
---|
265 | # print(DODS_DIR) |
---|
266 | |
---|
267 | # exit() |
---|
268 | |
---|
269 | # .. Extract centre/project list from table .. |
---|
270 | # ============================================ |
---|
271 | table_name = "conso.tbl_allocation" |
---|
272 | request = ( |
---|
273 | "SELECT DISTINCT A.centre, A.project, A.node_type, C.sub_project " |
---|
274 | "FROM conso.tbl_allocation AS A " |
---|
275 | "INNER JOIN conso.tbl_consumption AS C " |
---|
276 | "ON A.id = C.allocation_id " |
---|
277 | "WHERE A.is_active = TRUE " |
---|
278 | "ORDER BY A.centre, A.project, A.node_type, C.sub_project ;" |
---|
279 | ) |
---|
280 | |
---|
281 | cdb.select_db(cursor, request) |
---|
282 | # print(cursor.rowcount) |
---|
283 | # print(cursor.fetchall()) |
---|
284 | result = list(cursor) |
---|
285 | |
---|
286 | # .. Close connection to database .. |
---|
287 | # ================================== |
---|
288 | if args.verbose: |
---|
289 | print("Close connection") |
---|
290 | cdb.close_db(conn) |
---|
291 | |
---|
292 | # .. Build data dictionary .. |
---|
293 | # =========================== |
---|
294 | centre_dict = collections.OrderedDict() |
---|
295 | # for centre, project, node_type, sub_project in result: |
---|
296 | # if centre not in centre_dict: |
---|
297 | # centre_dict[centre] = collections.OrderedDict() |
---|
298 | # if project not in centre_dict[centre]: |
---|
299 | # centre_dict[centre][project] = [] |
---|
300 | # if node_type not in centre_dict[centre][project]: |
---|
301 | # centre_dict[centre][project].append(node_type) |
---|
302 | |
---|
303 | for centre, project, node_type, sub_project in result: |
---|
304 | if centre not in centre_dict: |
---|
305 | centre_dict[centre] = collections.OrderedDict() |
---|
306 | if project not in centre_dict[centre]: |
---|
307 | centre_dict[centre][project] = [] |
---|
308 | if (node_type, sub_project) not in centre_dict[centre][project]: |
---|
309 | centre_dict[centre][project].append((node_type, sub_project)) |
---|
310 | |
---|
311 | pp.pprint(centre_dict) |
---|
312 | |
---|
313 | # .. Data for html index file .. |
---|
314 | # ============================== |
---|
315 | idx_header = html_header("GENCI", css_file) |
---|
316 | idx_footer = html_footer() |
---|
317 | idx_main = html_main_index() |
---|
318 | |
---|
319 | # .. Loop over centres .. |
---|
320 | # ======================= |
---|
321 | for centre, projects in centre_dict.iteritems(): |
---|
322 | print("\n" + centre + "\n=====") |
---|
323 | |
---|
324 | # ... Add centre to index page ... |
---|
325 | # -------------------------------- |
---|
326 | idx_main = idx_main + ( |
---|
327 | "\n" |
---|
328 | " <div class=\"img\">\n" |
---|
329 | " <table>\n" |
---|
330 | " <tr>\n" |
---|
331 | " <th>{}</th>\n" |
---|
332 | " </tr>\n" |
---|
333 | .format(centre.upper()) |
---|
334 | ) |
---|
335 | |
---|
336 | # ... Loop over projects ... |
---|
337 | # -------------------------- |
---|
338 | for project, node_subp in projects.iteritems(): |
---|
339 | print(project + "\n--------") |
---|
340 | |
---|
341 | # Build project card if it doesn't already exist |
---|
342 | config_write(centre, project) |
---|
343 | |
---|
344 | # Build html page for project |
---|
345 | # subdir = os.path.join( |
---|
346 | # HTML_DIR, |
---|
347 | # centre, |
---|
348 | # ) |
---|
349 | |
---|
350 | project_file = os.path.join( |
---|
351 | # subdir, |
---|
352 | "html", |
---|
353 | "{}_{}.html".format(centre, project) |
---|
354 | ) |
---|
355 | |
---|
356 | # if not os.path.isfile(project_file): |
---|
357 | html_project( |
---|
358 | centre, |
---|
359 | project, |
---|
360 | node_subp, |
---|
361 | os.path.join("web", project_file), |
---|
362 | ) |
---|
363 | |
---|
364 | # Add project to index page |
---|
365 | idx_main = idx_main + ( |
---|
366 | " <tr>\n" |
---|
367 | " <td>\n" |
---|
368 | " <a href=\"{}\">\n" |
---|
369 | " {}\n" |
---|
370 | " </a>\n" |
---|
371 | " </td>\n" |
---|
372 | " </tr>\n" |
---|
373 | .format(project_file, project.upper()) |
---|
374 | ) |
---|
375 | |
---|
376 | # Loop over node types |
---|
377 | for (node_type, sub_project) in node_subp: |
---|
378 | print("- {} / {}".format(node_type, sub_project)) |
---|
379 | |
---|
380 | # ${ROOT_DIR}/bin/plot_bilan.py -vf --png gencmip6 tgcc thin/standard |
---|
381 | # --subproject dcpcmip6 |
---|
382 | command = [ |
---|
383 | os.path.join(ROOT_DIR, "bin", "plot_bilan.py"), |
---|
384 | "-vf", |
---|
385 | "--png", |
---|
386 | project, |
---|
387 | centre, |
---|
388 | node_type, |
---|
389 | ] |
---|
390 | if sub_project: |
---|
391 | command.extend(["--subproject", sub_project]) |
---|
392 | |
---|
393 | print(command) |
---|
394 | if not args.dryrun: |
---|
395 | try: |
---|
396 | subprocess.call(command) |
---|
397 | except Exception as rc: |
---|
398 | print("Error in plot for {}:\n{}".format(command, rc)) |
---|
399 | else: |
---|
400 | print(command) |
---|
401 | |
---|
402 | # Copy today's file in web directory |
---|
403 | filein = "bilan_{ce}_{pj}_{sp}_{nd}_{dt:%Y%m%d}".format( |
---|
404 | ce=centre, |
---|
405 | pj=project, |
---|
406 | sp=sub_project if sub_project else project, |
---|
407 | nd=node_type.replace("/", ""), |
---|
408 | dt=dt.datetime.today(), |
---|
409 | ) |
---|
410 | fileout = "bilan_{ce}_{pj}_{sp}_{nd}".format( |
---|
411 | ce=centre, |
---|
412 | pj=project, |
---|
413 | sp=sub_project if sub_project else project, |
---|
414 | nd=node_type.replace("/", ""), |
---|
415 | ) |
---|
416 | |
---|
417 | dirin = os.path.join( |
---|
418 | PLOT_DIR, |
---|
419 | centre, |
---|
420 | project, |
---|
421 | ) |
---|
422 | pdfout = os.path.join( |
---|
423 | HTML_DIR, |
---|
424 | # centre, |
---|
425 | "pdf", |
---|
426 | ) |
---|
427 | pngout = os.path.join( |
---|
428 | HTML_DIR, |
---|
429 | # centre, |
---|
430 | "img", |
---|
431 | ) |
---|
432 | |
---|
433 | dirnames = [pdfout, pngout] |
---|
434 | extnames = ["pdf", "png"] |
---|
435 | |
---|
436 | for dirname, extname in zip(dirnames, extnames): |
---|
437 | if not os.path.isdir(dirname): |
---|
438 | print("mkdir {}".format(dirname)) |
---|
439 | try: |
---|
440 | os.makedirs(dirname) |
---|
441 | except Exception as rc: |
---|
442 | print("Could not create {}:\n{}".format(dirname, rc)) |
---|
443 | |
---|
444 | if not args.dryrun: |
---|
445 | try: |
---|
446 | shutil.copy( |
---|
447 | os.path.join( |
---|
448 | dirin, |
---|
449 | "{}.{}".format(filein, extname), |
---|
450 | ), |
---|
451 | os.path.join( |
---|
452 | dirname, |
---|
453 | "{}.{}".format(fileout, extname), |
---|
454 | ), |
---|
455 | ) |
---|
456 | except Exception as rc: |
---|
457 | print("Could not copy {}.{}:\n{}".format(filein, extname, rc)) |
---|
458 | else: |
---|
459 | print( |
---|
460 | "{} => {}".format( |
---|
461 | os.path.join( |
---|
462 | dirin, |
---|
463 | "{}.{}".format(filein, extname), |
---|
464 | ), |
---|
465 | os.path.join( |
---|
466 | dirname, |
---|
467 | "{}.{}".format(fileout, extname), |
---|
468 | ), |
---|
469 | ) |
---|
470 | ) |
---|
471 | |
---|
472 | # shutil.copy( |
---|
473 | # os.path.join(dirin, filein + ".pdf"), |
---|
474 | # os.path.join(pdfout, fileout + ".pdf") |
---|
475 | # ) |
---|
476 | # shutil.copy( |
---|
477 | # os.path.join(dirin, filein + ".png"), |
---|
478 | # os.path.join(pngout, fileout + ".png") |
---|
479 | # ) |
---|
480 | |
---|
481 | # ... Close list for current centre ... |
---|
482 | # ------------------------------------- |
---|
483 | idx_main = idx_main + ( |
---|
484 | " </table>\n" |
---|
485 | " </div>\n" |
---|
486 | ) |
---|
487 | |
---|
488 | # .. Write index file .. |
---|
489 | # ====================== |
---|
490 | with open(index_file, "w") as fileout: |
---|
491 | fileout.write(idx_header + idx_main + idx_footer) |
---|
492 | |
---|
493 | date_fin = dt.datetime.now() |
---|
494 | print(date_fin - date_deb) |
---|
495 | |
---|
496 | # .. Copy everything to dods .. |
---|
497 | # ============================= |
---|
498 | print("Copy everything to dods") |
---|
499 | if not args.dryrun: |
---|
500 | try: |
---|
501 | res = copy_tree( |
---|
502 | HTML_DIR, |
---|
503 | DODS_DIR, |
---|
504 | preserve_mode=1, |
---|
505 | preserve_times=1, |
---|
506 | update=1 |
---|
507 | ) |
---|
508 | except Exception as rc: |
---|
509 | print("Error while copying to dods:\n{}".format(rc)) |
---|
510 | else: |
---|
511 | print("{} => {}".format(HTML_DIR, DODS_DIR)) |
---|
512 | |
---|
513 | if args.verbose: |
---|
514 | print(res) |
---|
515 | |
---|
516 | exit() |
---|
517 | |
---|
518 | |
---|
519 | |
---|
520 | |
---|
521 | |
---|
522 | |
---|
523 | |
---|
524 | |
---|
525 | |
---|
526 | |
---|
527 | |
---|
528 | |
---|
529 | |
---|
530 | |
---|
531 | |
---|
532 | |
---|
533 | |
---|
534 | |
---|
535 | |
---|
536 | |
---|
537 | |
---|
538 | |
---|
539 | for key in project_dict: |
---|
540 | project_dict[key] = [p for c, p in result if c == key] |
---|
541 | # .. Loop over centres .. |
---|
542 | # ======================= |
---|
543 | for centre, project_list in project_dict.iteritems(): |
---|
544 | print("\n" + centre) |
---|
545 | |
---|
546 | # ... Add centre to index page ... |
---|
547 | # -------------------------------- |
---|
548 | idx_main = idx_main + ( |
---|
549 | '\n' |
---|
550 | ' <h2>{}</h2>\n' |
---|
551 | ' <ul>\n'.format(centre.upper()) |
---|
552 | ) |
---|
553 | |
---|
554 | # ... Loop over projects ... |
---|
555 | # -------------------------- |
---|
556 | for project in project_list: |
---|
557 | print(project) |
---|
558 | |
---|
559 | # Build project card if it doesn't already exist |
---|
560 | config_write(centre, project) |
---|
561 | |
---|
562 | # Build html page for project |
---|
563 | subdir = os.path.join( |
---|
564 | HTML_DIR, |
---|
565 | centre, |
---|
566 | ) |
---|
567 | |
---|
568 | project_file = os.path.join( |
---|
569 | subdir, |
---|
570 | "{}_{}.html".format(centre, project) |
---|
571 | ) |
---|
572 | |
---|
573 | if not os.path.isfile(project_file): |
---|
574 | html_project(centre, project, project_file) |
---|
575 | |
---|
576 | # Add project to index page |
---|
577 | idx_main = idx_main + ( |
---|
578 | ' <li>\n' |
---|
579 | ' <a href="{}">{}</a>\n' |
---|
580 | ' </li>\n'.format( |
---|
581 | project_file, project.upper() |
---|
582 | ) |
---|
583 | ) |
---|
584 | |
---|
585 | # ... Close list for current centre ... |
---|
586 | # ------------------------------------- |
---|
587 | idx_main = idx_main + ' </ul>\n' |
---|
588 | |
---|
589 | # .. Write index file .. |
---|
590 | # ====================== |
---|
591 | with open(index_file, "w") as fileout: |
---|
592 | fileout.write(idx_header + idx_main + idx_footer) |
---|
593 | |
---|
594 | # .. Launch plot jobs .. |
---|
595 | # ====================== |
---|
596 | |
---|
597 | # allocs = [] |
---|
598 | # for row in cursor: |
---|
599 | # allocs.append(ccpt.AllocRow(row)) |
---|
600 | |
---|
601 | # # .. Process files .. |
---|
602 | # # =================== |
---|
603 | # for filename in filelist: |
---|
604 | |
---|
605 | # fileout = os.path.join(dirout, os.path.basename(filename)) |
---|
606 | # if os.path.isfile(fileout): |
---|
607 | # print( |
---|
608 | # "file {} already processed, remove file".format( |
---|
609 | # os.path.basename(filename) |
---|
610 | # ) |
---|
611 | # ) |
---|
612 | # if not args.dryrun: |
---|
613 | # try: |
---|
614 | # os.remove(filename) |
---|
615 | # except Exception as rc: |
---|
616 | # print("Could not move {}:\n{}".format(filename, rc)) |
---|
617 | # break |
---|
618 | |
---|
619 | # # .. Build dictionary from files .. |
---|
620 | # # ================================= |
---|
621 | # conso_per_alloc = {} |
---|
622 | |
---|
623 | # date, blocs = ccpt.parse_input_cpt( |
---|
624 | # filename, args.project, args.center, mode_conso=True |
---|
625 | # ) |
---|
626 | |
---|
627 | # for bloc in blocs: |
---|
628 | # if not bloc.alloc_id: |
---|
629 | # bloc.alloc_id = ccpt.get_project_id(bloc, allocs) |
---|
630 | # if not bloc.alloc_id: |
---|
631 | # print("no alloc id found, skip bloc") |
---|
632 | # print(bloc.machine, bloc.node, bloc.cpt_date) |
---|
633 | # pp.pprint(allocs) |
---|
634 | # continue |
---|
635 | # if bloc.alloc_id not in conso_per_alloc: |
---|
636 | # conso_per_alloc[bloc.alloc_id] = set() |
---|
637 | # for conso in bloc.consos: |
---|
638 | # conso_per_alloc[bloc.alloc_id].add(conso) |
---|
639 | |
---|
640 | # # .. Insert data in table, one alloc_id at a time .. |
---|
641 | # # ================================================== |
---|
642 | # for alloc_id, consos in conso_per_alloc.iteritems(): |
---|
643 | # # ... Create request sub string ... |
---|
644 | # # --------------------------------- |
---|
645 | # lines_req = [ |
---|
646 | # ( |
---|
647 | # "('{alloc}', " |
---|
648 | # "'{date}', " |
---|
649 | # "{total_hrs}, " |
---|
650 | # "{login}, " |
---|
651 | # "{create})" |
---|
652 | # ) .format( |
---|
653 | # alloc=alloc_id, |
---|
654 | # date=item.date, |
---|
655 | # total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'", |
---|
656 | # login="'"+item.login+"'" if item.login != "total" else "NULL", |
---|
657 | # create="CURRENT_TIMESTAMP", |
---|
658 | # ) for item in consos |
---|
659 | # ] |
---|
660 | |
---|
661 | # # ... Create full request ... |
---|
662 | # # --------------------------- |
---|
663 | # table_name = "conso.tbl_consumption" |
---|
664 | # request = ( |
---|
665 | # "INSERT INTO " + table_name + " (" |
---|
666 | # " allocation_id, " |
---|
667 | # " date, " |
---|
668 | # " total_hrs, " |
---|
669 | # " login, " |
---|
670 | # " row_create_date " |
---|
671 | # ") " |
---|
672 | # "VALUES " |
---|
673 | # ) |
---|
674 | # request = request + ", ".join(lines_req) |
---|
675 | |
---|
676 | # # ... Execute request ... |
---|
677 | # # ----------------------- |
---|
678 | # if args.verbose: |
---|
679 | # print("Execute request for alloc_id = {}".format(alloc_id)) |
---|
680 | # print(request) |
---|
681 | # cdb.insert_db(cursor, request) |
---|
682 | |
---|
683 | # # ... Commit inserts ... |
---|
684 | # # ---------------------- |
---|
685 | # if not args.dryrun: |
---|
686 | # if args.verbose: |
---|
687 | # print("Commit inserts") |
---|
688 | # cdb.commit_db(conn) |
---|
689 | |
---|
690 | # # .. Move processed file to project directory .. |
---|
691 | # # =============================================== |
---|
692 | # if args.verbose: |
---|
693 | # print("Move processed file:") |
---|
694 | # print("{} => {}".format(filename, dirout)) |
---|
695 | # if not args.dryrun: |
---|
696 | # try: |
---|
697 | # shutil.move(filename, dirout) |
---|
698 | # except Exception as rc: |
---|
699 | # print("Could not move {}:\n{}".format(filename, rc)) |
---|
700 | |
---|
701 | exit(0) |
---|