[3083] | 1 | #!/usr/bin/env python |
---|
| 2 | # -*- coding: utf-8 -*- |
---|
| 3 | |
---|
| 4 | # ==================================================================== # |
---|
| 5 | # Author: Sonia Labetoulle # |
---|
| 6 | # Contact: sonia.labetoulle _at_ ipsl.jussieu.fr # |
---|
| 7 | # Created: 2016 # |
---|
| 8 | # History: # |
---|
| 9 | # Modification: # |
---|
| 10 | # ==================================================================== # |
---|
| 11 | |
---|
| 12 | # This must come first |
---|
| 13 | from __future__ import print_function, unicode_literals, division |
---|
| 14 | |
---|
| 15 | # Standard library imports |
---|
| 16 | import os |
---|
| 17 | # import math |
---|
| 18 | from argparse import ArgumentParser |
---|
| 19 | import shutil |
---|
| 20 | import pprint |
---|
| 21 | import collections |
---|
| 22 | import ConfigParser as cp |
---|
| 23 | import datetime as dt |
---|
| 24 | import subprocess |
---|
| 25 | from distutils.dir_util import copy_tree |
---|
| 26 | |
---|
| 27 | # Application library imports |
---|
| 28 | import libconso_db as cdb |
---|
| 29 | # import libconso_cpt as ccpt |
---|
| 30 | import db_data |
---|
| 31 | |
---|
| 32 | pp = pprint.PrettyPrinter(indent=2) |
---|
| 33 | |
---|
| 34 | |
---|
| 35 | ####################################################################### |
---|
| 36 | def get_arguments(): |
---|
| 37 | parser = ArgumentParser() |
---|
| 38 | # parser.add_argument("project", action="store", |
---|
| 39 | # help="Project name") |
---|
| 40 | # parser.add_argument("center", action="store", |
---|
| 41 | # help="Center name (idris/tgcc)") |
---|
| 42 | |
---|
| 43 | parser.add_argument("-v", "--verbose", action="store_true", |
---|
| 44 | help="verbose mode") |
---|
| 45 | parser.add_argument("-d", "--dryrun", action="store_true", |
---|
| 46 | help="only print what is to be done") |
---|
| 47 | # parser.add_argument("-r", "--range", action="store", nargs=2, |
---|
| 48 | # help="date range: ssaammjj ssaammjj") |
---|
| 49 | return parser.parse_args() |
---|
| 50 | |
---|
| 51 | |
---|
| 52 | ######################################################################## |
---|
| 53 | def html_header(project, css_file): |
---|
| 54 | html = ( |
---|
| 55 | "<!DOCTYPE html>\n" |
---|
| 56 | "<html lang=\"fr-FR\">\n" |
---|
| 57 | "<head>\n" |
---|
| 58 | " <meta charset=\"UTF-8\">\n" |
---|
| 59 | " <meta name=\"description\" content=\"Suivi de consommation {pj}\">\n" |
---|
| 60 | " <meta name=\"author\" content=\"Sonia Labetoulle\">\n" |
---|
| 61 | " <title>Suivi de consommation {pj}</title>\n" |
---|
| 62 | " <link rel=\"stylesheet\" type=\"text/css\" " |
---|
| 63 | "href=\"{css}\">\n" |
---|
| 64 | "</head>\n" |
---|
| 65 | "\n" |
---|
| 66 | "<body>\n" |
---|
| 67 | " <div id=\"main\">\n" |
---|
| 68 | "\n".format(pj=project.upper(), css=css_file) |
---|
| 69 | ) |
---|
| 70 | |
---|
| 71 | return html |
---|
| 72 | |
---|
| 73 | |
---|
| 74 | ######################################################################## |
---|
| 75 | def html_footer(): |
---|
| 76 | html = ( |
---|
| 77 | "\n" |
---|
| 78 | " <div class=\"clear\"></div>\n" |
---|
| 79 | " </div>\n" |
---|
| 80 | "</body>\n" |
---|
| 81 | "</html>\n" |
---|
| 82 | ) |
---|
| 83 | |
---|
| 84 | return html |
---|
| 85 | |
---|
| 86 | |
---|
| 87 | ######################################################################## |
---|
| 88 | def html_main_index(): |
---|
| 89 | html = ( |
---|
| 90 | " <h1>Conso GENCI</h1>\n" |
---|
| 91 | ) |
---|
| 92 | |
---|
| 93 | return html |
---|
| 94 | |
---|
| 95 | |
---|
| 96 | ######################################################################## |
---|
| 97 | def html_main_project(centre, project, node_subp): |
---|
| 98 | |
---|
| 99 | html = ( |
---|
| 100 | " <h1>Conso {pj} ({ce})</h1>\n\n" |
---|
| 101 | .format(ce=centre.upper(), pj=project.upper()) |
---|
| 102 | ) |
---|
| 103 | |
---|
| 104 | for (node_type, sub_project) in node_subp: |
---|
| 105 | if sub_project: |
---|
| 106 | title = "Sous-imputation {sp} / {nd}".format( |
---|
| 107 | sp=sub_project if sub_project else project, |
---|
| 108 | nd=node_type.replace("/", "") |
---|
| 109 | ) |
---|
| 110 | else: |
---|
| 111 | title = "Conso globale {nd}".format( |
---|
| 112 | nd=node_type.replace("/", "") |
---|
| 113 | ) |
---|
| 114 | html = html + ( |
---|
| 115 | " <div class=\"img\">\n" |
---|
| 116 | " <h2>{tl}</h2>\n" |
---|
| 117 | " <a href=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\">\n" |
---|
| 118 | " <img class=\"data\"\n" |
---|
| 119 | " src=\"../img/bilan_{ce}_{pj}_{sp}_{nd}.png\"\n" |
---|
| 120 | " alt=\"{tl}\" />\n" |
---|
| 121 | " </a><br />\n" |
---|
| 122 | " <div class=\"lien\">\n" |
---|
| 123 | " <a href=\"../pdf/bilan_{ce}_{pj}_{sp}_{nd}.pdf\">\n" |
---|
| 124 | " <!--<img class=\"icon\" src=\"utils/pdf_2.jpg\" />-->\n" |
---|
| 125 | " Fichier pdf\n" |
---|
| 126 | " </a>\n" |
---|
| 127 | " </div>\n" |
---|
| 128 | " </div>\n".format( |
---|
| 129 | tl=title, |
---|
| 130 | ce=centre, |
---|
| 131 | pj=project, |
---|
| 132 | sp=sub_project if sub_project else project, |
---|
| 133 | nd=node_type.replace("/", "") |
---|
| 134 | ) |
---|
| 135 | ) |
---|
| 136 | |
---|
| 137 | return html |
---|
| 138 | |
---|
| 139 | |
---|
| 140 | ######################################################################## |
---|
| 141 | def html_project(centre, project, node_subp, project_file): |
---|
| 142 | |
---|
| 143 | if args.verbose: |
---|
| 144 | print("Create {}".format(project_file)) |
---|
| 145 | |
---|
| 146 | header = html_header(project, os.path.join("..", css_file)) |
---|
| 147 | footer = html_footer() |
---|
| 148 | main = html_main_project(centre, project, node_subp) |
---|
| 149 | |
---|
| 150 | subdir = os.path.dirname(project_file) |
---|
| 151 | if not os.path.isdir(subdir): |
---|
| 152 | try: |
---|
| 153 | os.mkdir(subdir) |
---|
| 154 | except Exception as rc: |
---|
| 155 | raise("Could not create directory {}:\n{}".format(subdir, rc)) |
---|
| 156 | |
---|
| 157 | with open(project_file, "w") as fileout: |
---|
| 158 | try: |
---|
| 159 | fileout.write(header + main + footer) |
---|
| 160 | except Exception as rc: |
---|
| 161 | raise("Could not create directory {}:\n{}".format(project_file, rc)) |
---|
| 162 | |
---|
| 163 | |
---|
| 164 | ######################################################################## |
---|
| 165 | def config_write(centre, project): |
---|
| 166 | config_file = os.path.join( |
---|
| 167 | CARD_DIR, |
---|
| 168 | "config_{}_{}.ini".format(centre, project) |
---|
| 169 | ) |
---|
| 170 | |
---|
| 171 | if not os.path.isfile(config_file): |
---|
| 172 | if args.verbose: |
---|
| 173 | print("Create {}".format(config_file)) |
---|
| 174 | |
---|
| 175 | config = cp.ConfigParser(allow_no_value=True) |
---|
| 176 | config.optionxform = str |
---|
| 177 | |
---|
| 178 | config.add_section("projet") |
---|
| 179 | config.set("projet", "name", project) |
---|
| 180 | |
---|
| 181 | config.add_section("directories") |
---|
| 182 | # config.set("directories", "ROOT_DIR", "/home/igcmg/ConsoGENCI") |
---|
| 183 | config.set("directories", "ROOT_DIR", ROOT_DIR) |
---|
| 184 | config.set("directories", "DATA", "%(ROOT_DIR)s/data") |
---|
| 185 | config.set("directories", "PLOT", "%(ROOT_DIR)s/plot") |
---|
| 186 | config.set("directories", "DODS") |
---|
| 187 | |
---|
| 188 | config.add_section("dods") |
---|
| 189 | config.set("dods", "DIR", "/home/igcmg/dods/ConsoGENCI") |
---|
| 190 | config.set("dods", "USER", "igcmg") |
---|
| 191 | config.set("dods", "SERVER", "ciclad") |
---|
| 192 | |
---|
| 193 | with open(config_file, "w") as fileout: |
---|
| 194 | try: |
---|
| 195 | config.write(fileout) |
---|
| 196 | except Exception as rc: |
---|
| 197 | raise("Could not write {}:\n{}".format(config_file, rc)) |
---|
| 198 | |
---|
| 199 | |
---|
| 200 | ####################################################################### |
---|
| 201 | if __name__ == "__main__": |
---|
| 202 | |
---|
| 203 | date_deb = dt.datetime.now() |
---|
| 204 | |
---|
| 205 | # .. Initialization .. |
---|
| 206 | # ==================== |
---|
| 207 | # ... Command line arguments ... |
---|
| 208 | # ------------------------------ |
---|
| 209 | args = get_arguments() |
---|
| 210 | if args.verbose: |
---|
| 211 | print(args) |
---|
| 212 | |
---|
| 213 | # ... Files and directories ... |
---|
| 214 | # ----------------------------- |
---|
| 215 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
---|
| 216 | CARD_DIR = os.path.join(ROOT_DIR, "card") |
---|
| 217 | DATA_DIR = os.path.join(ROOT_DIR, "data") |
---|
| 218 | PLOT_DIR = os.path.join(ROOT_DIR, "plot") |
---|
| 219 | HTML_DIR = os.path.join(ROOT_DIR, "web") |
---|
| 220 | DODS_DIR = os.path.join( |
---|
| 221 | os.path.dirname(ROOT_DIR), |
---|
| 222 | "dods", |
---|
| 223 | "ConsoGENCI", |
---|
| 224 | ) |
---|
| 225 | SUBMIT_DIR = os.getcwd() |
---|
| 226 | index_file = os.path.join( |
---|
| 227 | HTML_DIR, |
---|
| 228 | "index_full.html" |
---|
| 229 | ) |
---|
| 230 | # css_file = os.path.join( |
---|
| 231 | # HTML_DIR, |
---|
| 232 | # "utils", |
---|
| 233 | # "ConsoGENCI.css" |
---|
| 234 | # ) |
---|
| 235 | css_file = os.path.join( |
---|
| 236 | "utils", |
---|
| 237 | "ConsoGENCI.css" |
---|
| 238 | ) |
---|
| 239 | |
---|
| 240 | # pattern = ccpt.cpt_pattern(center=args.center, project=args.project) |
---|
| 241 | # dirin = os.path.join(DATA_DIR, args.center, "tmp") |
---|
| 242 | # dirout = os.path.join(DATA_DIR, args.center, args.project) |
---|
| 243 | # filelist = ccpt.find_input_files(dirin, pattern, args.range) |
---|
| 244 | |
---|
| 245 | # if not filelist: |
---|
| 246 | # print("No input files found, exit") |
---|
| 247 | # exit(1) |
---|
| 248 | |
---|
| 249 | # if args.verbose: |
---|
| 250 | # print("SUBMIT_DIR:", SUBMIT_DIR) |
---|
| 251 | # print("DATA_DIR:", DATA_DIR) |
---|
| 252 | # print("dirin:", dirin) |
---|
| 253 | |
---|
| 254 | # .. Connection to database .. |
---|
| 255 | # ============================ |
---|
| 256 | if args.verbose: |
---|
| 257 | print("Connection to database") |
---|
| 258 | conn, cursor = cdb.connect_db( |
---|
| 259 | db_data.db_host, |
---|
| 260 | db_data.db_name, |
---|
| 261 | db_data.db_user, |
---|
| 262 | db_data.db_pwd, |
---|
| 263 | ) |
---|
| 264 | |
---|
| 265 | # print(DODS_DIR) |
---|
| 266 | |
---|
| 267 | # exit() |
---|
| 268 | |
---|
| 269 | # .. Extract centre/project list from table .. |
---|
| 270 | # ============================================ |
---|
| 271 | table_name = "conso.tbl_allocation" |
---|
| 272 | request = ( |
---|
| 273 | "SELECT DISTINCT A.centre, A.project, A.node_type, C.sub_project " |
---|
| 274 | "FROM conso.tbl_allocation AS A " |
---|
| 275 | "INNER JOIN conso.tbl_consumption AS C " |
---|
| 276 | "ON A.id = C.allocation_id " |
---|
| 277 | "WHERE A.is_active = TRUE " |
---|
| 278 | "ORDER BY A.centre, A.project, A.node_type, C.sub_project ;" |
---|
| 279 | ) |
---|
| 280 | |
---|
| 281 | cdb.select_db(cursor, request) |
---|
| 282 | # print(cursor.rowcount) |
---|
| 283 | # print(cursor.fetchall()) |
---|
| 284 | result = list(cursor) |
---|
| 285 | |
---|
| 286 | # .. Close connection to database .. |
---|
| 287 | # ================================== |
---|
| 288 | if args.verbose: |
---|
| 289 | print("Close connection") |
---|
| 290 | cdb.close_db(conn) |
---|
| 291 | |
---|
| 292 | # .. Build data dictionary .. |
---|
| 293 | # =========================== |
---|
| 294 | centre_dict = collections.OrderedDict() |
---|
| 295 | # for centre, project, node_type, sub_project in result: |
---|
| 296 | # if centre not in centre_dict: |
---|
| 297 | # centre_dict[centre] = collections.OrderedDict() |
---|
| 298 | # if project not in centre_dict[centre]: |
---|
| 299 | # centre_dict[centre][project] = [] |
---|
| 300 | # if node_type not in centre_dict[centre][project]: |
---|
| 301 | # centre_dict[centre][project].append(node_type) |
---|
| 302 | |
---|
| 303 | for centre, project, node_type, sub_project in result: |
---|
| 304 | if centre not in centre_dict: |
---|
| 305 | centre_dict[centre] = collections.OrderedDict() |
---|
| 306 | if project not in centre_dict[centre]: |
---|
| 307 | centre_dict[centre][project] = [] |
---|
| 308 | if (node_type, sub_project) not in centre_dict[centre][project]: |
---|
| 309 | centre_dict[centre][project].append((node_type, sub_project)) |
---|
| 310 | |
---|
| 311 | pp.pprint(centre_dict) |
---|
| 312 | |
---|
| 313 | # .. Data for html index file .. |
---|
| 314 | # ============================== |
---|
| 315 | idx_header = html_header("GENCI", css_file) |
---|
| 316 | idx_footer = html_footer() |
---|
| 317 | idx_main = html_main_index() |
---|
| 318 | |
---|
| 319 | # .. Loop over centres .. |
---|
| 320 | # ======================= |
---|
| 321 | for centre, projects in centre_dict.iteritems(): |
---|
| 322 | print("\n" + centre + "\n=====") |
---|
| 323 | |
---|
| 324 | # ... Add centre to index page ... |
---|
| 325 | # -------------------------------- |
---|
| 326 | idx_main = idx_main + ( |
---|
| 327 | "\n" |
---|
| 328 | " <div class=\"img\">\n" |
---|
| 329 | " <table>\n" |
---|
| 330 | " <tr>\n" |
---|
| 331 | " <th>{}</th>\n" |
---|
| 332 | " </tr>\n" |
---|
| 333 | .format(centre.upper()) |
---|
| 334 | ) |
---|
| 335 | |
---|
| 336 | # ... Loop over projects ... |
---|
| 337 | # -------------------------- |
---|
| 338 | for project, node_subp in projects.iteritems(): |
---|
| 339 | print(project + "\n--------") |
---|
| 340 | |
---|
| 341 | # Build project card if it doesn't already exist |
---|
| 342 | config_write(centre, project) |
---|
| 343 | |
---|
| 344 | # Build html page for project |
---|
| 345 | # subdir = os.path.join( |
---|
| 346 | # HTML_DIR, |
---|
| 347 | # centre, |
---|
| 348 | # ) |
---|
| 349 | |
---|
| 350 | project_file = os.path.join( |
---|
| 351 | # subdir, |
---|
| 352 | "html", |
---|
| 353 | "{}_{}.html".format(centre, project) |
---|
| 354 | ) |
---|
| 355 | |
---|
| 356 | # if not os.path.isfile(project_file): |
---|
| 357 | html_project( |
---|
| 358 | centre, |
---|
| 359 | project, |
---|
| 360 | node_subp, |
---|
| 361 | os.path.join("web", project_file), |
---|
| 362 | ) |
---|
| 363 | |
---|
| 364 | # Add project to index page |
---|
| 365 | idx_main = idx_main + ( |
---|
| 366 | " <tr>\n" |
---|
| 367 | " <td>\n" |
---|
| 368 | " <a href=\"{}\">\n" |
---|
| 369 | " {}\n" |
---|
| 370 | " </a>\n" |
---|
| 371 | " </td>\n" |
---|
| 372 | " </tr>\n" |
---|
| 373 | .format(project_file, project.upper()) |
---|
| 374 | ) |
---|
| 375 | |
---|
| 376 | # Loop over node types |
---|
| 377 | for (node_type, sub_project) in node_subp: |
---|
| 378 | print("- {} / {}".format(node_type, sub_project)) |
---|
| 379 | |
---|
| 380 | # ${ROOT_DIR}/bin/plot_bilan.py -vf --png gencmip6 tgcc thin/standard |
---|
| 381 | # --subproject dcpcmip6 |
---|
| 382 | command = [ |
---|
| 383 | os.path.join(ROOT_DIR, "bin", "plot_bilan.py"), |
---|
| 384 | "-vf", |
---|
| 385 | "--png", |
---|
| 386 | project, |
---|
| 387 | centre, |
---|
| 388 | node_type, |
---|
| 389 | ] |
---|
| 390 | if sub_project: |
---|
| 391 | command.extend(["--subproject", sub_project]) |
---|
| 392 | |
---|
| 393 | print(command) |
---|
| 394 | if not args.dryrun: |
---|
| 395 | try: |
---|
| 396 | subprocess.call(command) |
---|
| 397 | except Exception as rc: |
---|
| 398 | print("Error in plot for {}:\n{}".format(command, rc)) |
---|
| 399 | else: |
---|
| 400 | print(command) |
---|
| 401 | |
---|
| 402 | # Copy today's file in web directory |
---|
| 403 | filein = "bilan_{ce}_{pj}_{sp}_{nd}_{dt:%Y%m%d}".format( |
---|
| 404 | ce=centre, |
---|
| 405 | pj=project, |
---|
| 406 | sp=sub_project if sub_project else project, |
---|
| 407 | nd=node_type.replace("/", ""), |
---|
| 408 | dt=dt.datetime.today(), |
---|
| 409 | ) |
---|
| 410 | fileout = "bilan_{ce}_{pj}_{sp}_{nd}".format( |
---|
| 411 | ce=centre, |
---|
| 412 | pj=project, |
---|
| 413 | sp=sub_project if sub_project else project, |
---|
| 414 | nd=node_type.replace("/", ""), |
---|
| 415 | ) |
---|
| 416 | |
---|
| 417 | dirin = os.path.join( |
---|
| 418 | PLOT_DIR, |
---|
| 419 | centre, |
---|
| 420 | project, |
---|
| 421 | ) |
---|
| 422 | pdfout = os.path.join( |
---|
| 423 | HTML_DIR, |
---|
| 424 | # centre, |
---|
| 425 | "pdf", |
---|
| 426 | ) |
---|
| 427 | pngout = os.path.join( |
---|
| 428 | HTML_DIR, |
---|
| 429 | # centre, |
---|
| 430 | "img", |
---|
| 431 | ) |
---|
| 432 | |
---|
| 433 | dirnames = [pdfout, pngout] |
---|
| 434 | extnames = ["pdf", "png"] |
---|
| 435 | |
---|
| 436 | for dirname, extname in zip(dirnames, extnames): |
---|
| 437 | if not os.path.isdir(dirname): |
---|
| 438 | print("mkdir {}".format(dirname)) |
---|
| 439 | try: |
---|
| 440 | os.makedirs(dirname) |
---|
| 441 | except Exception as rc: |
---|
| 442 | print("Could not create {}:\n{}".format(dirname, rc)) |
---|
| 443 | |
---|
| 444 | if not args.dryrun: |
---|
| 445 | try: |
---|
| 446 | shutil.copy( |
---|
| 447 | os.path.join( |
---|
| 448 | dirin, |
---|
| 449 | "{}.{}".format(filein, extname), |
---|
| 450 | ), |
---|
| 451 | os.path.join( |
---|
| 452 | dirname, |
---|
| 453 | "{}.{}".format(fileout, extname), |
---|
| 454 | ), |
---|
| 455 | ) |
---|
| 456 | except Exception as rc: |
---|
| 457 | print("Could not copy {}.{}:\n{}".format(filein, extname, rc)) |
---|
| 458 | else: |
---|
| 459 | print( |
---|
| 460 | "{} => {}".format( |
---|
| 461 | os.path.join( |
---|
| 462 | dirin, |
---|
| 463 | "{}.{}".format(filein, extname), |
---|
| 464 | ), |
---|
| 465 | os.path.join( |
---|
| 466 | dirname, |
---|
| 467 | "{}.{}".format(fileout, extname), |
---|
| 468 | ), |
---|
| 469 | ) |
---|
| 470 | ) |
---|
| 471 | |
---|
| 472 | # shutil.copy( |
---|
| 473 | # os.path.join(dirin, filein + ".pdf"), |
---|
| 474 | # os.path.join(pdfout, fileout + ".pdf") |
---|
| 475 | # ) |
---|
| 476 | # shutil.copy( |
---|
| 477 | # os.path.join(dirin, filein + ".png"), |
---|
| 478 | # os.path.join(pngout, fileout + ".png") |
---|
| 479 | # ) |
---|
| 480 | |
---|
| 481 | # ... Close list for current centre ... |
---|
| 482 | # ------------------------------------- |
---|
| 483 | idx_main = idx_main + ( |
---|
| 484 | " </table>\n" |
---|
| 485 | " </div>\n" |
---|
| 486 | ) |
---|
| 487 | |
---|
| 488 | # .. Write index file .. |
---|
| 489 | # ====================== |
---|
| 490 | with open(index_file, "w") as fileout: |
---|
| 491 | fileout.write(idx_header + idx_main + idx_footer) |
---|
| 492 | |
---|
| 493 | date_fin = dt.datetime.now() |
---|
| 494 | print(date_fin - date_deb) |
---|
| 495 | |
---|
| 496 | # .. Copy everything to dods .. |
---|
| 497 | # ============================= |
---|
| 498 | print("Copy everything to dods") |
---|
| 499 | if not args.dryrun: |
---|
| 500 | try: |
---|
| 501 | res = copy_tree( |
---|
| 502 | HTML_DIR, |
---|
| 503 | DODS_DIR, |
---|
| 504 | preserve_mode=1, |
---|
| 505 | preserve_times=1, |
---|
| 506 | update=1 |
---|
| 507 | ) |
---|
| 508 | except Exception as rc: |
---|
| 509 | print("Error while copying to dods:\n{}".format(rc)) |
---|
| 510 | else: |
---|
| 511 | print("{} => {}".format(HTML_DIR, DODS_DIR)) |
---|
| 512 | |
---|
| 513 | if args.verbose: |
---|
| 514 | print(res) |
---|
| 515 | |
---|
| 516 | exit() |
---|
| 517 | |
---|
| 518 | |
---|
| 519 | |
---|
| 520 | |
---|
| 521 | |
---|
| 522 | |
---|
| 523 | |
---|
| 524 | |
---|
| 525 | |
---|
| 526 | |
---|
| 527 | |
---|
| 528 | |
---|
| 529 | |
---|
| 530 | |
---|
| 531 | |
---|
| 532 | |
---|
| 533 | |
---|
| 534 | |
---|
| 535 | |
---|
| 536 | |
---|
| 537 | |
---|
| 538 | |
---|
| 539 | for key in project_dict: |
---|
| 540 | project_dict[key] = [p for c, p in result if c == key] |
---|
| 541 | # .. Loop over centres .. |
---|
| 542 | # ======================= |
---|
| 543 | for centre, project_list in project_dict.iteritems(): |
---|
| 544 | print("\n" + centre) |
---|
| 545 | |
---|
| 546 | # ... Add centre to index page ... |
---|
| 547 | # -------------------------------- |
---|
| 548 | idx_main = idx_main + ( |
---|
| 549 | '\n' |
---|
| 550 | ' <h2>{}</h2>\n' |
---|
| 551 | ' <ul>\n'.format(centre.upper()) |
---|
| 552 | ) |
---|
| 553 | |
---|
| 554 | # ... Loop over projects ... |
---|
| 555 | # -------------------------- |
---|
| 556 | for project in project_list: |
---|
| 557 | print(project) |
---|
| 558 | |
---|
| 559 | # Build project card if it doesn't already exist |
---|
| 560 | config_write(centre, project) |
---|
| 561 | |
---|
| 562 | # Build html page for project |
---|
| 563 | subdir = os.path.join( |
---|
| 564 | HTML_DIR, |
---|
| 565 | centre, |
---|
| 566 | ) |
---|
| 567 | |
---|
| 568 | project_file = os.path.join( |
---|
| 569 | subdir, |
---|
| 570 | "{}_{}.html".format(centre, project) |
---|
| 571 | ) |
---|
| 572 | |
---|
| 573 | if not os.path.isfile(project_file): |
---|
| 574 | html_project(centre, project, project_file) |
---|
| 575 | |
---|
| 576 | # Add project to index page |
---|
| 577 | idx_main = idx_main + ( |
---|
| 578 | ' <li>\n' |
---|
| 579 | ' <a href="{}">{}</a>\n' |
---|
| 580 | ' </li>\n'.format( |
---|
| 581 | project_file, project.upper() |
---|
| 582 | ) |
---|
| 583 | ) |
---|
| 584 | |
---|
| 585 | # ... Close list for current centre ... |
---|
| 586 | # ------------------------------------- |
---|
| 587 | idx_main = idx_main + ' </ul>\n' |
---|
| 588 | |
---|
| 589 | # .. Write index file .. |
---|
| 590 | # ====================== |
---|
| 591 | with open(index_file, "w") as fileout: |
---|
| 592 | fileout.write(idx_header + idx_main + idx_footer) |
---|
| 593 | |
---|
| 594 | # .. Launch plot jobs .. |
---|
| 595 | # ====================== |
---|
| 596 | |
---|
| 597 | # allocs = [] |
---|
| 598 | # for row in cursor: |
---|
| 599 | # allocs.append(ccpt.AllocRow(row)) |
---|
| 600 | |
---|
| 601 | # # .. Process files .. |
---|
| 602 | # # =================== |
---|
| 603 | # for filename in filelist: |
---|
| 604 | |
---|
| 605 | # fileout = os.path.join(dirout, os.path.basename(filename)) |
---|
| 606 | # if os.path.isfile(fileout): |
---|
| 607 | # print( |
---|
| 608 | # "file {} already processed, remove file".format( |
---|
| 609 | # os.path.basename(filename) |
---|
| 610 | # ) |
---|
| 611 | # ) |
---|
| 612 | # if not args.dryrun: |
---|
| 613 | # try: |
---|
| 614 | # os.remove(filename) |
---|
| 615 | # except Exception as rc: |
---|
| 616 | # print("Could not move {}:\n{}".format(filename, rc)) |
---|
| 617 | # break |
---|
| 618 | |
---|
| 619 | # # .. Build dictionary from files .. |
---|
| 620 | # # ================================= |
---|
| 621 | # conso_per_alloc = {} |
---|
| 622 | |
---|
| 623 | # date, blocs = ccpt.parse_input_cpt( |
---|
| 624 | # filename, args.project, args.center, mode_conso=True |
---|
| 625 | # ) |
---|
| 626 | |
---|
| 627 | # for bloc in blocs: |
---|
| 628 | # if not bloc.alloc_id: |
---|
| 629 | # bloc.alloc_id = ccpt.get_project_id(bloc, allocs) |
---|
| 630 | # if not bloc.alloc_id: |
---|
| 631 | # print("no alloc id found, skip bloc") |
---|
| 632 | # print(bloc.machine, bloc.node, bloc.cpt_date) |
---|
| 633 | # pp.pprint(allocs) |
---|
| 634 | # continue |
---|
| 635 | # if bloc.alloc_id not in conso_per_alloc: |
---|
| 636 | # conso_per_alloc[bloc.alloc_id] = set() |
---|
| 637 | # for conso in bloc.consos: |
---|
| 638 | # conso_per_alloc[bloc.alloc_id].add(conso) |
---|
| 639 | |
---|
| 640 | # # .. Insert data in table, one alloc_id at a time .. |
---|
| 641 | # # ================================================== |
---|
| 642 | # for alloc_id, consos in conso_per_alloc.iteritems(): |
---|
| 643 | # # ... Create request sub string ... |
---|
| 644 | # # --------------------------------- |
---|
| 645 | # lines_req = [ |
---|
| 646 | # ( |
---|
| 647 | # "('{alloc}', " |
---|
| 648 | # "'{date}', " |
---|
| 649 | # "{total_hrs}, " |
---|
| 650 | # "{login}, " |
---|
| 651 | # "{create})" |
---|
| 652 | # ) .format( |
---|
| 653 | # alloc=alloc_id, |
---|
| 654 | # date=item.date, |
---|
| 655 | # total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'", |
---|
| 656 | # login="'"+item.login+"'" if item.login != "total" else "NULL", |
---|
| 657 | # create="CURRENT_TIMESTAMP", |
---|
| 658 | # ) for item in consos |
---|
| 659 | # ] |
---|
| 660 | |
---|
| 661 | # # ... Create full request ... |
---|
| 662 | # # --------------------------- |
---|
| 663 | # table_name = "conso.tbl_consumption" |
---|
| 664 | # request = ( |
---|
| 665 | # "INSERT INTO " + table_name + " (" |
---|
| 666 | # " allocation_id, " |
---|
| 667 | # " date, " |
---|
| 668 | # " total_hrs, " |
---|
| 669 | # " login, " |
---|
| 670 | # " row_create_date " |
---|
| 671 | # ") " |
---|
| 672 | # "VALUES " |
---|
| 673 | # ) |
---|
| 674 | # request = request + ", ".join(lines_req) |
---|
| 675 | |
---|
| 676 | # # ... Execute request ... |
---|
| 677 | # # ----------------------- |
---|
| 678 | # if args.verbose: |
---|
| 679 | # print("Execute request for alloc_id = {}".format(alloc_id)) |
---|
| 680 | # print(request) |
---|
| 681 | # cdb.insert_db(cursor, request) |
---|
| 682 | |
---|
| 683 | # # ... Commit inserts ... |
---|
| 684 | # # ---------------------- |
---|
| 685 | # if not args.dryrun: |
---|
| 686 | # if args.verbose: |
---|
| 687 | # print("Commit inserts") |
---|
| 688 | # cdb.commit_db(conn) |
---|
| 689 | |
---|
| 690 | # # .. Move processed file to project directory .. |
---|
| 691 | # # =============================================== |
---|
| 692 | # if args.verbose: |
---|
| 693 | # print("Move processed file:") |
---|
| 694 | # print("{} => {}".format(filename, dirout)) |
---|
| 695 | # if not args.dryrun: |
---|
| 696 | # try: |
---|
| 697 | # shutil.move(filename, dirout) |
---|
| 698 | # except Exception as rc: |
---|
| 699 | # print("Could not move {}:\n{}".format(filename, rc)) |
---|
| 700 | |
---|
| 701 | exit(0) |
---|