#!/usr/bin/env python # -*- coding: utf-8 -*- # ==================================================================== # # Author: Sonia Labetoulle # # Contact: sonia.labetoulle _at_ ipsl.jussieu.fr # # Created: 2016 # # History: # # Modification: # # ==================================================================== # # This must come first from __future__ import print_function, unicode_literals, division # Standard library imports import os # import math from argparse import ArgumentParser import shutil import pprint import collections import ConfigParser as cp import datetime as dt import subprocess from distutils.dir_util import copy_tree # Application library imports import libconso_db as cdb # import libconso_cpt as ccpt import db_data pp = pprint.PrettyPrinter(indent=2) ####################################################################### def get_arguments(): parser = ArgumentParser() # parser.add_argument("project", action="store", # help="Project name") # parser.add_argument("center", action="store", # help="Center name (idris/tgcc)") parser.add_argument("-v", "--verbose", action="store_true", help="verbose mode") parser.add_argument("-d", "--dryrun", action="store_true", help="only print what is to be done") # parser.add_argument("-r", "--range", action="store", nargs=2, # help="date range: ssaammjj ssaammjj") return parser.parse_args() ######################################################################## def html_header(project, css_file): html = ( "\n" "\n" "\n" " \n" " \n" " \n" " Suivi de consommation {pj}\n" " \n" "\n" "\n" "\n" "
\n" "\n".format(pj=project.upper(), css=css_file) ) return html ######################################################################## def html_footer(): html = ( "\n" "
\n" "
\n" "\n" "\n" ) return html ######################################################################## def html_main_index(): html = ( "

Conso GENCI

\n" ) return html ######################################################################## def html_main_project(centre, project, node_subp): html = ( "

Conso {pj} ({ce})

\n\n" .format(ce=centre.upper(), pj=project.upper()) ) for (node_type, sub_project) in node_subp: if sub_project: title = "Sous-imputation {sp} / {nd}".format( sp=sub_project if sub_project else project, nd=node_type.replace("/", "") ) else: title = "Conso globale {nd}".format( nd=node_type.replace("/", "") ) html = html + ( "
\n" "

{tl}

\n" " \n" " \"{tl}\"\n" "
\n" "
\n" " \n" " \n" " Fichier pdf\n" " \n" "
\n" "
\n".format( tl=title, ce=centre, pj=project, sp=sub_project if sub_project else project, nd=node_type.replace("/", "") ) ) return html ######################################################################## def html_project(centre, project, node_subp, project_file): if args.verbose: print("Create {}".format(project_file)) header = html_header(project, os.path.join("..", css_file)) footer = html_footer() main = html_main_project(centre, project, node_subp) subdir = os.path.dirname(project_file) if not os.path.isdir(subdir): try: os.mkdir(subdir) except Exception as rc: raise("Could not create directory {}:\n{}".format(subdir, rc)) with open(project_file, "w") as fileout: try: fileout.write(header + main + footer) except Exception as rc: raise("Could not create directory {}:\n{}".format(project_file, rc)) ######################################################################## def config_write(centre, project): config_file = os.path.join( CARD_DIR, "config_{}_{}.ini".format(centre, project) ) if not os.path.isfile(config_file): if args.verbose: print("Create {}".format(config_file)) config = cp.ConfigParser(allow_no_value=True) config.optionxform = str config.add_section("projet") config.set("projet", "name", project) config.add_section("directories") # config.set("directories", "ROOT_DIR", "/home/igcmg/ConsoGENCI") config.set("directories", "ROOT_DIR", ROOT_DIR) config.set("directories", "DATA", "%(ROOT_DIR)s/data") config.set("directories", "PLOT", "%(ROOT_DIR)s/plot") config.set("directories", "DODS") config.add_section("dods") config.set("dods", "DIR", "/home/igcmg/dods/ConsoGENCI") config.set("dods", "USER", "igcmg") config.set("dods", "SERVER", "ciclad") with open(config_file, "w") as fileout: try: config.write(fileout) except Exception as rc: raise("Could not write {}:\n{}".format(config_file, rc)) ####################################################################### if __name__ == "__main__": date_deb = dt.datetime.now() # .. Initialization .. # ==================== # ... Command line arguments ... # ------------------------------ args = get_arguments() if args.verbose: print(args) # ... Files and directories ... # ----------------------------- ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CARD_DIR = os.path.join(ROOT_DIR, "card") DATA_DIR = os.path.join(ROOT_DIR, "data") PLOT_DIR = os.path.join(ROOT_DIR, "plot") HTML_DIR = os.path.join(ROOT_DIR, "web") DODS_DIR = os.path.join( os.path.dirname(ROOT_DIR), "dods", "ConsoGENCI", ) SUBMIT_DIR = os.getcwd() index_file = os.path.join( HTML_DIR, "index_full.html" ) # css_file = os.path.join( # HTML_DIR, # "utils", # "ConsoGENCI.css" # ) css_file = os.path.join( "utils", "ConsoGENCI.css" ) # pattern = ccpt.cpt_pattern(center=args.center, project=args.project) # dirin = os.path.join(DATA_DIR, args.center, "tmp") # dirout = os.path.join(DATA_DIR, args.center, args.project) # filelist = ccpt.find_input_files(dirin, pattern, args.range) # if not filelist: # print("No input files found, exit") # exit(1) # if args.verbose: # print("SUBMIT_DIR:", SUBMIT_DIR) # print("DATA_DIR:", DATA_DIR) # print("dirin:", dirin) # .. Connection to database .. # ============================ if args.verbose: print("Connection to database") conn, cursor = cdb.connect_db( db_data.db_host, db_data.db_name, db_data.db_user, db_data.db_pwd, ) # print(DODS_DIR) # exit() # .. Extract centre/project list from table .. # ============================================ table_name = "conso.tbl_allocation" request = ( "SELECT DISTINCT A.centre, A.project, A.node_type, C.sub_project " "FROM conso.tbl_allocation AS A " "INNER JOIN conso.tbl_consumption AS C " "ON A.id = C.allocation_id " "WHERE A.is_active = TRUE " "ORDER BY A.centre, A.project, A.node_type, C.sub_project ;" ) cdb.select_db(cursor, request) # print(cursor.rowcount) # print(cursor.fetchall()) result = list(cursor) # .. Close connection to database .. # ================================== if args.verbose: print("Close connection") cdb.close_db(conn) # .. Build data dictionary .. # =========================== centre_dict = collections.OrderedDict() # for centre, project, node_type, sub_project in result: # if centre not in centre_dict: # centre_dict[centre] = collections.OrderedDict() # if project not in centre_dict[centre]: # centre_dict[centre][project] = [] # if node_type not in centre_dict[centre][project]: # centre_dict[centre][project].append(node_type) for centre, project, node_type, sub_project in result: if centre not in centre_dict: centre_dict[centre] = collections.OrderedDict() if project not in centre_dict[centre]: centre_dict[centre][project] = [] if (node_type, sub_project) not in centre_dict[centre][project]: centre_dict[centre][project].append((node_type, sub_project)) pp.pprint(centre_dict) # .. Data for html index file .. # ============================== idx_header = html_header("GENCI", css_file) idx_footer = html_footer() idx_main = html_main_index() # .. Loop over centres .. # ======================= for centre, projects in centre_dict.iteritems(): print("\n" + centre + "\n=====") # ... Add centre to index page ... # -------------------------------- idx_main = idx_main + ( "\n" "
\n" " \n" " \n" " \n" " \n" .format(centre.upper()) ) # ... Loop over projects ... # -------------------------- for project, node_subp in projects.iteritems(): print(project + "\n--------") # Build project card if it doesn't already exist config_write(centre, project) # Build html page for project # subdir = os.path.join( # HTML_DIR, # centre, # ) project_file = os.path.join( # subdir, "html", "{}_{}.html".format(centre, project) ) # if not os.path.isfile(project_file): html_project( centre, project, node_subp, os.path.join("web", project_file), ) # Add project to index page idx_main = idx_main + ( " \n" " \n" " \n" .format(project_file, project.upper()) ) # Loop over node types for (node_type, sub_project) in node_subp: print("- {} / {}".format(node_type, sub_project)) # ${ROOT_DIR}/bin/plot_bilan.py -vf --png gencmip6 tgcc thin/standard # --subproject dcpcmip6 command = [ os.path.join(ROOT_DIR, "bin", "plot_bilan.py"), "-vf", "--png", project, centre, node_type, ] if sub_project: command.extend(["--subproject", sub_project]) print(command) if not args.dryrun: try: subprocess.call(command) except Exception as rc: print("Error in plot for {}:\n{}".format(command, rc)) else: print(command) # Copy today's file in web directory filein = "bilan_{ce}_{pj}_{sp}_{nd}_{dt:%Y%m%d}".format( ce=centre, pj=project, sp=sub_project if sub_project else project, nd=node_type.replace("/", ""), dt=dt.datetime.today(), ) fileout = "bilan_{ce}_{pj}_{sp}_{nd}".format( ce=centre, pj=project, sp=sub_project if sub_project else project, nd=node_type.replace("/", ""), ) dirin = os.path.join( PLOT_DIR, centre, project, ) pdfout = os.path.join( HTML_DIR, # centre, "pdf", ) pngout = os.path.join( HTML_DIR, # centre, "img", ) dirnames = [pdfout, pngout] extnames = ["pdf", "png"] for dirname, extname in zip(dirnames, extnames): if not os.path.isdir(dirname): print("mkdir {}".format(dirname)) try: os.makedirs(dirname) except Exception as rc: print("Could not create {}:\n{}".format(dirname, rc)) if not args.dryrun: try: shutil.copy( os.path.join( dirin, "{}.{}".format(filein, extname), ), os.path.join( dirname, "{}.{}".format(fileout, extname), ), ) except Exception as rc: print("Could not copy {}.{}:\n{}".format(filein, extname, rc)) else: print( "{} => {}".format( os.path.join( dirin, "{}.{}".format(filein, extname), ), os.path.join( dirname, "{}.{}".format(fileout, extname), ), ) ) # shutil.copy( # os.path.join(dirin, filein + ".pdf"), # os.path.join(pdfout, fileout + ".pdf") # ) # shutil.copy( # os.path.join(dirin, filein + ".png"), # os.path.join(pngout, fileout + ".png") # ) # ... Close list for current centre ... # ------------------------------------- idx_main = idx_main + ( "
{}
\n" " \n" " {}\n" " \n" "
\n" "
\n" ) # .. Write index file .. # ====================== with open(index_file, "w") as fileout: fileout.write(idx_header + idx_main + idx_footer) date_fin = dt.datetime.now() print(date_fin - date_deb) # .. Copy everything to dods .. # ============================= print("Copy everything to dods") if not args.dryrun: try: res = copy_tree( HTML_DIR, DODS_DIR, preserve_mode=1, preserve_times=1, update=1 ) except Exception as rc: print("Error while copying to dods:\n{}".format(rc)) else: print("{} => {}".format(HTML_DIR, DODS_DIR)) if args.verbose: print(res) exit() for key in project_dict: project_dict[key] = [p for c, p in result if c == key] # .. Loop over centres .. # ======================= for centre, project_list in project_dict.iteritems(): print("\n" + centre) # ... Add centre to index page ... # -------------------------------- idx_main = idx_main + ( '\n' '

{}

\n' ' \n' # .. Write index file .. # ====================== with open(index_file, "w") as fileout: fileout.write(idx_header + idx_main + idx_footer) # .. Launch plot jobs .. # ====================== # allocs = [] # for row in cursor: # allocs.append(ccpt.AllocRow(row)) # # .. Process files .. # # =================== # for filename in filelist: # fileout = os.path.join(dirout, os.path.basename(filename)) # if os.path.isfile(fileout): # print( # "file {} already processed, remove file".format( # os.path.basename(filename) # ) # ) # if not args.dryrun: # try: # os.remove(filename) # except Exception as rc: # print("Could not move {}:\n{}".format(filename, rc)) # break # # .. Build dictionary from files .. # # ================================= # conso_per_alloc = {} # date, blocs = ccpt.parse_input_cpt( # filename, args.project, args.center, mode_conso=True # ) # for bloc in blocs: # if not bloc.alloc_id: # bloc.alloc_id = ccpt.get_project_id(bloc, allocs) # if not bloc.alloc_id: # print("no alloc id found, skip bloc") # print(bloc.machine, bloc.node, bloc.cpt_date) # pp.pprint(allocs) # continue # if bloc.alloc_id not in conso_per_alloc: # conso_per_alloc[bloc.alloc_id] = set() # for conso in bloc.consos: # conso_per_alloc[bloc.alloc_id].add(conso) # # .. Insert data in table, one alloc_id at a time .. # # ================================================== # for alloc_id, consos in conso_per_alloc.iteritems(): # # ... Create request sub string ... # # --------------------------------- # lines_req = [ # ( # "('{alloc}', " # "'{date}', " # "{total_hrs}, " # "{login}, " # "{create})" # ) .format( # alloc=alloc_id, # date=item.date, # total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'", # login="'"+item.login+"'" if item.login != "total" else "NULL", # create="CURRENT_TIMESTAMP", # ) for item in consos # ] # # ... Create full request ... # # --------------------------- # table_name = "conso.tbl_consumption" # request = ( # "INSERT INTO " + table_name + " (" # " allocation_id, " # " date, " # " total_hrs, " # " login, " # " row_create_date " # ") " # "VALUES " # ) # request = request + ", ".join(lines_req) # # ... Execute request ... # # ----------------------- # if args.verbose: # print("Execute request for alloc_id = {}".format(alloc_id)) # print(request) # cdb.insert_db(cursor, request) # # ... Commit inserts ... # # ---------------------- # if not args.dryrun: # if args.verbose: # print("Commit inserts") # cdb.commit_db(conn) # # .. Move processed file to project directory .. # # =============================================== # if args.verbose: # print("Move processed file:") # print("{} => {}".format(filename, dirout)) # if not args.dryrun: # try: # shutil.move(filename, dirout) # except Exception as rc: # print("Could not move {}:\n{}".format(filename, rc)) exit(0)