Index: /TOOLS/ConsoGENCI/trunk/bin/init_alloc_tbl.py
===================================================================
--- /TOOLS/ConsoGENCI/trunk/bin/init_alloc_tbl.py	(revision 2750)
+++ /TOOLS/ConsoGENCI/trunk/bin/init_alloc_tbl.py	(revision 2751)
@@ -19,16 +19,34 @@
 
 
-########################################
+#######################################################################
 if __name__ == "__main__":
 
   project_list = [
     {
-      "name": "gencmip6",
+      "name": "gen0826",
       "centre": "tgcc",
       "machine": "curie",
-      "node": "thin",
-      "alloc": 5000000,
-      "start": "20150101",
-      "end": "20150630",
+      "node": "fat/large",
+      "alloc": 114000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gen0826",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "hybrid",
+      "alloc": 25000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gen0826",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 114000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
     },
     {
@@ -36,8 +54,35 @@
       "centre": "tgcc",
       "machine": "curie",
-      "node": "thin",
+      "node": "thin/standard",
+      "alloc": 5000000,
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-06-30 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
       "alloc": 15000000,
-      "start": "20150701",
-      "end": "20151231",
+      "start": "2015-07-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 25000000,
+      "start": "2016-01-01 00:00:00",
+      "end": "2016-06-30 23:59:59",
+    },
+    {
+      "name": "gencmip6",
+      "centre": "tgcc",
+      "machine": "curie",
+      "node": "thin/standard",
+      "alloc": 50000000,
+      "start": "2016-07-01 00:00:00",
+      "end": "2016-12-31 23:59:59",
     },
     {
@@ -47,6 +92,15 @@
       "node": "standard",
       "alloc": 2000000,
-      "start": "20150101",
-      "end": "20151231",
+      "start": "2015-01-01 00:00:00",
+      "end": "2015-12-31 23:59:59",
+    },
+    {
+      "name": "rgzi",
+      "centre": "idris",
+      "machine": "ada",
+      "node": "standard",
+      "alloc": 4000000,
+      "start": "2016-01-01 00:00:00",
+      "end": "2016-12-31 23:59:59",
     },
   ]
@@ -99,2 +153,4 @@
 
   cdb.commit_db(conn)
+
+  cdb.close_db(conn)
Index: /TOOLS/ConsoGENCI/trunk/bin/init_conso_tbl.py
===================================================================
--- /TOOLS/ConsoGENCI/trunk/bin/init_conso_tbl.py	(revision 2750)
+++ /TOOLS/ConsoGENCI/trunk/bin/init_conso_tbl.py	(revision 2751)
@@ -1,12 +1,4 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
-# ==================================================================== #
-# ssh readonly@prodiguer-test-db.ipsl.upmc.fr                          #
-# psql -U prodiguer_db_user prodiguer                                  #
-#                                                                      #
-# ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432   #
-# ==================================================================== #
-
 
 # This must come first
@@ -18,10 +10,14 @@
 # import datetime as dt
 from argparse import ArgumentParser
+import pprint
 
 # Application library imports
-import libconsodb as cdb
+import libconso_db as cdb
+import libconso_cpt as ccpt
+
+pp = pprint.PrettyPrinter(indent=2)
 
 
-########################################
+#######################################################################
 def get_arguments():
   parser = ArgumentParser()
@@ -53,5 +49,5 @@
 
 
-########################################
+#######################################################################
 if __name__ == "__main__":
 
@@ -64,15 +60,22 @@
     print(args)
 
-  ROOT_DIR = "/home_local/slipsl/ConsoGENCMIP6/init_db"
+  # ... Files and directories ...
+  # -----------------------------
+  ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
   DATA_DIR = os.path.join(ROOT_DIR, "data")
-
   SUBMIT_DIR = os.getcwd()
 
-  # project = "rgzi"
-  # centre = "idris"
+  if args.verbose:
+    print("SUBMIT_DIR:", SUBMIT_DIR)
+    print("DATA_DIR:", DATA_DIR)
 
-  conso_per_login = {}
+  pattern = ccpt.cpt_pattern(args.center, args.project)
+  dirin = os.path.join(DATA_DIR, args.center, args.project)
+  filelist = ccpt.find_input_files(dirin, pattern, args.range)
 
-  # Connection info
+  # .. Connection to database ..
+  # ============================
+  if args.verbose:
+    print("Connection to database")
   db_host = "134.157.170.104"
   # db_port = "5432"
@@ -80,56 +83,55 @@
   db_user = "prodiguer_db_user"
   # db_pwd = "secret"
-
-  if args.center == "idris":
-    pattern = "compta_*.dat"
-  elif args.center == "tgcc":
-    pattern = "ccc_myproject.dat_*"
-  else:
-    print("Unknown center {}".format(args.center))
-    exit()
-
-  filelist = cdb.find_input_files(DATA_DIR, pattern, args.range)
-
-  # Build dictionary from files
-  # for filename in cdb.find_idris_files(DATA_DIR):
-  for filename in filelist:
-    print(filename)
-    # jour, heure, alloc, consos = \
-    date, alloc, consos = \
-          cdb.parse_input_cpt(filename, args.project, args.center)
-          # cdb.parse_idris_cpt(filename, args.project)
-
-    # date = dt.datetime.strptime("{} {}".format(jour, heure), "%d/%m/%Y %H:%M")
-
-    for item in consos:
-      login, conso = item
-      if login not in conso_per_login:
-        conso_per_login[login] = set()
-      conso_per_login[login].add((date, conso, ))
-
   conn, cursor = cdb.connect_db(db_host, db_name, db_user)
 
-  # Extract allocation id from table
+  # .. Extract allocation id from table ..
+  # ======================================
   table_name = "conso.tbl_allocation"
   request = (
-    "SELECT id "
+    # "SELECT id "
+    "SELECT * "
     "FROM " + table_name + " "
     "WHERE project = '" + args.project + "'"
     "  AND centre = '" + args.center + "'"
+    # "  AND start_date < '" + dt.datetime.strftime(date, "%Y-%m-%d") + "'"
+    # "  AND end_date > '" + dt.datetime.strftime(date, "%Y-%m-%d") + "'"
+    "ORDER BY start_date"
     ";"
   )
 
   cdb.select_db(cursor, request)
-  if cursor.rowcount != 1:
-    print(cursor.fetchall())
-    exit()
+  # print(cursor.rowcount)
+  # print(cursor.fetchall())
 
-  (allocation_id, ) = cursor.fetchone()
+  allocs = []
+  for row in cursor:
+    allocs.append(ccpt.AllocRow(row))
 
-  for login, conso_list in conso_per_login.iteritems():
+  # .. Build dictionary from files ..
+  # =================================
+  conso_per_alloc = {}
+  for filename in filelist:
+    date, blocs = ccpt.parse_input_cpt(
+      filename, args.project, args.center, mode_conso=True
+    )
 
-    # test = "'"+login+"'" if login != "total" else "NULL"
-    # print(test)
+    for bloc in blocs:
+      if not bloc.alloc_id:
+        bloc.alloc_id = ccpt.get_project_id(bloc, allocs)
+        if not bloc.alloc_id:
+          print("no alloc id found, skip bloc")
+          print(bloc.machine, bloc.node, bloc.cpt_date)
+          pp.pprint(allocs)
+          continue
+      if bloc.alloc_id not in conso_per_alloc:
+        conso_per_alloc[bloc.alloc_id] = set()
+      for conso in bloc.consos:
+        conso_per_alloc[bloc.alloc_id].add(conso)
 
+  # .. Insert data in table, one alloc_id at a time ..
+  # ==================================================
+  for alloc_id, consos in conso_per_alloc.iteritems():
+    # ... Create request sub string ...
+    # ---------------------------------
     lines_req = [
       (
@@ -140,12 +142,18 @@
         "{create})"
       ) .format(
-        alloc=allocation_id,
-        date=date,
-        total_hrs=conso if not math.isnan(conso) else "'NaN'",
-        login="'"+login+"'" if login != "total" else "NULL",
+        alloc=alloc_id,
+        date=item.date,
+        total_hrs=item.conso if not math.isnan(item.conso) else "'NaN'",
+        login="'"+item.login+"'" if item.login != "total" else "NULL",
         create="CURRENT_TIMESTAMP",
-      ) for (date, conso) in conso_list
+      ) for item in consos
     ]
 
+    # print(lines_req)
+
+    # exit()
+
+    # ... Create full request ...
+    # ---------------------------
     table_name = "conso.tbl_consumption"
     request = (
@@ -160,8 +168,21 @@
     )
     request = request + ", ".join(lines_req)
-    print(request)
 
+    # ... Execute request ...
+    # -----------------------
+    if args.verbose:
+      print("Execute request for alloc_id = {}".format(alloc_id))
     cdb.insert_db(cursor, request)
 
+    # ... Commit inserts ...
+    # ----------------------
     if not args.dryrun:
+      if args.verbose:
+        print("Commit inserts")
       cdb.commit_db(conn)
+
+  # .. Close connection to database ..
+  # ==================================
+  if args.verbose:
+    print("Close connection")
+  cdb.close_db(conn)
Index: /TOOLS/ConsoGENCI/trunk/bin/libconso.py
===================================================================
--- /TOOLS/ConsoGENCI/trunk/bin/libconso.py	(revision 2750)
+++ /TOOLS/ConsoGENCI/trunk/bin/libconso.py	(revision 2751)
@@ -13,5 +13,5 @@
 import subprocess
 import datetime as dt
-import numpy as np
+# import numpy as np
 import ConfigParser as cp
 
Index: /TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py
===================================================================
--- /TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py	(revision 2751)
+++ /TOOLS/ConsoGENCI/trunk/bin/libconso_cpt.py	(revision 2751)
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# =================================================================== #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr                         #
+# psql -U prodiguer_db_user prodiguer                                 #
+#                                                                     #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432  #
+# =================================================================== #
+
+
+# This must come first
+from __future__ import print_function, unicode_literals, division
+
+# Standard library imports
+import os
+import glob
+# import psycopg2
+# import psycopg2.extras
+import pprint
+
+# Application library imports
+
+pp = pprint.PrettyPrinter(indent=2)
+
+
+#######################################################################
+class ProjectBloc(object):
+
+  #--------------------------------------------------------------------
+  def __init__(
+    self,
+    name,
+    center,
+    machine=None,
+    node=None,
+    date_beg=None,
+    date_end=None,
+    alloc=None,
+    line_beg=None,
+    line_end=None,
+  ):
+    self.name     = name
+    self.center   = center
+    self.machine  = machine
+    self.node     = node
+    self.date_beg = date_beg
+    self.date_end = date_end
+    self.alloc    = alloc
+    self.line_beg = line_beg
+    self.line_end = line_end
+    self.alloc_id = None
+
+  #--------------------------------------------------------------------
+  def __repr__(self):
+    return "{}/{} = {}".format(
+      self.name,
+      self.center,
+      self.alloc,
+    )
+
+
+#######################################################################
+class ConsoItem(object):
+
+  #--------------------------------------------------------------------
+  def __init__(self, login, date, conso):
+    self.login = login
+    self.date  = date
+    self.conso = conso
+
+  #--------------------------------------------------------------------
+  def __repr__(self):
+    return "{}/{:%Y%m%d} = {}".format(
+      self.login,
+      self.date,
+      self.conso,
+    )
+
+
+#######################################################################
+class AllocRow(object):
+
+  #--------------------------------------------------------------------
+  def __init__(self, row):
+    self.name     = row["project"]
+    self.alloc_id = row["id"]
+    self.machine  = row["machine"]
+    self.centre   = row["centre"]
+    self.node     = row["node_type"]
+    self.date_beg = row["start_date"]
+    self.date_end = row["end_date"]
+    self.alloc    = row["total_hrs"]
+
+  #--------------------------------------------------------------------
+  def __repr__(self):
+    return "{}/{}/{:%Y%m%d}/{:%Y%m%d} ({})".format(
+      self.machine,
+      self.node,
+      self.date_beg,
+      self.date_end,
+      self.alloc_id,
+    )
+
+
+#######################################################################
+def get_project_id(bloc, allocs):
+
+  res = None
+
+  for alloc in allocs:
+    # print(
+    #   bloc.cpt_date,
+    #   alloc.date_end,
+    #   bloc.cpt_date <= alloc.date_end
+    # )
+    if bloc.machine == alloc.machine   and \
+       bloc.node == alloc.node         and \
+       bloc.cpt_date >= alloc.date_beg and \
+       bloc.cpt_date <= alloc.date_end:
+      res = alloc.alloc_id
+      break
+
+  return res
+
+
+#######################################################################
+def cpt_pattern(project, center):
+
+  return "cpt_{}_{}_*.dat".format(project, center)
+
+
+#######################################################################
+def parse_input_cpt(filename, project, center, mode_conso=False):
+
+  if center == "idris":
+    return parse_idris_cpt(filename, project)
+  elif center == "tgcc":
+    return parse_tgcc_cpt(filename, project, mode_conso)
+  else:
+    print("Unknown center {}".format(center))
+    exit()
+
+
+#######################################################################
+def parse_tgcc_cpt(filename, project, mode_conso=False):
+
+  import datetime as dt
+
+  # Determine first and last lines of project bloc(s),
+  # and basic project info
+  blocs = []
+  with open(filename, "r") as filein:
+    for num, ligne in enumerate(filein):
+      if "Accounting" in ligne:
+        _, _, _, name, _, machine, node, _, day = ligne.split()
+        bloc = ProjectBloc(
+          name.lower(),
+          "tgcc",
+          machine.lower(),
+          node.lower(),
+        )
+        bloc.cpt_date = dt.datetime.strptime(
+          "{} {}".format(day, "23:59"), "%Y-%m-%d %H:%M"
+        )
+        bloc.line_beg = num + 1
+      if "Allocated" in ligne:
+        bloc.alloc = float(ligne.split()[-1])
+      if "deadline" in ligne:
+        bloc.date_end = dt.datetime.strptime(
+          "{}".format(ligne.split()[-1]), "%Y-%m-%d"
+        )
+        bloc.date_end = dt.date(
+          bloc.date_end.year,
+          bloc.date_end.month,
+          bloc.date_end.day,
+        )
+        bloc.line_end = num + 1
+        if bloc.name == project:
+          blocs.append(bloc)
+
+  if mode_conso:
+    for bloc in blocs:
+      bloc.consos = set()
+      with open(filename, "r") as filein:
+        for _ in xrange(bloc.line_beg - 1):
+          next(filein)
+        for num, ligne in enumerate(filein):
+          if num > bloc.line_end - bloc.line_beg:
+            break
+          if len(ligne.split()) == 2 and \
+             ligne.split()[0] != "Allocated":
+            login, conso = ligne.lower().split()
+            conso = float(conso)
+            # bloc["consos"].add((login.lower(), conso))
+            bloc.consos.add(
+              ConsoItem(
+                login.lower(),
+                bloc.cpt_date,
+                conso,
+              )
+            )
+
+  return blocs[0].cpt_date, blocs
+
+
+#######################################################################
+def parse_idris_cpt(filename, project):
+
+  import datetime as dt
+
+  print(filename)
+
+  blocs = []
+
+  with open(filename, "r") as filein:
+    # bloc = {}
+    bloc = ProjectBloc(
+      project,
+      "idris",
+    )
+    bloc.consos = set()
+    for num, ligne in enumerate(filein):
+      if "mise a jour" in ligne:
+        jour, heure = ligne.strip().split()[-2:]
+        bloc.cpt_date = dt.datetime.strptime(
+          "{} {}".format(jour, heure), "%d/%m/%Y %H:%M"
+        )
+        bloc.date_end = dt.date(bloc.cpt_date.year, 12, 31)
+        # bloc["date_end"] = dt.datetime.strptime(
+        #   "{} {}".format(jour, heure), "%d/%m/%Y %H:%M"
+        # )
+      elif "Comptabilite" in ligne:
+        bloc.machine = ligne.strip().split()[-1].strip(".")
+        bloc.node = "standard"
+      elif "Heures" in ligne:
+        bloc.alloc = int(ligne.strip().split()[-1])
+      elif "Totaux" in ligne:
+        conso = float(ligne.strip().split()[1])
+        # bloc.consos.add(("total", conso))
+        bloc.consos.add(
+          ConsoItem(
+            "total",
+            bloc.cpt_date,
+            conso,
+          )
+        )
+      elif project in ligne:
+        login = ligne.strip().split()[0]
+        try:
+          conso = float(ligne.strip().split()[2])
+        except ValueError:
+          conso = float("nan")
+        # bloc.consos.add((login, conso))
+        bloc.consos.add(
+          ConsoItem(
+            login.lower(),
+            bloc.cpt_date,
+            conso,
+          )
+        )
+
+    blocs.append(bloc)
+
+  return blocs[0].cpt_date, blocs
+
+
+#######################################################################
+def date_from_filename(filename):
+  """
+  """
+  return os.path.basename(filename).split("_")[-2]
+
+
+#######################################################################
+def find_input_files(data_dir, pattern, date_range):
+  """
+  """
+  file_list = glob.glob(os.path.join(data_dir, pattern))
+
+  if date_range:
+    res = []
+    for filename in file_list:
+      file_date = date_from_filename(filename)
+      if file_date >= min(date_range) and \
+         file_date <= max(date_range):
+        res.append(filename)
+  else:
+    res = file_list
+
+  return set(res)
+
+
+#######################################################################
+if __name__ == "__main__":
+  pass
Index: /TOOLS/ConsoGENCI/trunk/bin/libconso_db.py
===================================================================
--- /TOOLS/ConsoGENCI/trunk/bin/libconso_db.py	(revision 2751)
+++ /TOOLS/ConsoGENCI/trunk/bin/libconso_db.py	(revision 2751)
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# =================================================================== #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr                         #
+# psql -U prodiguer_db_user prodiguer                                 #
+#                                                                     #
+# ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432  #
+# =================================================================== #
+
+
+# This must come first
+from __future__ import print_function, unicode_literals, division
+
+# Standard library imports
+import psycopg2
+import psycopg2.extras
+
+# Application library imports
+
+
+#######################################################################
+def connect_db(db_host, db_name, db_user):
+
+  conn_string = "host={} dbname={} user={}".format(
+    db_host, db_name, db_user
+  )
+
+  # # Print the connection string we will use to connect
+  # print("Connecting to database\n  -> {}".format(conn_string))
+
+  # Get a connection, if a connect cannot be made an exception will be
+  # raised here
+  try :
+    conn = psycopg2.connect(conn_string)
+  except Exception as rc:
+    print("Problem with DB connect:\n{}".format(rc))
+    exit(1)
+
+  # conn.cursor will return a cursor object, you can use this cursor
+  # to perform queries
+  # cursor = conn.cursor()
+  cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+  print("  => Connected!\n")
+
+  return conn, cursor
+
+
+#######################################################################
+def select_db(cursor, request):
+
+  # Execute query
+  try:
+    cursor.execute(request)
+  except Exception as rc:
+    print("Problem with select:\n{}".format(rc))
+    exit(1)
+
+  # # retrieve the records from the database
+  # records = cursor.fetchall()
+
+  # # print out the records using pretty print
+  # # note that the NAMES of the columns are not shown, instead just indexes.
+  # # for most people this isn't very useful so we'll show you how to return
+  # # columns as a dictionary (hash) in the next example.
+  # pprint(records)
+
+
+#######################################################################
+def insert_db(cursor, request):
+  """
+  """
+
+  # Execute query
+  try:
+    cursor.execute(request)
+  except Exception as rc:
+    print("Problem with insert:\n{}".format(rc))
+    exit(1)
+
+
+#######################################################################
+def commit_db(conn):
+  """
+  """
+
+  # Execute query
+  try:
+    conn.commit()
+  except Exception as rc:
+    print("Problem with commit:\n{}".format(rc))
+    exit(1)
+
+
+#######################################################################
+def close_db(conn):
+  """
+  """
+
+  try:
+    conn.close()
+  except Exception as rc:
+    print("Problem with close:\n{}".format(rc))
+    exit(1)
+
+
+#######################################################################
+if __name__ == "__main__":
+  pass
