[2751] | 1 | #!/usr/bin/env python |
---|
| 2 | # -*- coding: utf-8 -*- |
---|
| 3 | |
---|
[2775] | 4 | # ==================================================================== # |
---|
| 5 | # Author: Sonia Labetoulle # |
---|
| 6 | # Contact: sonia.labetoulle _at_ ipsl.jussieu.fr # |
---|
| 7 | # Created: 2016 # |
---|
| 8 | # History: # |
---|
| 9 | # Modification: # |
---|
| 10 | # ==================================================================== # |
---|
| 11 | |
---|
[2751] | 12 | # =================================================================== # |
---|
| 13 | # ssh readonly@prodiguer-test-db.ipsl.upmc.fr # |
---|
| 14 | # psql -U prodiguer_db_user prodiguer # |
---|
| 15 | # # |
---|
| 16 | # ssh readonly@prodiguer-test-db.ipsl.upmc.fr -L 5432:localhost:5432 # |
---|
| 17 | # =================================================================== # |
---|
| 18 | |
---|
| 19 | |
---|
| 20 | # This must come first |
---|
| 21 | from __future__ import print_function, unicode_literals, division |
---|
| 22 | |
---|
| 23 | # Standard library imports |
---|
| 24 | import os |
---|
| 25 | import glob |
---|
| 26 | # import psycopg2 |
---|
| 27 | # import psycopg2.extras |
---|
| 28 | import pprint |
---|
| 29 | |
---|
| 30 | # Application library imports |
---|
| 31 | |
---|
| 32 | pp = pprint.PrettyPrinter(indent=2) |
---|
| 33 | |
---|
| 34 | |
---|
| 35 | ####################################################################### |
---|
| 36 | class ProjectBloc(object): |
---|
| 37 | |
---|
| 38 | #-------------------------------------------------------------------- |
---|
| 39 | def __init__( |
---|
| 40 | self, |
---|
| 41 | name, |
---|
| 42 | center, |
---|
| 43 | machine=None, |
---|
| 44 | node=None, |
---|
| 45 | date_beg=None, |
---|
| 46 | date_end=None, |
---|
| 47 | alloc=None, |
---|
| 48 | line_beg=None, |
---|
| 49 | line_end=None, |
---|
| 50 | ): |
---|
| 51 | self.name = name |
---|
| 52 | self.center = center |
---|
| 53 | self.machine = machine |
---|
| 54 | self.node = node |
---|
| 55 | self.date_beg = date_beg |
---|
| 56 | self.date_end = date_end |
---|
| 57 | self.alloc = alloc |
---|
| 58 | self.line_beg = line_beg |
---|
| 59 | self.line_end = line_end |
---|
| 60 | self.alloc_id = None |
---|
| 61 | |
---|
| 62 | #-------------------------------------------------------------------- |
---|
| 63 | def __repr__(self): |
---|
| 64 | return "{}/{} = {}".format( |
---|
| 65 | self.name, |
---|
| 66 | self.center, |
---|
| 67 | self.alloc, |
---|
| 68 | ) |
---|
| 69 | |
---|
| 70 | |
---|
| 71 | ####################################################################### |
---|
| 72 | class ConsoItem(object): |
---|
| 73 | |
---|
| 74 | #-------------------------------------------------------------------- |
---|
| 75 | def __init__(self, login, date, conso): |
---|
| 76 | self.login = login |
---|
| 77 | self.date = date |
---|
| 78 | self.conso = conso |
---|
| 79 | |
---|
| 80 | #-------------------------------------------------------------------- |
---|
| 81 | def __repr__(self): |
---|
| 82 | return "{}/{:%Y%m%d} = {}".format( |
---|
| 83 | self.login, |
---|
| 84 | self.date, |
---|
| 85 | self.conso, |
---|
| 86 | ) |
---|
| 87 | |
---|
| 88 | |
---|
| 89 | ####################################################################### |
---|
| 90 | class AllocRow(object): |
---|
| 91 | |
---|
| 92 | #-------------------------------------------------------------------- |
---|
| 93 | def __init__(self, row): |
---|
| 94 | self.name = row["project"] |
---|
| 95 | self.alloc_id = row["id"] |
---|
| 96 | self.machine = row["machine"] |
---|
| 97 | self.centre = row["centre"] |
---|
| 98 | self.node = row["node_type"] |
---|
| 99 | self.date_beg = row["start_date"] |
---|
| 100 | self.date_end = row["end_date"] |
---|
| 101 | self.alloc = row["total_hrs"] |
---|
| 102 | |
---|
| 103 | #-------------------------------------------------------------------- |
---|
| 104 | def __repr__(self): |
---|
| 105 | return "{}/{}/{:%Y%m%d}/{:%Y%m%d} ({})".format( |
---|
| 106 | self.machine, |
---|
| 107 | self.node, |
---|
| 108 | self.date_beg, |
---|
| 109 | self.date_end, |
---|
| 110 | self.alloc_id, |
---|
| 111 | ) |
---|
| 112 | |
---|
| 113 | |
---|
| 114 | ####################################################################### |
---|
| 115 | def get_project_id(bloc, allocs): |
---|
| 116 | |
---|
| 117 | res = None |
---|
| 118 | |
---|
| 119 | for alloc in allocs: |
---|
| 120 | # print( |
---|
| 121 | # bloc.cpt_date, |
---|
| 122 | # alloc.date_end, |
---|
| 123 | # bloc.cpt_date <= alloc.date_end |
---|
| 124 | # ) |
---|
| 125 | if bloc.machine == alloc.machine and \ |
---|
| 126 | bloc.node == alloc.node and \ |
---|
| 127 | bloc.cpt_date >= alloc.date_beg and \ |
---|
| 128 | bloc.cpt_date <= alloc.date_end: |
---|
| 129 | res = alloc.alloc_id |
---|
| 130 | break |
---|
| 131 | |
---|
| 132 | return res |
---|
| 133 | |
---|
| 134 | |
---|
| 135 | ####################################################################### |
---|
| 136 | def cpt_pattern(project, center): |
---|
| 137 | |
---|
[2775] | 138 | return "cpt_{}_{}_*.dat".format(center, project) |
---|
[2751] | 139 | |
---|
| 140 | |
---|
| 141 | ####################################################################### |
---|
| 142 | def parse_input_cpt(filename, project, center, mode_conso=False): |
---|
| 143 | |
---|
| 144 | if center == "idris": |
---|
| 145 | return parse_idris_cpt(filename, project) |
---|
| 146 | elif center == "tgcc": |
---|
| 147 | return parse_tgcc_cpt(filename, project, mode_conso) |
---|
| 148 | else: |
---|
| 149 | print("Unknown center {}".format(center)) |
---|
| 150 | exit() |
---|
| 151 | |
---|
| 152 | |
---|
| 153 | ####################################################################### |
---|
| 154 | def parse_tgcc_cpt(filename, project, mode_conso=False): |
---|
| 155 | |
---|
| 156 | import datetime as dt |
---|
| 157 | |
---|
| 158 | # Determine first and last lines of project bloc(s), |
---|
| 159 | # and basic project info |
---|
| 160 | blocs = [] |
---|
| 161 | with open(filename, "r") as filein: |
---|
| 162 | for num, ligne in enumerate(filein): |
---|
| 163 | if "Accounting" in ligne: |
---|
| 164 | _, _, _, name, _, machine, node, _, day = ligne.split() |
---|
| 165 | bloc = ProjectBloc( |
---|
| 166 | name.lower(), |
---|
| 167 | "tgcc", |
---|
| 168 | machine.lower(), |
---|
| 169 | node.lower(), |
---|
| 170 | ) |
---|
| 171 | bloc.cpt_date = dt.datetime.strptime( |
---|
| 172 | "{} {}".format(day, "23:59"), "%Y-%m-%d %H:%M" |
---|
| 173 | ) |
---|
| 174 | bloc.line_beg = num + 1 |
---|
| 175 | if "Allocated" in ligne: |
---|
| 176 | bloc.alloc = float(ligne.split()[-1]) |
---|
| 177 | if "deadline" in ligne: |
---|
| 178 | bloc.date_end = dt.datetime.strptime( |
---|
| 179 | "{}".format(ligne.split()[-1]), "%Y-%m-%d" |
---|
| 180 | ) |
---|
| 181 | bloc.date_end = dt.date( |
---|
| 182 | bloc.date_end.year, |
---|
| 183 | bloc.date_end.month, |
---|
| 184 | bloc.date_end.day, |
---|
| 185 | ) |
---|
| 186 | bloc.line_end = num + 1 |
---|
| 187 | if bloc.name == project: |
---|
| 188 | blocs.append(bloc) |
---|
| 189 | |
---|
| 190 | if mode_conso: |
---|
| 191 | for bloc in blocs: |
---|
| 192 | bloc.consos = set() |
---|
| 193 | with open(filename, "r") as filein: |
---|
| 194 | for _ in xrange(bloc.line_beg - 1): |
---|
| 195 | next(filein) |
---|
| 196 | for num, ligne in enumerate(filein): |
---|
| 197 | if num > bloc.line_end - bloc.line_beg: |
---|
| 198 | break |
---|
| 199 | if len(ligne.split()) == 2 and \ |
---|
| 200 | ligne.split()[0] != "Allocated": |
---|
| 201 | login, conso = ligne.lower().split() |
---|
| 202 | conso = float(conso) |
---|
| 203 | # bloc["consos"].add((login.lower(), conso)) |
---|
| 204 | bloc.consos.add( |
---|
| 205 | ConsoItem( |
---|
| 206 | login.lower(), |
---|
| 207 | bloc.cpt_date, |
---|
| 208 | conso, |
---|
| 209 | ) |
---|
| 210 | ) |
---|
| 211 | |
---|
| 212 | return blocs[0].cpt_date, blocs |
---|
| 213 | |
---|
| 214 | |
---|
| 215 | ####################################################################### |
---|
| 216 | def parse_idris_cpt(filename, project): |
---|
| 217 | |
---|
| 218 | import datetime as dt |
---|
| 219 | |
---|
| 220 | print(filename) |
---|
| 221 | |
---|
| 222 | blocs = [] |
---|
| 223 | |
---|
| 224 | with open(filename, "r") as filein: |
---|
| 225 | # bloc = {} |
---|
| 226 | bloc = ProjectBloc( |
---|
| 227 | project, |
---|
| 228 | "idris", |
---|
| 229 | ) |
---|
| 230 | bloc.consos = set() |
---|
| 231 | for num, ligne in enumerate(filein): |
---|
| 232 | if "mise a jour" in ligne: |
---|
| 233 | jour, heure = ligne.strip().split()[-2:] |
---|
| 234 | bloc.cpt_date = dt.datetime.strptime( |
---|
| 235 | "{} {}".format(jour, heure), "%d/%m/%Y %H:%M" |
---|
| 236 | ) |
---|
| 237 | bloc.date_end = dt.date(bloc.cpt_date.year, 12, 31) |
---|
| 238 | # bloc["date_end"] = dt.datetime.strptime( |
---|
| 239 | # "{} {}".format(jour, heure), "%d/%m/%Y %H:%M" |
---|
| 240 | # ) |
---|
| 241 | elif "Comptabilite" in ligne: |
---|
| 242 | bloc.machine = ligne.strip().split()[-1].strip(".") |
---|
| 243 | bloc.node = "standard" |
---|
| 244 | elif "Heures" in ligne: |
---|
| 245 | bloc.alloc = int(ligne.strip().split()[-1]) |
---|
| 246 | elif "Totaux" in ligne: |
---|
| 247 | conso = float(ligne.strip().split()[1]) |
---|
| 248 | # bloc.consos.add(("total", conso)) |
---|
| 249 | bloc.consos.add( |
---|
| 250 | ConsoItem( |
---|
| 251 | "total", |
---|
| 252 | bloc.cpt_date, |
---|
| 253 | conso, |
---|
| 254 | ) |
---|
| 255 | ) |
---|
| 256 | elif project in ligne: |
---|
| 257 | login = ligne.strip().split()[0] |
---|
| 258 | try: |
---|
| 259 | conso = float(ligne.strip().split()[2]) |
---|
| 260 | except ValueError: |
---|
| 261 | conso = float("nan") |
---|
| 262 | # bloc.consos.add((login, conso)) |
---|
| 263 | bloc.consos.add( |
---|
| 264 | ConsoItem( |
---|
| 265 | login.lower(), |
---|
| 266 | bloc.cpt_date, |
---|
| 267 | conso, |
---|
| 268 | ) |
---|
| 269 | ) |
---|
| 270 | |
---|
| 271 | blocs.append(bloc) |
---|
| 272 | |
---|
| 273 | return blocs[0].cpt_date, blocs |
---|
| 274 | |
---|
| 275 | |
---|
| 276 | ####################################################################### |
---|
| 277 | def date_from_filename(filename): |
---|
| 278 | """ |
---|
| 279 | """ |
---|
| 280 | return os.path.basename(filename).split("_")[-2] |
---|
| 281 | |
---|
| 282 | |
---|
| 283 | ####################################################################### |
---|
| 284 | def find_input_files(data_dir, pattern, date_range): |
---|
| 285 | """ |
---|
| 286 | """ |
---|
| 287 | file_list = glob.glob(os.path.join(data_dir, pattern)) |
---|
| 288 | |
---|
| 289 | if date_range: |
---|
| 290 | res = [] |
---|
| 291 | for filename in file_list: |
---|
| 292 | file_date = date_from_filename(filename) |
---|
| 293 | if file_date >= min(date_range) and \ |
---|
| 294 | file_date <= max(date_range): |
---|
| 295 | res.append(filename) |
---|
| 296 | else: |
---|
| 297 | res = file_list |
---|
| 298 | |
---|
[2775] | 299 | # return set(res) |
---|
| 300 | return sorted(res) |
---|
[2751] | 301 | |
---|
| 302 | |
---|
| 303 | ####################################################################### |
---|
| 304 | if __name__ == "__main__": |
---|
| 305 | pass |
---|