#!/usr/bin/env python3 ''' Script to check water conservation in the IPSL coupled model Here, some common utilities to all scripts Warning, to install, configure, run, use any of included software or to read the associated documentation you'll need at least one (1) brain in a reasonably working order. Lack of this implement will void any warranties (either express or implied). Authors assumes no responsability for errors, omissions, data loss, or any other consequences caused directly or indirectly by the usage of his software by incorrectly or partially configured personal SVN information $Author$ $Date$ $Revision$ $Id: $ $HeadURL$ ''' import os, sys import functools import time import configparser, re import numpy as np import libIGCM_sys def ReadConfig ( ini_file, default_ini_file='defaults.ini' ) : ''' Reads experiment parameters Reads config file to set all defaults parameters Reads config file to set all experiment parameters Reads config.card files if specified in ''' ## Read file with defaults values ## ------------------------------ params = configparser.ConfigParser ( interpolation=configparser.ExtendedInterpolation() ) params.optionxform = str # To keep capitals params.read (default_ini_file) print ( f'\nConfig file readed : {default_ini_file} ' ) ## Read Experiment config file ## ---------------------------- exp_params = configparser.ConfigParser ( interpolation=configparser.ExtendedInterpolation() ) exp_params.optionxform = str # To keep capitals exp_params.read (ini_file) print ( f'\nConfig file readed : {ini_file} ' ) ## Reading config.card if possible ## ------------------------------- if 'Experiment' in params.keys () : ## Read Experiment on parameter file if possible if 'ConfigCard' in exp_params['Experiment'].keys () : ConfigCard = exp_params['Experiment']['ConfigCard'] print ( f'{ConfigCard=}' ) if ConfigCard : ## Read config card if it exists # Text existence of ConfigCard if os.path.exists ( ConfigCard ) : print ( f'Reading Config Card : {ConfigCard}' ) ## Creates parser for reading .ini input file config = configparser.ConfigParser (interpolation=configparser.ExtendedInterpolation() ) config.optionxform = str # To keep capitals config.read (ConfigCard) for VarName in ['JobName', 'ExperimentName', 'SpaceName', 'LongName', 'ModelName', 'TagName', 'ORCA_version', 'ExpType', 'PeriodLength', 'ResolAtm', 'ResolOce' ] : if VarName in config['UserChoices'].keys() : exp_params['Experiment'][VarName] = str2value ( config['UserChoices'][VarName] ) if 'Post' in config.sections() : if 'PackFrequency' in config['Post'] : PackFrequency = config['Post']['PackFrequency'] exp_params['Experiment']['PackFrequency'] = PackFrequency else : raise FileExistsError ( f"File not found : {ConfigCard = }" ) ## Reading config file ## ------------------- for Section in exp_params.keys () : params[Section].update ( exp_params[Section] ) print ( f'\nConfig file readed : {ini_file} ' ) return config2dict (params) def SetDatesAndFiles ( pdict ) : ''' From readed experiment parameters, set all needed parameters ''' ## Set libIGCM and machine dependant values ## ---------------------------------------- pdict['Experiment']['ICO'] = 'ICO' in pdict['Experiment']['ResolAtm'] pdict['Experiment']['LMDZ'] = 'LMD' in pdict['Experiment']['ResolAtm'] mm = libIGCM_sys.config ( TagName = pdict['Experiment']['TagName'], SpaceName = pdict['Experiment']['SpaceName'], ExperimentName= pdict['Experiment']['ExperimentName'], JobName = pdict['Experiment']['JobName'], User = pdict['Experiment']['User'], Group = pdict['Experiment']['Group'], ARCHIVE = pdict['libIGCM']['ARCHIVE'], SCRATCHDIR = pdict['libIGCM']['SCRATCHDIR'], STORAGE = pdict['libIGCM']['STORAGE'], R_IN = pdict['libIGCM']['R_IN'], R_OUT = pdict['libIGCM']['R_OUT'], R_FIG = pdict['libIGCM']['R_FIG'], TmpDir = pdict['Files']['TmpDir'], R_SAVE = pdict['libIGCM']['R_SAVE'], R_FIGR = pdict['libIGCM']['R_FIGR'], R_BUFR = pdict['libIGCM']['R_BUFR'], R_BUF_KSH = pdict['libIGCM']['R_BUF_KSH'], POST_DIR = pdict['libIGCM']['POST_DIR'], L_EXP = pdict['libIGCM']['L_EXP'] ) pdict['Files']['TmpDir'] = mm['TmpDir'] for key in mm.keys() : pdict['libIGCM'][key] = mm[key] ## Complete configuration ## ---------------------- if not pdict['Experiment']['NEMO'] and pdict['Experiment']['ORCA_version'] : if '1.2' in pdict['Experiment']['ORCA_version'] : pdict['Experiment']['NEMO'] = 3.6 if '4.0' in pdict['Experiment']['ORCA_version'] : pdict['Experiment']['NEMO'] = 4.0 if '4.2' in pdict['Experiment']['ORCA_version'] : pdict['Experiment']['NEMO'] = 4.2 if pdict['Experiment']['ATM_HIS'] : if not pdict['Experiment']['SRF_HIS'] : pdict['Experiment']['SRF_HIS'] = pdict['Experiment']['ATM_HIS'] if not pdict['Experiment']['RUN_HIS'] : pdict['Experiment']['RUN_HIS'] = pdict['Experiment']['ATM_HIS'] ## ## Reading prec if not pdict['Config']['readPrec'] : pdict['Config']['readPrec'] = np.float64 else : if pdict['Config']['readPrec'] in ["float", "float64", "r8", "double", "" ] : pdict['Config']['readPrec'] = float if pdict['Config']['readPrec'] in [ "float32", "r4", "single", "" ] : pdict['Config']['readPrec'] = np.float32 if pdict['Config']['readPrec'] in [ "float16", "r2", "half" , "" ] : pdict['Config']['readPrec'] = np.float16 ## Defines begining and end of experiment ## -------------------------------------- if not pdict['Experiment']['DateBegin'] : pdict['Experiment']['DateBegin'] = f"{pdict['Experiment']['YearBegin']}0101" else : YearBegin, MonthBegin, DayBegin = SplitDate ( pdict['Experiment']['DateBegin'] ) DateBegin = FormatToGregorian (pdict['Experiment']['DateBegin']) pdict['Experiment']['YearBegin'] = YearBegin if not pdict['Experiment']['DateEnd'] : pdict['Experiment']['DateEnd'] = f"{pdict['Experiment']['YearEnd']}1231" else : YearEnd, MonthEnd, DayEnd = SplitDate ( pdict['Experiment']['DateEnd'] ) DateEnd = FormatToGregorian (pdict['Experiment']['DateEnd']) pdict['Experiment']['DateEnd'] = DateEnd if not pdict['Experiment']['PackFrequency'] : PackFrequencypdict['Experiment']['PackFrequency'] = YearEnd - YearBegin + 1 if isinstance ( pdict['Experiment']['PackFrequency'], str ) : if 'Y' in pdict['Experiment']['PackFrequency'] : zPackFrequency = pdict['Experiment']['PackFrequency'].replace ( 'Y', '') pdict['Experiment']['PackFrequency'] = f'{zPackFrequency}Y' pdict['Experiment']['zPackFrequency'] = int (zPackFrequency) if 'M' in pdict['Experiment']['PackFrequency'] : zPackFrequency = pdict['Experiment']['PackFrequency'].replace ( 'M', '') pdict['Experiment']['PackFrequency'] = f'{zPackFrequency}M' pdict['Experiment']['zPackFrequency'] = int (zPackFrequency) ## Set directory to extract files ## ------------------------------ if not pdict['Files']['FileDir'] : pdict['Files']['FileDir'] = os.path.join ( pdict['Files']['TmpDir'], f"WATER_{pdict['Experiment']['JobName']}" ) if not os.path.isdir ( pdict['Files']['FileDir'] ) : os.makedirs ( pdict['Files']['FileDir'] ) FileOut = str2value (pdict['Files']['FileOut']) if not FileOut : FileOut = f"ATM_waterbudget_{pdict['Experiment']['JobName']}_{pdict['Experiment']['YearBegin']}_{pdict['Experiment']['YearEnd']}" if pdict['Experiment']['ICO'] : if pdict['Experiment']['ATM_HIS'] == 'latlon' : FileOut = f'{FileOut}_LATLON' if pdict['Experiment']['ATM_HIS'] == 'ico' : FileOut = f'{FileOut}_ICO' if pdict['Config']['readPrec'] == np.float32 : FileOut = f'{FileOut}_float32' FileOut = f'{FileOut}.out' pdict['Files']['FileOut'] = FileOut f_out = open ( FileOut, mode = 'w', encoding="utf-8" ) pdict['Files']['f_out'] = f_out echo (' ', f_out) echo ( f"JobName : {pdict['Experiment']['JobName']}" , f_out=f_out ) echo ( f"Comment : {pdict['Experiment']['Comment']}" , f_out=f_out ) echo ( f"TmpDir : {pdict['Files']['TmpDir']}" , f_out=f_out ) echo ( f"FileDir : {pdict['Files']['FileDir']}" , f_out=f_out ) echo ( f"\nDealing with {pdict['libIGCM']['L_EXP']}", f_out ) ## Creates model output directory names ## ------------------------------------ if not pdict['Files']['FreqDir'] : if pdict['Experiment']['Freq'] == "MO" : pdict['Files']['FreqDir'] = os.path.join ('Output' , 'MO' ) if pdict['Experiment']['Freq'] == "SE" : pdict['Files']['FreqDir'] = os.path.join ('Analyse', 'SE' ) if not pdict['Files']['dir_ATM_his'] : pdict['Files']['dir_ATM_his'] = os.path.join ( pdict['libIGCM']['R_SAVE'], "ATM", pdict['Files']['FreqDir'] ) if not pdict['Files']['dir_SRF_his'] : pdict['Files']['dir_SRF_his'] = os.path.join ( pdict['libIGCM']['R_SAVE'], "SRF", pdict['Files']['FreqDir'] ) if not pdict['Files']['dir_OCE_his'] : pdict['Files']['dir_OCE_his'] = os.path.join ( pdict['libIGCM']['R_SAVE'], "OCE", pdict['Files']['FreqDir'] ) if not pdict['Files']['dir_ICE_his'] : pdict['Files']['dir_ICE_his'] = os.path.join ( pdict['libIGCM']['R_SAVE'], "ICE", pdict['Files']['FreqDir'] ) echo ( 'The analysis relies on files from the following model output directories : ', f_out ) echo ( f"{pdict['Files']['dir_ATM_his'] = }" , f_out ) echo ( f"{pdict['Files']['dir_SRF_his'] = }" , f_out ) echo ( f"{pdict['Files']['dir_OCE_his'] = }" , f_out ) echo ( f"{pdict['Files']['dir_ICE_his'] = }" , f_out ) ## Creates files names ## -------------------- if not pdict['Experiment']['Period'] : if pdict['Experiment']['Freq'] == 'MO' : pdict['Experiment']['Period'] = f"{pdict['Experiment']['DateBegin']}_{pdict['Experiment']['DateEnd']}_1M" if pdict['Experiment']['Freq'] == 'SE' : pdict['Experiment']['Period'] = f"SE_{pdict['Files']['DateBegin']}_{pdict['Files']['DateEnd']}_1M" echo ( f"Period : {pdict['Experiment']['Period']}", f_out ) if not pdict['Files']['FileCommon'] : pdict['Files']['FileCommon'] = f"{pdict['Experiment']['JobName']}_{pdict['Experiment']['Period']}" if not pdict['Files']['Title'] : pdict['Files']['Title'] = f"{pdict['Experiment']['JobName']} : {pdict['Experiment']['Freq']} : {pdict['Experiment']['DateBegin']} - {pdict['Experiment']['DateEnd']}" echo ('\nOpen history files', f_out ) if not pdict['Files']['file_ATM_his'] : if pdict['Experiment']['ATM_HIS'] == 'latlon' : pdict['Files']['file_ATM_his'] = os.path.join ( pdict['Files']['dir_ATM_his'], f"{pdict['Files']['FileCommon']}_histmth.nc" ) if pdict['Experiment']['ATM_HIS'] == 'ico' : pdict['Files']['file_ATM_his'] = os.path.join ( pdict['Files']['dir_ATM_his'], f"{pdict['Files']['FileCommon']}_histmth_ico.nc" ) if not pdict['Files']['file_SRF_his'] : if pdict['Experiment']['ATM_HIS'] == 'latlon' : pdict['Files']['file_SRF_his'] = os.path.join ( pdict['Files']['dir_SRF_his'], f"{pdict['Files']['FileCommon']}_sechiba_history.nc" ) if pdict['Experiment']['ATM_HIS'] == 'ico' : pdict['Files']['file_SRF_his'] = os.path.join ( pdict['Files']['dir_SRF_his'], f"{pdict['Files']['FileCommon']}_sechiba_history_ico.nc" ) if pdict['Experiment']['SRF'] and pdict['Experiment']['Routing'] == 'SIMPLE' : if not pdict['Files']['file_RUN_his'] : if pdict['Experiment']['RUN_HIS'] == 'latlon' : pdict['Files']['file_RUN_his'] = os.path.join ( pdict['Files']['dir_SRF_his'], f"{pdict['Files']['FileCommon']}_sechiba_history.nc" ) if pdict['Experiment']['SRF_HIS'] == 'ico' : pdict['Files']['file_RUN_his'] = os.path.join ( pdict['Files']['dir_SRF_his'], f"{pdict['Files']['FileCommon']}_sechiba_history_ico.nc" ) echo ( f"{pdict['Files']['file_ATM_his'] = }", f_out ) if pdict['Experiment']['SRF'] : echo ( f"{pdict['Files']['file_SRF_his'] = }", f_out ) if pdict['Experiment']['Routing'] == 'SIMPLE' : echo ( f"{pdict['Files']['file_RUN_his'] = }", f_out ) if not pdict['Files']['file_OCE_his'] : pdict['Files']['file_OCE_his'] = os.path.join ( pdict['Files']['dir_OCE_his'], f"{pdict['Files']['FileCommon']}_grid_T.nc" ) if not pdict['Files']['file_OCE_sca'] : pdict['Files']['file_OCE_sca'] = os.path.join ( pdict['Files']['dir_OCE_his'], f"{pdict['Files']['FileCommon']}_scalar.nc" ) if not pdict['Files']['file_OCE_srf'] : pdict['Files']['file_OCE_srf'] = os.path.join ( pdict['Files']['dir_OCE_his'], f"{pdict['Files']['FileCommon']}_sbc.nc" ) if not pdict['Files']['file_ICE_his'] : pdict['Files']['file_ICE_his'] = os.path.join ( pdict['Files']['dir_ICE_his'], f"{pdict['Files']['FileCommon']}_icemod.nc" ) return pdict def SetRestartNames ( pdict, f_out) : zdict = pdict if not zdict['Files']['TarRestartDate_beg'] : zdict['Files']['TarRestartDate_beg'] = DateMinusOneDay ( zdict['Experiment']['DateBegin'] ) if not zdict['Files']['TarRestartDate_end'] : zdict['Files']['TarRestartDate_end'] = FormatToGregorian ( zdict['Experiment']['DateEnd'] ) if not zdict['Files']['TarRestartPeriod_beg'] : zdict['Files']['TarRestartPeriod_beg_DateEnd'] = zdict['Files']['TarRestartDate_beg'] zdict['Files']['TarRestartPeriod_beg_DateBeg'] = DateAddYear ( zdict['Files']['TarRestartPeriod_beg_DateEnd'], -zdict['Experiment']['zPackFrequency'] ) zdict['Files']['TarRestartPeriod_beg_DateBeg'] = DatePlusOneDay ( zdict['Files']['TarRestartPeriod_beg_DateBeg'] ) zdict['Files']['TarRestartPeriod_beg'] = f"{zdict['Files']['TarRestartPeriod_beg_DateBeg']}_{zdict['Files']['TarRestartPeriod_beg_DateEnd']}" echo ( f"Tar period for initial restart : {zdict['Files']['TarRestartPeriod_beg']}", f_out) if not zdict['Files']['TarRestartPeriod_end'] : zdict['Files']['TarRestartPeriod_end_DateEnd'] = zdict['Files']['TarRestartDate_end'] zdict['Files']['TarRestartPeriod_end_DateBeg'] = DateAddYear ( zdict['Files']['TarRestartPeriod_end_DateEnd'], -zdict['Experiment']['zPackFrequency'] ) zdict['Files']['TarRestartPeriod_end_DateBeg'] = DatePlusOneDay ( zdict['Files']['TarRestartPeriod_end_DateBeg']) zdict['Files']['TarRestartPeriod_end'] = f"{zdict['Files']['TarRestartPeriod_end_DateBeg']}_{zdict['Files']['TarRestartPeriod_end_DateEnd']}" echo ( f"Tar period for final restart : {zdict['Files']['TarRestartPeriod_end']}", f_out ) echo ( f"Restart dates - Start : {zdict['Files']['TarRestartPeriod_beg']} / End : {zdict['Files']['TarRestartPeriod_end']}", f_out) if not zdict['Files']['tar_restart_beg'] : zdict['Files']['tar_restart_beg'] = os.path.join ( zdict['libIGCM']['R_SAVE'], 'RESTART', f"{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartPeriod_beg']}_restart.tar" ) if not zdict['Files']['tar_restart_end'] : zdict['Files']['tar_restart_end'] = os.path.join ( zdict['libIGCM']['R_SAVE'], 'RESTART', f"{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartPeriod_end']}_restart.tar" ) echo ( f"{ zdict['Files']['tar_restart_beg'] = }", f_out ) echo ( f"{ zdict['Files']['tar_restart_end'] = }", f_out ) ##-- Names of tar files with restarts if not zdict['Files']['tar_restart_beg_ATM'] : zdict['Files']['tar_restart_beg_ATM'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_beg_DYN'] : zdict['Files']['tar_restart_beg_DYN'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_beg_RUN'] : zdict['Files']['tar_restart_beg_RUN'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_beg_OCE'] : zdict['Files']['tar_restart_beg_OCE'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_beg_ICE'] : zdict['Files']['tar_restart_beg_ICE'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_end_ATM'] : zdict['Files']['tar_restart_end_ATM'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['tar_restart_end_DYN'] : zdict['Files']['tar_restart_end_DYN'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['tar_restart_end_RUN'] : zdict['Files']['tar_restart_end_RUN'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['tar_restart_end_OCE'] : zdict['Files']['tar_restart_end_OCE'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['tar_restart_end_ICE'] : zdict['Files']['tar_restart_end_ICE'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['file_DYN_beg'] : if zdict['Experiment']['LMDZ'] : zdict['Files']['file_DYN_beg'] = f"{zdict['Files']['FileDir']}/ATM_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_restart.nc" if zdict['Experiment']['ICO'] : zdict['Files']['file_DYN_beg'] = f"{zdict['Files']['FileDir']}/ICO_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_restart.nc" if not zdict['Files']['file_DYN_end'] : if zdict['Experiment']['LMDZ'] : zdict['Files']['file_DYN_end'] = f"{zdict['Files']['FileDir']}/ATM_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_restart.nc" if zdict['Experiment']['ICO'] : zdict['Files']['file_DYN_end'] = f"{zdict['Files']['FileDir']}/ICO_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_restart.nc" if zdict['Experiment']['SRF'] : if not zdict['Files']['file_SRF_beg'] : zdict['Files']['file_SRF_beg'] = f"{zdict['Files']['FileDir']}/SRF_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_sechiba_rest.nc" if not zdict['Files']['file_SRF_end'] : zdict['Files']['file_SRF_end'] = f"{zdict['Files']['FileDir']}/SRF_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_sechiba_rest.nc" if zdict['Experiment']['SRF'] : if not zdict['Files']['tar_restart_beg_SRF'] : zdict['Files']['tar_restart_beg_SRF'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_end_SRF'] : zdict['Files']['tar_restart_end_SRF'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['file_ATM_beg'] : zdict['Files']['file_ATM_beg'] = f"{zdict['Files']['FileDir']}/ATM_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_restartphy.nc" if not zdict['Files']['file_ATM_end'] : zdict['Files']['file_ATM_end'] = f"{zdict['Files']['FileDir']}/ATM_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_restartphy.nc" if zdict['Experiment']['SRF'] : echo ( f"{zdict['Files']['file_SRF_beg'] = }", f_out) echo ( f"{zdict['Files']['file_SRF_end'] = }", f_out) if zdict['Experiment']['ICO'] : if not zdict['Files']['file_DYN_aire'] : zdict['Files']['file_DYN_aire'] = os.path.join ( zdict['libIGCM']['R_IN'], 'ATM', 'GRID', f"{zdict['Experiment']['ResolAtm']}_grid.nc" ) if zdict['Experiment']['SRF'] and zdict['Experiment']['Routing'] == 'SIMPLE' : if not zdict['Files']['file_RUN_beg'] : zdict['Files']['file_RUN_beg'] = f"{ zdict['Files']['FileDir']}/SRF_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_routing_restart.nc" if not zdict['Files']['file_RUN_end'] : zdict['Files']['file_RUN_end'] = f"{ zdict['Files']['FileDir']}/SRF_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_routing_restart.nc" if not zdict['Files']['tar_restart_beg_OCE'] : zdict['Files']['tar_restart_beg_OCE'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_beg_ICE'] : zdict['Files']['tar_restart_beg_ICE'] = zdict['Files']['tar_restart_beg'] if not zdict['Files']['tar_restart_end_OCE'] : zdict['Files']['tar_restart_end_OCE'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['tar_restart_end_ICE'] : zdict['Files']['tar_restart_end_ICE'] = zdict['Files']['tar_restart_end'] if not zdict['Files']['file_OCE_beg'] : zdict['Files']['file_OCE_beg'] = f"{zdict['Files']['FileDir']}/OCE_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_restart.nc" if not zdict['Files']['file_OCE_end'] : zdict['Files']['file_OCE_end'] = f"{zdict['Files']['FileDir']}/OCE_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_restart.nc" if not zdict['Files']['file_ICE_beg'] : zdict['Files']['file_ICE_beg'] = f"{zdict['Files']['FileDir']}/ICE_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_beg']}_restart_icemod.nc" if not zdict['Files']['file_ICE_end'] : zdict['Files']['file_ICE_end'] = f"{zdict['Files']['FileDir']}/ICE_{zdict['Experiment']['JobName']}_{zdict['Files']['TarRestartDate_end']}_restart_icemod.nc" return zdict def config2dict ( pconf ) : ''' Convert a config parser object into a dictionary ''' zdict = {} for section in pconf.keys () : zdict[section] = {} for key in pconf[section].keys() : zz = str2value ( pconf[section][key] ) zdict[section].update ( {key:zz} ) return zdict def dict2config ( pdict ): ''' Convert a dictionary into a configparser object The dictionary should have two levels (see configparser) ''' zconf = configparser.ConfigParser ( interpolation=configparser.ExtendedInterpolation() ) zconf.optionxform = str # To keep capitals for section in pdict.keys () : zconf[section] = {} for key in pdict[section].keys() : zconf[section][key] = str (pdict[section][key] ) return zconf # def updateConf ( pconf, pdict ): # ''' # Update a config parser with a dictionary # ''' # zconf = pconf # for section in pdict.keys() : # if section not in pconf.keys() : pconf[section] = {} # for key in pdict[section].keys() : # pconf[section][key] = str(pdict[section][key]) # return pconf def echo (string, f_out, end='\n') : '''Function to print to stdout *and* output file''' print ( str(string), end=end ) sys.stdout.flush () f_out.write ( str(string) + end ) f_out.flush () def str2value ( pz ) : ''' Tries to convert a string into integer, real, boolean or None ''' zz = pz zz = setBool (zz) zz = setNum (zz) zz = setNone (zz) return zz def setBool (chars) : '''Convert specific char string in boolean if possible''' z_set_bool = chars if isinstance (chars, str) : for key in configparser.ConfigParser.BOOLEAN_STATES.keys () : if chars.lower() == key : z_set_bool = configparser.ConfigParser.BOOLEAN_STATES[key] return z_set_bool def setNum (chars) : '''Convert specific char string in integer or real if possible''' if isinstance (chars, str) : realnum = re.compile ("^[-+]?[0-9]*\.?[0-9]+(e[-+]?[0-9]+)?$") is_number = realnum.match(chars.strip()) != None is_int = chars.strip().isdigit() if is_number : if is_int : zset_num = int (chars) else : zset_num = float (chars) else : zset_num = chars else : zset_num = chars return zset_num def setNone (chars) : '''Convert specific char string to None if possible''' zset_none = chars if isinstance (chars, str) : if chars in ['None', 'NONE', 'none'] : zset_none = None return zset_none def unDefined (char) : '''True if a variable is not defined, or set to None''' if char in globals () : if globals()[char] is None : zun_defined = True else : zun_defined = False else : zun_defined = True return zun_defined def Defined (char) : '''True if a variable is defined and not equal to None''' if char in globals () : if globals()[char] is None : zdefined = False else : zdefined = True else : zdefined = False return zdefined def Ksum (tab) : '''Kahan summation algorithm, also known as compensated summation https://en.wikipedia.org/wiki/Kahan_summation_algorithm ''' # Prepare the accumulator. ksum = 0.0 # A running compensation for lost low-order bits. comp = 0.0 for xx in np.where ( np.isnan(tab), 0., tab ) : # comp is zero the first time around. yy = xx - comp # Alas, sum is big, y small, so low-order digits of y are lost. tt = ksum + yy # (tt - Ksum) cancels the high-order part of y; subtracting y recovers negative (low part of yy) comp = (tt - ksum) - yy # Algebraically, comp should always be zero. Beware overly-aggressive optimizing compilers ! ksum = tt # Next time around, the lost low part will be added to y in a fresh attempt. return ksum def Ssum (tab) : '''Precision summation by sorting by absolute values''' ssum = np.sum ( tab[np.argsort(np.abs(tab))] ) return ssum def KSsum (tab) : '''Precision summation by sorting by absolute value, and applying Kahan summation''' kssum = Ksum ( tab[np.argsort(np.abs(tab))] ) return kssum # Choosing algorithm for precise summation Psum = KSsum def IsLeapYear ( year, CalendarType="Gregorian" ) : '''True is Year is a leap year''' year = int ( year ) zis_leap_year = None # What is the calendar : if CalendarType in [ '360d', '360_day', 'noleap', '365_day'] : zis_leap_year = False if CalendarType in [ 'all_leap', '366_day' ] : zis_leap_year = True # a year is a leap year if it is even divisible by 4 # but not evenly divisible by 100 # unless it is evenly divisible by 400 # if it is evenly divisible by 400 it must be a leap year if not zis_leap_year and np.mod ( year, 400 ) == 0 : zis_leap_year = True # if it is evenly divisible by 100 it must not be a leap year if not zis_leap_year and np.mod ( year, 100 ) == 0 : zis_leap_year = False # if it is evenly divisible by 4 it must be a leap year if not zis_leap_year and np.mod ( year, 4 ) == 0 : zis_leap_year = True if not zis_leap_year : zis_leap_year = False return zis_leap_year def DateFormat ( date ) : '''Get date format : [yy]yymmdd is Gregorian [yy]yy-mm-dd is Human ''' if isinstance (date, str) : if '-' in date : zdate_format = 'Human' else : zdate_format = 'Gregorian' if isinstance (date, int) : zdate_format = 'Gregorian' return zdate_format def PrintDate ( ye, mo, da, pformat ) : '''Return a date in the requested format ''' if pformat == 'Human' : zPrintDate = f'{ye:04d}-{mo:02d}-{da:02d}' if pformat == 'Gregorian' : zPrintDate = f'{ye:04d}{mo:02d}{da:02d}' return zPrintDate def FormatToGregorian ( date ) : '''From a yyyy-mm-dd or yyymmdd date format returns a yyymmdd date format ''' ye, mo, da = SplitDate ( date ) return f'{ye:04d}{mo:02d}{da:02d}' def FormatToHuman ( date ) : '''From a yyyymmdd or yyymmdd date format returns a yyy-mm-dd date format ''' ye, mo, da = SplitDate ( date ) return f'{ye:04d}-{mo:02d}-{da:02d}' def SplitDate ( date ) : '''Split Date in format [yy]yymmdd or [yy]yy-mm-dd to yy, mm, dd ''' if isinstance (date, str) : if '-' in date : ye, mo, da = date.split ('-') else : ye, mo, da = date[:-4], date[-4:-2], date[-2:] if isinstance (date, int) : da = np.mod ( date, 100) mo = np.mod ( date//100, 100) ye = date // 10000 ye = int(ye) ; mo = int(mo) ; da=int(da) return ye, mo, da def DateAddYear ( date, year_inc=1 ) : '''Add on year(s) to date in format [yy]yymmdd or [yy]yy-mm-dd''' zformat = DateFormat ( date ) ye, mo, da = SplitDate ( date ) ye_new = ye + year_inc return PrintDate ( ye_new, mo, da, zformat) def DateMinusOneDay ( date ) : '''Substracts one day to date in format [yy]yymmdd or [yy]yy-mm-dd''' zformat = DateFormat ( date ) mth_length = np.array ( [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] ) ye, mo, da = SplitDate ( date ) if IsLeapYear ( ye) : mth_length[1] += 1 ye = int(ye) ; mo = int(mo) ; da=int(da) if da == 1 : if mo == 1 : da_new, mo_new, ye_new = mth_length[-1 ], 12 , ye - 1 else : da_new, mo_new, ye_new = mth_length[mo-2], mo - 1, ye else : da_new, mo_new, ye_new = da - 1, mo, ye return PrintDate ( ye_new, mo_new, da_new, zformat) def DatePlusOneDay ( date ) : '''Add one day to date in format [yy]yymmdd or [yy]yy-mm-dd''' zformat = DateFormat ( date ) mth_length = np.array ( [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] ) ye, mo, da = SplitDate ( date ) if IsLeapYear ( ye ) : mth_length[1] += 1 ye_new = ye mo_new = mo da_new = da+1 if da_new > mth_length [mo_new-1] : da_new = 1 mo_new = mo_new + 1 if mo_new == 13 : mo_new = 1 ye_new += 1 return PrintDate ( ye_new, mo_new, da_new, zformat ) class Timer : '''Measures execution time of a function''' def __str__ (self): return str (self.__class__) def __name__ (self): return self.__class__.__name__ def __init__ (self, func, debug=False, timer=True) : functools.update_wrapper (self, func) self.func = func self.num_calls = 0 self.cumul_time = 0. self.debug = debug self.timer = timer def __call__ (self, *args, **kwargs) : self.num_calls += 1 if self.debug : print ( f'>-- Calling {self.__name__} --------------------------------------------------' ) args_repr = [f"{repr (a)}" for a in args] kwargs_repr = [f"{k}={v!r}" for k, v in kwargs.items ()] signature = ", ".join (args_repr + kwargs_repr) print ( f"Signature : ({signature}") start_time = time.perf_counter () values = self.func (*args, **kwargs) end_time = time.perf_counter () run_time = end_time - start_time self.cumul_time += run_time if self.timer : print ( f"--> Calling {self.__name__!r} : {run_time:.3f} secs / cumul {self.cumul_time:.3f} # {self.num_calls:2d} calls") if self.debug : print ( '<------------------------------------------------------------' ) return values