[2212] | 1 | #include "mem_checker.hpp" |
---|
[2420] | 2 | #include "cxios.hpp" |
---|
[2212] | 3 | #include "mpi.hpp" |
---|
| 4 | #include <string> |
---|
| 5 | #include <map> |
---|
| 6 | #include <iostream> |
---|
| 7 | #include <sstream> |
---|
| 8 | |
---|
[2418] | 9 | #include <fcntl.h> |
---|
| 10 | #include <iomanip> |
---|
| 11 | #include <unistd.h> |
---|
| 12 | #include <cstring> |
---|
| 13 | |
---|
[2212] | 14 | namespace xios |
---|
| 15 | { |
---|
| 16 | CMemChecker CMemChecker::dummy_("") ; |
---|
| 17 | std::map<std::string,CMemChecker> CMemChecker::allMemChecker_; |
---|
| 18 | bool CMemChecker::enabled_=true; |
---|
| 19 | bool CMemChecker::first_=true; |
---|
[2418] | 20 | double CMemChecker::vsize_init_=0; |
---|
[2425] | 21 | double CMemChecker::rss_init_=0; |
---|
| 22 | double CMemChecker::vmhwm_init_=0; |
---|
[2418] | 23 | double CMemChecker::time_init_=0; |
---|
[2420] | 24 | std::ofstream CMemChecker::fout_; |
---|
| 25 | int CMemChecker::flush_counter_=1; |
---|
| 26 | |
---|
[2426] | 27 | CMemChecker::CMemChecker(const std::string& name) : name_(name) |
---|
[2212] | 28 | { |
---|
| 29 | if (first_) check() ; |
---|
| 30 | reset(); |
---|
| 31 | } |
---|
| 32 | |
---|
| 33 | void CMemChecker::check(void) |
---|
| 34 | { |
---|
| 35 | std::ifstream statStream("/proc/self/stat",std::ios_base::in); |
---|
| 36 | enabled_ &= statStream.good() ; |
---|
| 37 | first_=false ; |
---|
| 38 | } |
---|
[2426] | 39 | |
---|
| 40 | // Return vector of memories amount : |
---|
| 41 | // - vector[0] : virtual memory (vsize) from /proc/self/stat |
---|
| 42 | // - vector[1] : resident set size (rss) from /proc/self/stat |
---|
| 43 | std::vector<double> CMemChecker::getMem(void) |
---|
[2212] | 44 | { |
---|
[2426] | 45 | std::vector<double> memories(0); |
---|
| 46 | memories.clear(); |
---|
| 47 | |
---|
[2212] | 48 | if (first_) check() ; |
---|
[2426] | 49 | if (!enabled_) return memories; |
---|
[2212] | 50 | std::ifstream statStream("/proc/self/stat",std::ios_base::in); |
---|
| 51 | std::string dummy ; |
---|
| 52 | for(int i=1;i<=22;i++) statStream>>dummy ; |
---|
| 53 | unsigned long vsize; |
---|
| 54 | statStream>>vsize ; |
---|
[2426] | 55 | memories.push_back(vsize); |
---|
| 56 | unsigned long rss; |
---|
| 57 | statStream>>rss ; |
---|
| 58 | memories.push_back(rss*4096); |
---|
| 59 | return memories ; |
---|
[2212] | 60 | } |
---|
[2425] | 61 | |
---|
[2426] | 62 | // Return vector of memories amount, register values (see _init) as first call as reference : |
---|
[2425] | 63 | // - vector[0] : virtual memory (vsize) from /proc/self/stat |
---|
| 64 | // - vector[1] : resident set size (rss) from /proc/self/stat |
---|
| 65 | // - vector[2] : Peak resident set size (VmHWM) from /proc/self/status |
---|
| 66 | std::vector<double> CMemChecker::getMemories(void) |
---|
| 67 | { |
---|
| 68 | std::vector<double> memories; |
---|
| 69 | |
---|
| 70 | if (first_) check() ; |
---|
| 71 | if (!enabled_) return memories; |
---|
| 72 | std::ifstream statStream("/proc/self/stat",std::ios_base::in); |
---|
| 73 | std::string dummy ; |
---|
| 74 | for(int i=1;i<=22;i++) statStream>>dummy ; |
---|
| 75 | |
---|
| 76 | // 1 - Virtual |
---|
[2426] | 77 | unsigned long vsize; |
---|
[2425] | 78 | statStream>>vsize ; |
---|
| 79 | if (vsize_init_==0) { |
---|
| 80 | vsize_init_ = vsize; |
---|
| 81 | time_init_=MPI_Wtime(); |
---|
| 82 | } |
---|
| 83 | vsize -= vsize_init_; |
---|
| 84 | memories.push_back(vsize); |
---|
| 85 | |
---|
| 86 | // 2 - RSS |
---|
[2426] | 87 | unsigned long rss; |
---|
[2425] | 88 | statStream>>rss ; |
---|
| 89 | if (rss_init_==0) { |
---|
| 90 | rss_init_ = rss; |
---|
| 91 | } |
---|
| 92 | rss -= rss_init_; |
---|
| 93 | rss *= 4096; //getconf("PAGE_SIZE"); |
---|
| 94 | memories.push_back(rss); |
---|
| 95 | |
---|
| 96 | // 3 - Peak |
---|
| 97 | char sbuf[1024]; |
---|
| 98 | std::ifstream file( "/proc/self/status" ); |
---|
| 99 | if( file.fail() ) { |
---|
| 100 | return memories; |
---|
| 101 | } |
---|
| 102 | int fd = open( "/proc/self/status", O_RDONLY, 0 ); |
---|
| 103 | int num_read=read( fd, sbuf, ( sizeof sbuf )-1 ); |
---|
| 104 | close( fd ); |
---|
| 105 | if( !num_read ) { |
---|
| 106 | return memories; |
---|
| 107 | } |
---|
| 108 | // Peak resident set size |
---|
| 109 | char *S=strstr( sbuf, "VmHWM:" )+6; |
---|
| 110 | double vmhwm = ( int )atoi( S ); |
---|
| 111 | if (vmhwm_init_==0) { |
---|
| 112 | vmhwm_init_ = vmhwm; |
---|
| 113 | } |
---|
| 114 | vmhwm -= vmhwm_init_; |
---|
| 115 | memories.push_back(vmhwm); |
---|
| 116 | |
---|
| 117 | return memories; |
---|
| 118 | } |
---|
| 119 | |
---|
[2418] | 120 | void CMemChecker::logMem( std::string id, bool finalizeLog ) |
---|
| 121 | { |
---|
[2420] | 122 | if ( !CXios::logMemory ) return ; |
---|
| 123 | |
---|
[2418] | 124 | int rk = 0; |
---|
| 125 | MPI_Comm_rank( MPI_COMM_WORLD, &rk ); |
---|
[2419] | 126 | std::string logName("xios_memory_"+std::to_string(rk)+".csv"); |
---|
[2425] | 127 | vector<double> mem = getMemories(); |
---|
| 128 | if (!mem[0]) { |
---|
[2420] | 129 | fout_.open( logName ); |
---|
[2425] | 130 | fout_ << "time,event,vsize,rss,VmHWM" << std::endl; |
---|
[2418] | 131 | } |
---|
| 132 | |
---|
[2420] | 133 | fout_.precision(4); |
---|
[2418] | 134 | // Time format : YYYY-MM-DD HH:MM:SS.XXX -> seconds * 1000. |
---|
[2425] | 135 | fout_ << (MPI_Wtime()-time_init_) << "," << id |
---|
| 136 | << "," << mem[0]/1024./1024. |
---|
| 137 | << "," << mem[1]/1024./1024. |
---|
| 138 | << "," << mem[2]/1024. |
---|
| 139 | << std::endl; |
---|
[2418] | 140 | |
---|
[2420] | 141 | if ((MPI_Wtime()-time_init_)>flush_counter_*600.) |
---|
| 142 | { |
---|
| 143 | fout_.flush(); |
---|
| 144 | flush_counter_++; |
---|
| 145 | } |
---|
| 146 | |
---|
| 147 | if (finalizeLog) |
---|
| 148 | { |
---|
| 149 | fout_.close(); |
---|
| 150 | } |
---|
[2418] | 151 | } |
---|
| 152 | |
---|
[2212] | 153 | |
---|
| 154 | void CMemChecker::suspend(void) |
---|
| 155 | { |
---|
| 156 | if (first_) check() ; |
---|
| 157 | if (!enabled_) return ; |
---|
[2426] | 158 | if (!suspended_) |
---|
| 159 | { |
---|
| 160 | vector<double> mem = getMem(); |
---|
| 161 | cumulatedMem_.resize( mem.size() ); |
---|
| 162 | for (int i=0;i<mem.size() ; i++) |
---|
| 163 | { |
---|
| 164 | cumulatedMem_[i] += mem[i] - lastMem_[i]; |
---|
| 165 | } |
---|
| 166 | } |
---|
[2212] | 167 | suspended_ = true; |
---|
| 168 | } |
---|
| 169 | |
---|
| 170 | void CMemChecker::resume(void) |
---|
| 171 | { |
---|
| 172 | if (first_) check() ; |
---|
| 173 | if (!enabled_) return ; |
---|
[2426] | 174 | if (suspended_) |
---|
| 175 | { |
---|
| 176 | vector<double> mem = getMem(); |
---|
| 177 | lastMem_.resize( mem.size() ); |
---|
| 178 | for (int i=0;i<mem.size() ; i++) |
---|
| 179 | { |
---|
| 180 | lastMem_[i] = mem[i]; |
---|
| 181 | } |
---|
| 182 | } |
---|
[2212] | 183 | suspended_ = false; |
---|
| 184 | } |
---|
[2418] | 185 | |
---|
[2212] | 186 | void CMemChecker::reset(void) |
---|
| 187 | { |
---|
| 188 | if (first_) check() ; |
---|
| 189 | if (!enabled_) return ; |
---|
[2426] | 190 | cumulatedMem_.clear(); |
---|
[2212] | 191 | suspended_ = true; |
---|
| 192 | } |
---|
| 193 | |
---|
[2426] | 194 | std::vector<double> CMemChecker::getCumulatedMem(void) |
---|
[2212] | 195 | { |
---|
[2426] | 196 | std::vector<double> memories; |
---|
| 197 | memories.clear(); |
---|
[2212] | 198 | if (first_) check() ; |
---|
[2426] | 199 | if (!enabled_) return memories; |
---|
[2212] | 200 | return cumulatedMem_; |
---|
| 201 | } |
---|
| 202 | |
---|
| 203 | CMemChecker& CMemChecker::get(const std::string name) |
---|
| 204 | { |
---|
| 205 | if (first_) check() ; |
---|
| 206 | if (!enabled_) return dummy_ ; |
---|
| 207 | else |
---|
| 208 | { |
---|
| 209 | std::map<std::string,CMemChecker>::iterator it = allMemChecker_.find(name); |
---|
| 210 | if (it == allMemChecker_.end()) |
---|
| 211 | it = allMemChecker_.insert(std::make_pair(name, CMemChecker(name))).first; |
---|
| 212 | return it->second; |
---|
| 213 | } |
---|
| 214 | } |
---|
| 215 | |
---|
| 216 | std::string CMemChecker::getAllCumulatedMem(void) |
---|
| 217 | { |
---|
| 218 | if (first_) check() ; |
---|
| 219 | if (!enabled_) return std::string(" MemChecker : memory consumption report not available") ; |
---|
| 220 | std::ostringstream strOut ; |
---|
| 221 | const double Kb=1024 ; |
---|
| 222 | const double Mb=Kb*1024 ; |
---|
| 223 | const double Gb=Mb*1024 ; |
---|
[2235] | 224 | const double Tb=Gb*1024 ; |
---|
[2212] | 225 | for(std::map<std::string,CMemChecker>::iterator it=allMemChecker_.begin();it!=allMemChecker_.end();++it) |
---|
| 226 | { |
---|
[2426] | 227 | vector<double> mem=it->second.getCumulatedMem() ; |
---|
| 228 | if (mem.size()>0) |
---|
| 229 | { |
---|
| 230 | strOut<<"MemChecker : "<<it->first<<" --> consumed memory : " ; |
---|
| 231 | if ((fabs(mem[0])>=Tb)||(fabs(mem[1])>=Tb)) strOut<< mem[0] / Tb<<" Tb - "<<mem[1] / Tb<<" Tb"<<std::endl ; |
---|
| 232 | else if ((fabs(mem[0])>=Gb)||(fabs(mem[1])>=Gb)) strOut<< mem[0] / Gb<<" Gb - "<<mem[1] / Gb<<" Gb"<<std::endl ; |
---|
| 233 | else if ((fabs(mem[0])>=Mb)||(fabs(mem[1])>=Mb)) strOut<< mem[0] / Mb<<" Mb - "<<mem[1] / Mb<<" Mb"<<std::endl ; |
---|
| 234 | else if ((fabs(mem[0])>=Kb)||(fabs(mem[1])>=Kb)) strOut<< mem[0] / Kb<<" Kb - "<<mem[1] / Kb<<" Kb"<<std::endl ; |
---|
| 235 | else strOut<< mem[0] <<" bytes - "<<mem[1] <<" bytes"<<std::endl ; |
---|
| 236 | } |
---|
[2212] | 237 | } |
---|
| 238 | return strOut.str() ; |
---|
| 239 | } |
---|
| 240 | } |
---|