Functions | |
| def | hexify |
| def | md5sum |
| def | CheckDirectory |
| def | IsLocked |
| def | GetTimestamp |
| def | SelectDbForUpdate |
| def | DownloadDb |
| def | main |
Variables | |
| string | HOST = "http://docbes3.ihep.ac.cn/db/" |
| int | BLOCKSIZE = 1024 |
| string | gzsuffix = ".gz" |
| string | tmpsuffix = ".tmp" |
| def dbupdate::hexify | ( | s | ) |
| def dbupdate::md5sum | ( | path | ) |
Definition at line 19 of file dbupdate.py.
00019 : 00020 sumstring = '' 00021 try: 00022 f = open(path,'rb') 00023 try: 00024 import hashlib 00025 sum = hashlib.md5() 00026 except ImportError: 00027 # for Python << 2.5 00028 import md5 00029 sum = md5.new() 00030 00031 while 1: 00032 block = f.read(BLOCKSIZE) 00033 if not block: 00034 break 00035 sum.update(block) 00036 sumstring = hexify(sum.digest()) 00037 f.close() 00038 except: 00039 print "Error computing md5checksum of ["+path+"]." 00040 00041 return sumstring 00042 00043 def CheckDirectory(destdir):
| def dbupdate::CheckDirectory | ( | destdir | ) |
Definition at line 44 of file dbupdate.py.
00044 : 00045 if(not os.path.exists(destdir)): 00046 os.mkdir(destdir) 00047 00048 if (os.path.exists(destdir) and not os.path.isdir(destdir)): 00049 print destdir+" exists, and it is not a directory!" 00050 return False 00051 00052 if (not os.access(destdir,os.F_OK and os.W_OK and os.X_OK)): 00053 print destdir + " is not writeable" 00054 00055 os.chdir(destdir) 00056 return True 00057 def IsLocked():
| def dbupdate::IsLocked | ( | ) |
Definition at line 58 of file dbupdate.py.
00058 : 00059 try: 00060 urllib2.urlopen(HOST+"/LOCK") 00061 return True 00062 except urllib2.URLError,e: 00063 msg = "Can't access server "+HOST 00064 if hasattr(e, 'reason'): 00065 #some common e.reason values: 00066 # (-2, 'Name or service not known') 00067 # (113, 'No route to host') 00068 try: msg += ": [%s]" % e.reason[1] 00069 except IndexError: pass 00070 if hasattr(e, 'code') and hasattr(e, 'msg'): 00071 #in this case e is HTTPError, a subclass of URLError) 00072 #some common *e.code, e.msg) values: 00073 # (404, 'Not Found') 00074 # (403, 'Forbidden') 00075 # (500, 'Internal Server Error') 00076 if(e.code==404): return False; 00077 msg += ": HTTP error [%s, %s]" % (e.code, e.msg) 00078 msg += "." 00079 print msg 00080 exit(-1) 00081 def GetTimestamp():
| def dbupdate::GetTimestamp | ( | ) |
Definition at line 82 of file dbupdate.py.
00082 : 00083 Files={} 00084 try: 00085 opener = urllib2.build_opener() 00086 sock = opener.open(HOST+"/db.timestamp") 00087 text = sock.read() 00088 sock.close() 00089 lines=text.splitlines() 00090 except urllib2.URLError,e: 00091 msg = "Can't access server "+HOST 00092 if hasattr(e, 'reason'): 00093 #some common e.reason values: 00094 # (-2, 'Name or service not known') 00095 # (113, 'No route to host') 00096 try: msg += ": [%s]" % e.reason[1] 00097 except IndexError: pass 00098 if hasattr(e, 'code') and hasattr(e, 'msg'): 00099 #in this case e is HTTPError, a subclass of URLError) 00100 #some common *e.code, e.msg) values: 00101 # (404, 'Not Found') 00102 # (403, 'Forbidden') 00103 # (500, 'Internal Server Error') 00104 msg += ": HTTP error [%s, %s]" % (e.code, e.msg) 00105 print msg 00106 exit(-1) 00107 except: 00108 print "Could not retrieve timestamp" 00109 exit(-1) 00110 timestamp=int(lines[0]) 00111 for line in lines[1:]: 00112 temp1,temp2 = line.split() 00113 Files[temp2]=temp1 00114 return timestamp,Files 00115 def SelectDbForUpdate(timestamp,Files):
| def dbupdate::SelectDbForUpdate | ( | timestamp, | ||
| Files | ||||
| ) |
Definition at line 116 of file dbupdate.py.
00116 : 00117 LoadList = [] 00118 00119 if(os.path.exists("db.timestamp")): 00120 ts = open("db.timestamp") 00121 lines = ts.readlines(); 00122 ts.close() 00123 00124 if(not lines[0].strip().isdigit()): 00125 print "File db.timestamp is broken. Re-installing everything from scratch." 00126 try: os.unlink("db.timestamp") 00127 except: pass 00128 for fname in Files.keys(): 00129 LoadList.append(fname) 00130 return LoadList 00131 00132 cur_timestamp = int(lines[0]) 00133 00134 if( cur_timestamp == timestamp ): 00135 print "Warning: current database is up-to-date. Do nothing." 00136 return LoadList 00137 elif( cur_timestamp >= timestamp ): 00138 print "Warning! Please check: current database is NEWER than the remote replica. Do nothing." 00139 return LoadList 00140 00141 for fname in Files.keys(): 00142 if(not os.path.exists(fname)): 00143 print "Warning: " + fname + " is lost" 00144 LoadList.append(fname) 00145 else: 00146 chksum = md5sum(fname) 00147 if(not Files[fname] == chksum): 00148 print "Checksum of "+fname+" is different" 00149 LoadList.append(fname) 00150 else: 00151 print "No db.timestamp is present. Considering fresh installation." 00152 for fname in Files.keys(): 00153 LoadList.append(fname) 00154 return LoadList 00155 00156 def DownloadDb(dbname):
| def dbupdate::DownloadDb | ( | dbname | ) |
Definition at line 157 of file dbupdate.py.
00157 : 00158 tmpdbname = dbname+tmpsuffix+gzsuffix 00159 remotedbname = dbname+gzsuffix 00160 if( os.path.exists(tmpdbname)): 00161 os.unlink(tmpdbname) 00162 00163 try: 00164 # check lock 00165 for i in range(10): 00166 if(IsLocked()): 00167 var="" 00168 if(i>0): var="\r" 00169 sys.stdout.write(var+"Lock file found. It looks like server replica is being updated. Sleeping for "+str(i+1)+" min") 00170 sys.stdout.flush() 00171 time.sleep(1) 00172 elif (i>0): 00173 print "\nThe lock was removed" 00174 break 00175 else: 00176 break 00177 if(i==9): 00178 print "\nTimeout: 10 min waiting to unlock the server. Please try again later, or complain." 00179 exit(-1) 00180 00181 ret = os.system("wget "+HOST+"/"+remotedbname+" -O "+tmpdbname) 00182 except os.error,e: 00183 print "Error %d: %s" % (e.args[0], e.args[1]) 00184 try:os.unlink(tmpdbname) 00185 except: pass 00186 return "" 00187 except: 00188 print "Unable download file "+remotedbname 00189 try: os.unlink(tmpdbname) 00190 except: pass 00191 exit(-1) 00192 00193 if(ret != 0 or not os.path.exists(tmpdbname) or os.path.getsize(tmpdbname)==0): 00194 print "Unable download database "+tmpdbname 00195 if(os.path.exists(tmpdbname)): 00196 os.unlink(tmpdbname) 00197 return "" 00198 00199 ret = os.system("gunzip -f "+tmpdbname) 00200 if(not ret == 0): 00201 print "Unable to uncompress file "+tmpdbname 00202 return "" 00203 00204 return dbname+tmpsuffix 00205 def main(argv):
| def dbupdate::main | ( | argv | ) |
Definition at line 206 of file dbupdate.py.
00206 : 00207 argc=len(argv) 00208 destdir="" 00209 if(argc==2): 00210 destdir=argv[1] 00211 elif(argc>2): 00212 print "Usage: dbupdate.py [LOCAL_DB_DIR]" 00213 exit(0) 00214 00215 if(destdir==""): 00216 var = os.getenv("SITEROOT") 00217 if(var <> None): 00218 destdir=os.path.join(var,"database") 00219 00220 if(destdir==""): 00221 print "Destination directory neither given explicitly, nor can be derived from $BesArea. Use local directory." 00222 destdir="./" 00223 00224 print "Install/update databases in the directory "+destdir 00225 00226 # Check wget is in the PATH 00227 if(os.system("wget &> /dev/null")==32512): 00228 print "Unable to find wget. Please install it first" 00229 exit(-1); 00230 00231 # Change directory to destdir and check if it is writable 00232 if(not CheckDirectory(destdir)): 00233 exit(-1) 00234 00235 # Get timestamp and checksums 00236 timestamp,Files = GetTimestamp() 00237 00238 # Check if db files are already loaded, and if they should be updated 00239 ListToLoad = SelectDbForUpdate(timestamp,Files) 00240 00241 if(len(ListToLoad)>0): 00242 print "Going to download files: ", ListToLoad 00243 00244 # Download databases to temporary files 00245 ListDone = [] 00246 for db in ListToLoad: 00247 fname=DownloadDb(db) 00248 if(not fname == ""): 00249 ListDone.append(fname) 00250 00251 # Check size and md5sum and rename temporary db to production db 00252 for tmpdbname in ListDone: 00253 chksum = md5sum(tmpdbname) 00254 dbname = tmpdbname.replace(tmpsuffix,"") 00255 if(chksum == Files[dbname]): 00256 os.rename(tmpdbname,dbname) 00257 print "Update database "+dbname 00258 else: 00259 print "Error: MD5 checksum is wrong for file "+tmpdbname 00260 00261 # Update timestamp 00262 f=open("db.timestamp","w") 00263 f.truncate() 00264 f.write(str(timestamp)+"\n") 00265 for db in Files.keys(): 00266 f.write(Files[db]+" "+db+"\n") 00267 f.close() 00268 00269 # Exit 00270 # exit(0) 00271 if __name__ == '__main__':
string dbupdate::HOST = "http://docbes3.ihep.ac.cn/db/" [static] |
Definition at line 11 of file dbupdate.py.
int dbupdate::BLOCKSIZE = 1024 [static] |
Definition at line 12 of file dbupdate.py.
string dbupdate::gzsuffix = ".gz" [static] |
Definition at line 13 of file dbupdate.py.
string dbupdate::tmpsuffix = ".tmp" [static] |
Definition at line 14 of file dbupdate.py.
1.4.7