00001
00002 import sys
00003
00004
00005 import os, os.path
00006 import tempfile
00007 import urllib2
00008 import time
00009
00010
00011 HOST = "http://docbes3.ihep.ac.cn/db/"
00012 BLOCKSIZE = 1024*1024
00013 gzsuffix = ".gz"
00014 tmpsuffix=".tmp"
00015
00016 def hexify(s):
00017 return ("%02x"*len(s)) % tuple(map(ord, s))
00018
00019 def md5sum(path):
00020 sumstring = ''
00021 try:
00022 f = open(path,'rb')
00023 try:
00024 import hashlib
00025 sum = hashlib.md5()
00026 except ImportError:
00027
00028 import md5
00029 sum = md5.new()
00030
00031 while 1:
00032 block = f.read(BLOCKSIZE)
00033 if not block:
00034 break
00035 sum.update(block)
00036 sumstring = hexify(sum.digest())
00037 f.close()
00038 except:
00039 print "Error computing md5checksum of ["+path+"]."
00040
00041 return sumstring
00042
00043
00044 def CheckDirectory(destdir):
00045 if(not os.path.exists(destdir)):
00046 os.mkdir(destdir)
00047
00048 if (os.path.exists(destdir) and not os.path.isdir(destdir)):
00049 print destdir+" exists, and it is not a directory!"
00050 return False
00051
00052 if (not os.access(destdir,os.F_OK and os.W_OK and os.X_OK)):
00053 print destdir + " is not writeable"
00054
00055 os.chdir(destdir)
00056 return True
00057
00058 def IsLocked():
00059 try:
00060 urllib2.urlopen(HOST+"/LOCK")
00061 return True
00062 except urllib2.URLError,e:
00063 msg = "Can't access server "+HOST
00064 if hasattr(e, 'reason'):
00065
00066
00067
00068 try: msg += ": [%s]" % e.reason[1]
00069 except IndexError: pass
00070 if hasattr(e, 'code') and hasattr(e, 'msg'):
00071
00072
00073
00074
00075
00076 if(e.code==404): return False;
00077 msg += ": HTTP error [%s, %s]" % (e.code, e.msg)
00078 msg += "."
00079 print msg
00080 exit(-1)
00081
00082 def GetTimestamp():
00083 Files={}
00084 try:
00085 opener = urllib2.build_opener()
00086 sock = opener.open(HOST+"/db.timestamp")
00087 text = sock.read()
00088 sock.close()
00089 lines=text.splitlines()
00090 except urllib2.URLError,e:
00091 msg = "Can't access server "+HOST
00092 if hasattr(e, 'reason'):
00093
00094
00095
00096 try: msg += ": [%s]" % e.reason[1]
00097 except IndexError: pass
00098 if hasattr(e, 'code') and hasattr(e, 'msg'):
00099
00100
00101
00102
00103
00104 msg += ": HTTP error [%s, %s]" % (e.code, e.msg)
00105 print msg
00106 exit(-1)
00107 except:
00108 print "Could not retrieve timestamp"
00109 exit(-1)
00110 timestamp=int(lines[0])
00111 for line in lines[1:]:
00112 temp1,temp2 = line.split()
00113 Files[temp2]=temp1
00114 return timestamp,Files
00115
00116 def SelectDbForUpdate(timestamp,Files):
00117 LoadList = []
00118
00119 if(os.path.exists("db.timestamp")):
00120 ts = open("db.timestamp")
00121 lines = ts.readlines();
00122 ts.close()
00123
00124 if(not lines[0].strip().isdigit()):
00125 print "File db.timestamp is broken. Re-installing everything from scratch."
00126 try: os.unlink("db.timestamp")
00127 except: pass
00128 for fname in Files.keys():
00129 LoadList.append(fname)
00130 return LoadList
00131
00132 cur_timestamp = int(lines[0])
00133
00134 if( cur_timestamp == timestamp ):
00135 print "Warning: current database is up-to-date. Do nothing."
00136 return LoadList
00137 elif( cur_timestamp >= timestamp ):
00138 print "Warning! Please check: current database is NEWER than the remote replica. Do nothing."
00139 return LoadList
00140
00141 for fname in Files.keys():
00142 if(not os.path.exists(fname)):
00143 print "Warning: " + fname + " is lost"
00144 LoadList.append(fname)
00145 else:
00146 chksum = md5sum(fname)
00147 if(not Files[fname] == chksum):
00148 print "Checksum of "+fname+" is different"
00149 LoadList.append(fname)
00150 else:
00151 print "No db.timestamp is present. Considering fresh installation."
00152 for fname in Files.keys():
00153 LoadList.append(fname)
00154 return LoadList
00155
00156
00157 def DownloadDb(dbname):
00158 tmpdbname = dbname+tmpsuffix+gzsuffix
00159 remotedbname = dbname+gzsuffix
00160 if( os.path.exists(tmpdbname)):
00161 os.unlink(tmpdbname)
00162
00163 try:
00164
00165 for i in range(10):
00166 if(IsLocked()):
00167 var=""
00168 if(i>0): var="\r"
00169 sys.stdout.write(var+"Lock file found. It looks like server replica is being updated. Sleeping for "+str(i+1)+" min")
00170 sys.stdout.flush()
00171 time.sleep(1)
00172 elif (i>0):
00173 print "\nThe lock was removed"
00174 break
00175 else:
00176 break
00177 if(i==9):
00178 print "\nTimeout: 10 min waiting to unlock the server. Please try again later, or complain."
00179 exit(-1)
00180
00181 ret = os.system("wget "+HOST+"/"+remotedbname+" -O "+tmpdbname)
00182 except os.error,e:
00183 print "Error %d: %s" % (e.args[0], e.args[1])
00184 try:os.unlink(tmpdbname)
00185 except: pass
00186 return ""
00187 except:
00188 print "Unable download file "+remotedbname
00189 try: os.unlink(tmpdbname)
00190 except: pass
00191 exit(-1)
00192
00193 if(ret != 0 or not os.path.exists(tmpdbname) or os.path.getsize(tmpdbname)==0):
00194 print "Unable download database "+tmpdbname
00195 if(os.path.exists(tmpdbname)):
00196 os.unlink(tmpdbname)
00197 return ""
00198
00199 ret = os.system("gunzip -f "+tmpdbname)
00200 if(not ret == 0):
00201 print "Unable to uncompress file "+tmpdbname
00202 return ""
00203
00204 return dbname+tmpsuffix
00205
00206 def main(argv):
00207 argc=len(argv)
00208 destdir=""
00209 if(argc==2):
00210 destdir=argv[1]
00211 elif(argc>2):
00212 print "Usage: dbupdate.py [LOCAL_DB_DIR]"
00213 exit(0)
00214
00215 if(destdir==""):
00216 var = os.getenv("SITEROOT")
00217 if(var <> None):
00218 destdir=os.path.join(var,"database")
00219
00220 if(destdir==""):
00221 print "Destination directory neither given explicitly, nor can be derived from $BesArea. Use local directory."
00222 destdir="./"
00223
00224 print "Install/update databases in the directory "+destdir
00225
00226
00227 if(os.system("wget &> /dev/null")==32512):
00228 print "Unable to find wget. Please install it first"
00229 exit(-1);
00230
00231
00232 if(not CheckDirectory(destdir)):
00233 exit(-1)
00234
00235
00236 timestamp,Files = GetTimestamp()
00237
00238
00239 ListToLoad = SelectDbForUpdate(timestamp,Files)
00240
00241 if(len(ListToLoad)>0):
00242 print "Going to download files: ", ListToLoad
00243
00244
00245 ListDone = []
00246 for db in ListToLoad:
00247 fname=DownloadDb(db)
00248 if(not fname == ""):
00249 ListDone.append(fname)
00250
00251
00252 for tmpdbname in ListDone:
00253 chksum = md5sum(tmpdbname)
00254 dbname = tmpdbname.replace(tmpsuffix,"")
00255 if(chksum == Files[dbname]):
00256 os.rename(tmpdbname,dbname)
00257 print "Update database "+dbname
00258 else:
00259 print "Error: MD5 checksum is wrong for file "+tmpdbname
00260
00261
00262 f=open("db.timestamp","w")
00263 f.truncate()
00264 f.write(str(timestamp)+"\n")
00265 for db in Files.keys():
00266 f.write(Files[db]+" "+db+"\n")
00267 f.close()
00268
00269
00270
00271
00272 if __name__ == '__main__':
00273 main(sys.argv)
00274