Go to:
Gentoo Home
Documentation
Forums
Lists
Bugs
Planet
Store
Wiki
Get Gentoo!
Gentoo's Bugzilla – Attachment 22068 Details for
Bug 35535
[PATCH] `emerge sync` refactoring
Home
|
New
–
[Ex]
|
Browse
|
Search
|
Privacy Policy
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
new sync module
sync.py (text/plain), 10.34 KB, created by
Marius Mauch (RETIRED)
on 2003-12-11 20:03:13 UTC
(
hide
)
Description:
new sync module
Filename:
MIME Type:
Creator:
Marius Mauch (RETIRED)
Created:
2003-12-11 20:03:13 UTC
Size:
10.34 KB
patch
obsolete
># sync module, used by `emerge --sync` > >import portage, os, socket, time, shutil, re, string > >def warn(mytext): > portage.writemsg("!!! "+mytext+"\n") > >def info(mytext): > portage.writemsg("* "+mytext+"\n") > >def msg(mytext): > portage.writemsg(mytext+"\n") > >class Connection: > def create(self, src, dest, settings): > src = src.strip() > dest = os.path.normpath(dest) > > pos = src.find("://")+3 > if pos < 0: > raise Exception("invalid SYNC setting: %s" % src) > elif src[:pos] not in transports.keys(): > raise Exception("unknown transport type in SYNC: %s" % src[:pos]) > else: > transtype = transports[src[:pos]][0] > myconnection = transports[src[:pos]][1] > > os.umask(0022) > > if not os.path.exists(dest): > os.makedirs(dest, mode=0755) > > return myconnection(src, dest, settings) > > # check if a given command is available > def toolcheck(self, cmd, pkg, quiet=True): > if os.access(cmd, os.X_OK) and os.path.isfile(cmd): > return True > elif quiet: > return False > else: > warn("%s does not exist or is not executable, so %s-sync support" % (cmd, self.transtype)) > warn("is disabled. Emerge \"%s\" to enable %s-sync support" % (pkg, self.transtype)) > return False > >connection = Connection() > >class CvsConnection(Connection): > def __init__(self, src, dest, settings): > self.transtype = "cvs" > self.syncuri = src[6:] > if not self.syncuri[0] == ":": > self.syncuri = ":"+self.syncuri > self.portdir = dest > self.settings = settings > > def setup(self): > # checking for old SYNC form > if self.syncuri.count(":") == 1: > info("The old syntax for cvs-sync with the implicit gentoo-x86 module is") > info("deprecated. Please change your make.conf to the new syntax:") > info(" cvs://access:user@server:cvsroot:cvsmodule") > info("(access should be \"ext\" for ssh-based CVS as used by Gentoo)") > self.cvsaccess = "ext" > self.cvsroot = self.syncuri > self.cvsmodule = "gentoo-x86" > elif self.syncuri.count(":") in [3, 4]: > self.cvsaccess=self.syncuri.split(":")[1] > mpos=self.syncuri.rindex(":") > self.cvsmodule=self.syncuri[mpos+1:] > self.cvsroot=self.syncuri[:mpos] > else: > warn("invalid cvs SYNC setting: cvs://%s, syntax is:" % self.syncuri) > warn(" cvs://:access:user@server:cvsroot:cvsmodule") > return -1 > > if not self.toolcheck("/usr/bin/cvs", "dev-util/cvs", quiet=False): > return -1 > > def sync(self): > cvsdir=os.path.dirname(self.portdir) > if not os.path.exists(self.portdir+"/CVS"): > #initial checkout > msg(">>> starting initial cvs checkout with %s ...\n" % self.syncuri) > if self.cvsaccess != "ext": > rValue = portage.spawn("cd "+cvsdir+"; cvs -d "+self.cvsroot+" login",free=1) > if rValue: > warn("cvs login error [exitcode %d]; aborting." % rValue) > return rValue > if os.path.exists(cvsdir+"/"+self.cvsmodule): > warn("existing %s/%s directory; aborting." % (cvsdir, self.cvsmodule)) > return -1 > rValue = portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+self.cvsroot+" co "+self.cvsmodule,free=1) > if rValue: > warn("cvs checkout error [exitcode %d]; aborting." % rValue) > return rValue > if cvsdir != self.portdir: > portage.movefile(cvsdir+"/"+self.cvsmodule,self.portdir) > return 0 > else: > #cvs update > msg(">>> starting cvs update with cvs://%s ..." % self.syncuri) > rValue=portage.spawn("cd "+self.portdir+"; cvs -z0 -q update -dP",free=1) > return rValue > >class RsyncBaseConnection(Connection): > options = ["-rlptDvz", "--progress", "--stats", "--delete", \ > "--delete-after", "--exclude='distfiles/*'", \ > "--exclude='local/*'", "--exclude='packages/*'"] > > def __init__(self, src, dest, settings): > self.syncuri = src > self.portdir = dest > self.settings = settings > > > def base_setup(self): > if not self.toolcheck("/usr/bin/rsync", "net-misc/rsync", quiet=False): > return -1 > > self.rsyncCmd = "/usr/bin/rsync" > > # add general options > for op in self.options: > self.rsyncCmd = self.rsyncCmd+" "+op > > if self.settings.has_key("RSYNC_EXCLUDEFROM"): > if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]): > self.rsyncCmd = self.rsyncCmd + " --exclude-from " + self.settings["RSYNC_EXCLUDEFROM"] > else: > warn("RSYNC_EXCLUDEFROM specified, but file does not exist.") > > > def checkRsyncExitcode(self, exitcode): > if exitcode == 1: > warn("Rsync has reported that there is a syntax error. Please ensure") > warn("that your sync statement is proper.") > warn("(sync setting is: %s" % self.syncuri) > elif exitcode == 11: > warn("Rsync has reported that there is a File IO error. Normally") > warn("this means your disk is full, but can be caused by corruption") > warn("on the filesystem that contains the target directory. Please") > warn("investigate and try again after the problem has been fixed.") > warn("(target directory is: %s" % self.portdir) > elif exitcode == 20: > warn("Rsync was killed before it finished.") > elif exitcode > 0: > warn("Rsync has not successfully finished. It is recommended that you keep") > warn("trying or that you use the 'emerge-webrsync' option if you are unable") > warn("to use rsync due to firewall or other restrictions. This should be a") > warn("temporary problem unless complications exist with your network") > warn("(and possibly your system's filesystem) configuration.") > > >class RsyncConnection(RsyncBaseConnection): > def __init__(self, src, dest, settings): > self.transtype = "rsync" > self.syncuri = src > self.portdir = dest > self.settings = settings > self.synced = False > > def setup(self): > self.base_setup() > try: > timeout=int(self.settings["RSYNC_TIMEOUT"]) > except: > timeout = 180 > self.rsyncCmd = self.rsyncCmd+" --timeout="+str(timeout) > try: > self.maxretries = int(self.settings["RSYNC_RETRIES"]) > except: > self.maxretries = 3 > > > def sync(self): > # reading timestamps > if self.synced: > raise Exception("This method may be called only once") > > servertimestampdir = self.settings["PORTAGE_TMPDIR"]+"/sync/" > > content = portage.grabfile(self.portdir+"/metadata/timestamp.chk") > if content: > localtimestamp = time.mktime(time.strptime(content[0], "%a, %d %b %Y %H:%M:%S +0000")) > else: > localtimestamp = 0 > > if not os.path.exists(servertimestampdir): > os.mkdir(servertimestampdir) > os.chown(servertimestampdir, os.getuid(), portage.portage_gid) > os.chmod(servertimestampdir, 06775) > > if os.path.exists(servertimestampdir+"/timestamp.chk"): > os.unlink(servertimestampdir+"/timestamp.chk") > > retries = 0 > hostname = re.split("rsync://([^/]*)", self.syncuri)[1]; > updatecache = True > for i in range(0, self.maxretries): > try: > ip = socket.gethostbyname(hostname) > dosyncuri = string.replace(self.syncuri, "//"+hostname+"/", "//"+ip+"/", 1) > except Exception, e: > msg("Notice: %s" % str(e)) > dosyncuri = self.syncuri > > if retries == 0: > info(">>> starting rsync with %s [%s] ..." % (self.syncuri, ip)) > else: > info("\n\n>>> Starting retry %d of %d" % (retries, self.maxretries)) > > # fetching the timestamp first > msg(">>> checking server timestamp ...") > chkcommand = self.rsyncCmd+" "+dosyncuri+"/metadata/timestamp.chk "+servertimestampdir > exitcode = portage.spawn(chkcommand, self.settings, free=1) > > if exitcode == 0: > try: > servertimestamp = time.mktime(time.strptime( \ > portage.grabfile(servertimestampdir+"/timestamp.chk")[0], \ > "%a, %d %b %Y %H:%M:%S +0000")) > except: > servertimestamp = 0 > > if (servertimestamp != 0) and (servertimestamp == localtimestamp): > msg(">>> cancelling sync because this server timestamp is the same as local timestamp\n") > updatecache = False > break > elif (servertimestamp != 0) and (servertimestamp < localtimestamp): > msg(">>> skipping because this server timestamp is older than local timestamp ...\n") > elif (servertimestamp == 0) or (servertimestamp > localtimestamp): > # here's the actual sync > synccommand = self.rsyncCmd+" "+dosyncuri+"/* "+self.portdir > exitcode = portage.spawn(synccommand,self.settings,free=1) > self.synced = True > if exitcode in [0,1,2,3,4,11,14,20,21]: > break > elif exitcode in [0,1,2,3,4,11,14,20,21]: > break > > retries = retries+1 > > if retries <= self.maxretries: > msg(">>> retry ...") > time.sleep(11) > else: > # over retries > # exit loop > updatecache = False > break > > self.checkRsyncExitcode(exitcode) > if updatecache: > exitcode = exitcode + 128 > return exitcode > >class SnapshotConnection(RsyncBaseConnection): > def __init__(self, src, dest, settings): > self.transtype = "snapshot" > self.syncuri = src > self.portdir = dest > self.settings = settings > self.synced = False > > > def setup(self): > self.base_setup() > > > def sync(self): > # this is a rewrite of the old emerge-webrsync code > if self.synced: > raise Exception("This method may be called only once") > > attempts = 0 > downloaded = False > minsize = 1024*1024 > syncdir = self.settings["PORTAGE_TMPDIR"]+"/websync" > if not os.path.exists(syncdir+"/portage"): > os.makedirs(syncdir+"/portage", 0755) > os.chdir(syncdir) > while attempts < 40 and not downloaded: > mytime = time.localtime(time.time() - (attempts*24*60*60)) > mysnapshot = "portage-"+time.strftime("%Y%m%d", mytime)+".tar.bz2" > if os.path.exists(self.settings["DISTDIR"]+"/"+mysnapshot): > #and os.lstat(mysnapshot)[6] > minsize: > info("snapshot %s found in %s" % (mysnapshot, self.settings["DISTDIR"])) > downloaded = True > break > for m in self.syncuri.split(): > myfetchcmd = self.settings["FETCHCOMMAND"]+" "+m+"/snapshots/"+mysnapshot > info("trying to download snapshot %s from %s" % (mysnapshot, m)) > if portage.spawn(myfetchcmd, free=1) == 0: > if os.path.exists(mysnapshot) and os.lstat(mysnapshot)[6] > minsize: > info("%s successfully downloaded" % mysnapshot) > downloaded = True > break > attempts += 1 > if not downloaded: > warn("could not download a complete snapshot") > return -1 > > shutil.copy2(self.settings["DISTDIR"]+"/"+mysnapshot, "./"+mysnapshot) > portage.spawn("tar xjvf "+mysnapshot, free=1) > > self.rsyncCmd=self.rsyncCmd+" \'"+syncdir+"/portage/*\' "+os.path.normpath(self.portdir)+"/" > msg(">>> starting rsync with local portage tree ...") > exitcode = portage.spawn(self.rsyncCmd,self.settings,free=1) > self.synced = True > > self.checkRsyncExitcode(exitcode) > shutil.rmtree(syncdir) > return exitcode > >transports = { > "rsync://": ["rsync",RsyncConnection], > "cvs://": ["cvs",CvsConnection], > "http://": ["snapshot",SnapshotConnection], > "ftp://": ["snapshot",SnapshotConnection] >}
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 35535
:
22067
|
22068
|
22069
|
23407
|
23408
|
23409
|
23410
|
23411