rsync module

rsync.py (text/plain), 5.13 KB, created by Marius Mauch (RETIRED) on 2004-01-08 12:10:20 UTC

from aux import *	# to be removed
import os,time,re,portage,socket,string
import sync

class RsyncBaseConnection(sync.Connection):
	options = ["-rlptDvz", "--progress", "--stats", "--delete", \
			   "--delete-after", "--exclude='distfiles/*'", \
			   "--exclude='local/*'", "--exclude='packages/*'"]

	def __init__(self, src, dest, settings):
		self.syncuri = src
		self.portdir = dest
		self.settings = settings


	def base_setup(self):
		if not self.toolcheck("/usr/bin/rsync", "net-misc/rsync", quiet=False):
			return -1
			
		self.rsyncCmd = "/usr/bin/rsync"

		# add general options
		for op in self.options:
			self.rsyncCmd = self.rsyncCmd+" "+op
		
		if self.settings.has_key("RSYNC_EXCLUDEFROM"):	
			if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]):
				self.rsyncCmd = self.rsyncCmd + " --exclude-from " + self.settings["RSYNC_EXCLUDEFROM"]
			else:
				warn("RSYNC_EXCLUDEFROM specified, but file does not exist.")
		

	def checkRsyncExitcode(self, exitcode):
		if exitcode == 1:
			warn("Rsync has reported that there is a syntax error. Please ensure")
			warn("that your sync statement is proper.")
			warn("(sync setting is: %s" % self.syncuri)
		elif exitcode == 11:
			warn("Rsync has reported that there is a File IO error. Normally")
			warn("this means your disk is full, but can be caused by corruption")
			warn("on the filesystem that contains the target directory. Please")
			warn("investigate and try again after the problem has been fixed.")
			warn("(target directory is: %s" % self.portdir)
		elif exitcode == 20:
			warn("Rsync was killed before it finished.")
		elif exitcode > 0:
			warn("Rsync has not successfully finished. It is recommended that you keep")
			warn("trying or that you use the 'emerge-webrsync' option if you are unable")
			warn("to use rsync due to firewall or other restrictions. This should be a")
			warn("temporary problem unless complications exist with your network")
			warn("(and possibly your system's filesystem) configuration.")
	

class RsyncConnection(RsyncBaseConnection):
	def __init__(self, src, dest, settings):
		self.transtype = "rsync"
		self.syncuri = src
		self.portdir = dest
		self.settings = settings
		self.synced = False

	def setup(self):
		self.base_setup()
		try:
			timeout=int(self.settings["RSYNC_TIMEOUT"])
		except:
			timeout = 180
		self.rsyncCmd = self.rsyncCmd+" --timeout="+str(timeout)
		try:
			self.maxretries = int(self.settings["RSYNC_RETRIES"])
		except:
			self.maxretries = 3


	def sync(self):
		# reading timestamps
		if self.synced:
			raise Exception("This method may be called only once")

		servertimestampdir = self.settings["PORTAGE_TMPDIR"]+"/sync/"

		content = portage.grabfile(self.portdir+"/metadata/timestamp.chk")
		if content:
			localtimestamp = time.mktime(time.strptime(content[0], "%a, %d %b %Y %H:%M:%S +0000"))
		else:
			localtimestamp = 0

		if not os.path.exists(servertimestampdir):
			os.mkdir(servertimestampdir)
			os.chown(servertimestampdir, os.getuid(), portage.portage_gid)
			os.chmod(servertimestampdir, 06775)

		if os.path.exists(servertimestampdir+"/timestamp.chk"):
			os.unlink(servertimestampdir+"/timestamp.chk")

		retries = 0
		hostname = re.split("rsync://([^/]*)", self.syncuri)[1];
		updatecache = True
		for i in range(0, self.maxretries):
			try:
				ip = socket.gethostbyname(hostname)
				dosyncuri = string.replace(self.syncuri, "//"+hostname+"/", "//"+ip+"/", 1)
			except Exception, e:
				ip = hostname
				msg("Notice: %s" % str(e))
				dosyncuri = self.syncuri

			if retries == 0:
				info(">>> starting rsync with %s [%s] ..." % (self.syncuri, ip))
			else:
				info("\n\n>>> Starting retry %d of %d" % (retries, self.maxretries))

			# fetching the timestamp first
			msg(">>> checking server timestamp ...")
			chkcommand = self.rsyncCmd+" "+dosyncuri+"/metadata/timestamp.chk "+servertimestampdir
			exitcode = portage.spawn(chkcommand, self.settings, free=1)

			if exitcode == 0:
				try:
					servertimestamp = time.mktime(time.strptime( \
						portage.grabfile(servertimestampdir+"/timestamp.chk")[0], \
										"%a, %d %b %Y %H:%M:%S +0000"))
				except:
					servertimestamp = 0
				
				if (servertimestamp != 0) and (servertimestamp == localtimestamp):
					msg(">>> cancelling sync because this server timestamp is the same as local timestamp\n")
					updatecache = False
					break
				elif (servertimestamp != 0) and (servertimestamp < localtimestamp):
					msg(">>> skipping because this server timestamp is older than local timestamp ...\n")
				elif (servertimestamp == 0) or (servertimestamp > localtimestamp):
					# here's the actual sync
					synccommand = self.rsyncCmd+" "+dosyncuri+"/* "+self.portdir
					exitcode = portage.spawn(synccommand,self.settings,free=1)
					self.synced = True
					if exitcode in [0,1,2,3,4,11,14,20,21]:
						break
			elif exitcode in [0,1,2,3,4,11,14,20,21]:
				break

			retries = retries+1

			if retries <= self.maxretries:
				msg(">>> retry ...")
				time.sleep(11)
			else:
				# over retries
				# exit loop
				updatecache = False
				break

		self.checkRsyncExitcode(exitcode)
		if updatecache:
			exitcode = exitcode + 128
		return exitcode

print "Loading rsync"
sync.addTransport("rsync", RsyncConnection)