[patch] patch for emerge

emerge-sync.diff (text/plain), 11.48 KB, created by Marius Mauch (RETIRED) on 2003-12-11 20:16:16 UTC

--- bin/emerge.org	2003-12-11 22:32:49.000000000 +0100
+++ bin/emerge	2003-12-12 05:12:53.000000000 +0100
@@ -7,11 +7,13 @@
 os.environ["PORTAGE_CALLER"]="emerge"
 sys.path = ["/usr/lib/portage/pym"]+sys.path
 
-import emergehelp,xpak,string,re,commands,time,shutil,traceback,atexit,signal,socket,types
+import emergehelp,xpak,string,re,commands,time,traceback,atexit,signal,types
 from stat import *
 from output import *
 
 import portage
+import sync
+
 if (not sys.stdout.isatty()) or (portage.settings["NOCOLOR"] in ["yes","true"]):
 	nocolor()
 
@@ -1826,206 +1828,92 @@
 if "--debug" in myopts:
 	edebug=1
 
-if myaction in ["sync","rsync"] and (not "--help" in myopts):
-	if "--pretend" in myopts:
-		print "emerge: the \"sync\" action does not support \"--pretend.\""
-		sys.exit(1)
 
-	emergelog(" === rsync")
-	myportdir=portage.settings["PORTDIR"]
-	if myportdir[-1]=="/":
-		myportdir=myportdir[:-1]
-	if not os.path.exists(myportdir):
-		print ">>>",myportdir,"not found, creating it."
-		os.makedirs(myportdir,0755)	
-	syncuri=string.rstrip(portage.settings["SYNC"])
-	os.umask(0022)
-	if syncuri[:8]=="rsync://":
-		if not os.path.exists("/usr/bin/rsync"):
-			print "!!! /usr/bin/rsync does not exist, so rsync support is disabled."
-			print "!!! Type \"emerge net-misc/rsync\" to enable rsync support."
-			sys.exit(1)
-		mytimeout=180
-		if portage.settings.has_key("RSYNC_TIMEOUT"):
+def updatecache(myportdir):
+	print "\n>>> Updating Portage cache with %s...  " % myportdir, 
+	os.umask(0002)
+	portage.db["/"]["porttree"].dbapi.flush_auxcache()
+	if os.path.exists(portage.dbcachedir):
+		portage.spawn("rm -Rf "+portage.dbcachedir,portage.settings,free=1)
+	try:
+		os.mkdir(portage.dbcachedir)
+		os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
+		os.chmod(portage.dbcachedir, 06775)
+		os.umask(002)
+	except:
+		pass
+	mynodes=portage.portdb.cp_all()
+	portage.eclassdb["modifications_limit"] = 1000
+	for x in mynodes:
+		myxsplit=x.split("/")
+		if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
+			os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
+			os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
+			os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
+		mymatches=portage.portdb.xmatch("match-all",x)
+		for y in mymatches:
+			update_spinner()
 			try:
-				mytimeout=int(portage.settings["RSYNC_TIMEOUT"])
+				ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
 			except:
 				pass
-		rsynccommand="/usr/bin/rsync -rlptDvz --progress --stats --delete --delete-after --timeout="+str(mytimeout)+" --exclude='distfiles/*' --exclude='local/*' --exclude='packages/*' "
-		if portage.settings.has_key("RSYNC_EXCLUDEFROM"):
-			if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]):
-				rsynccommand=rsynccommand+" --exclude-from "+portage.settings["RSYNC_EXCLUDEFROM"]
-			else:
-				print "!!! RSYNC_EXCLUDEFROM specified, but file does not exist."
-		if portage.settings.has_key("RSYNC_RATELIMIT"):
-			rsynccommand = rsynccommand+ " --bwlimit="+portage.settings["RSYNC_RATELIMIT"]+" "
-
-		servertimestampdir=portage.settings["PORTAGE_TMPDIR"]+"/sync/"
-
-		content=portage.grabfile(myportdir+"/metadata/timestamp.chk")
-		if (content):
-			mytimestamp=time.mktime(time.strptime(content[0], "%a, %d %b %Y %H:%M:%S +0000"))
-		else:
-			mytimestamp=0
-
-		if not os.path.exists(servertimestampdir):
-			os.mkdir(servertimestampdir)
-			os.chown(servertimestampdir, os.getuid(), portage.portage_gid)
-			os.chmod(servertimestampdir, 06775)
-
-		if os.path.exists(servertimestampdir+"/timestamp.chk"):
-			os.unlink(servertimestampdir+"/timestamp.chk")
-
-		#exitcode=0
-		try:
-			maxretries=int(portage.settings["RSYNC_RETRIES"])
-		except:
-			maxretries=3 #default number of retries
-
-		retries=0
-		hostname=re.split("rsync://([^/]*)",syncuri)[1];
-		updatecache_flg=True
-		while (1):
-			try:
-				ip=socket.gethostbyname(hostname)
-				dosyncuri=string.replace(syncuri, "//"+hostname+"/", "//"+ip+"/", 1)
-			except Exception, e:
-				print "Notice:",str(e)
-				dosyncuri = syncuri
-
-			if (retries==0):
-				print ">>> starting rsync with "+syncuri+"..."
-			else:
-				print "\n\n>>> Starting retry %d of %d"% (retries,maxretries)
-
-			print ">>> checking server timestamp ..."
-			mycommand=rsynccommand+" "+dosyncuri+"/metadata/timestamp.chk "+servertimestampdir
-			exitcode=portage.spawn(mycommand,portage.settings,free=1)
-			if (exitcode==0):
-				try:
-					servertimestamp = time.mktime(time.strptime(portage.grabfile(servertimestampdir+"/timestamp.chk")[0], "%a, %d %b %Y %H:%M:%S +0000"))
-				except:
-					servertimestamp = 0
-				
-				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
-					print ">>> cancelling sync because this server timestamp is the same as local timestamp"
-					print
-					updatecache_flg=False
-					break
-				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
-					print ">>> skipping because this server timestamp is older than local timestamp ..."
-					print
-				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
-					# actual sync
-					mycommand=rsynccommand+" "+dosyncuri+"/* "+myportdir
-					exitcode=portage.spawn(mycommand,portage.settings,free=1)
-					if exitcode in [0,1,2,3,4,11,14,20,21]:
-						break
-			elif exitcode in [0,1,2,3,4,11,14,20,21]:
-				break
-
-			retries=retries+1
+	portage.eclassdb["modifications_limit"] = 1
+	portage.save_eclassdb(forced=1)
+	portage.spawn("chmod -R g+rw "+portage.dbcachedir, portage.settings, free=1)
+	sys.stdout.write("\b\b  ...done!\n\n")
+	sys.stdout.flush()
 
-			if retries<=maxretries:
-				print ">>> retry ..."
-				time.sleep(11)
-			else:
-				# over retries
-				# exit loop
-				updatecache_flg=False
-				break
 
-		if (exitcode>0):
-			print
-			if exitcode==1:
-				print darkred("!!!")+green(" Rsync has reported that there is a syntax error. Please ensure")
-				print darkred("!!!")+green(" that your SYNC statement is proper.")
-				print darkred("!!!")+green(" SYNC="+portage.settings["SYNC"])
-			elif exitcode==11:
-				print darkred("!!!")+green(" Rsync has reported that there is a File IO error. Normally")
-				print darkred("!!!")+green(" this means your disk is full, but can be caused by corruption")
-				print darkred("!!!")+green(" on the filesystem that contains PORTDIR. Please investigate")
-				print darkred("!!!")+green(" and try again after the problem has been fixed.")
-				print darkred("!!!")+green(" PORTDIR="+portage.settings["PORTDIR"])
-			elif exitcode==20:
-				print darkred("!!!")+green(" Rsync was killed before it finished.")
-			else:
-				print darkred("!!!")+green(" Rsync has not successfully finished. It is recommended that you keep")
-				print darkred("!!!")+green(" trying or that you use the 'emerge-webrsync' option if you are unable")
-				print darkred("!!!")+green(" to use rsync due to firewall or other restrictions. This should be a")
-				print darkred("!!!")+green(" temporary problem unless complications exist with your network")
-				print darkred("!!!")+green(" (and possibly your system's filesystem) configuration.")
-			print
-			sys.exit(exitcode)
-	elif syncuri[:6]=="cvs://":
-		if not os.path.exists("/usr/bin/cvs"):
-			print "!!! /usr/bin/cvs does not exist, so rsync support is disabled."
-			print "!!! Type \"emerge dev-util/cvs\" to enable CVS support."
+if myaction == "sync" and (not "--help" in myopts):
+	if "--pretend" in myopts:
+		print "emerge: the \"sync\" action does not support \"--pretend.\""
+		sys.exit(1)
+	
+	src = []
+	dest = []
+	
+	# check mode
+	if not myfiles or "all" in myfiles:
+		src.append(portage.settings["SYNC"])
+		dest.append(portage.settings["PORTDIR"])
+		print ">>> adding PORTDIR to sync queue"
+	if myfiles:
+		overlayfile = portage.grabdict("/etc/portage/overlays")
+		if not overlayfile:
+			print red("*"), "Repository sync requested, but no overlay specifications found in"
+			print red("*"), "    /etc/portage/overlays"
 			sys.exit(1)
-		cvsroot=syncuri[6:]
-		cvsdir=os.path.dirname(myportdir)
-		if not os.path.exists(myportdir+"/CVS"):
-			#initial checkout
-			print ">>> starting initial cvs checkout with "+syncuri+"..."
-			if not portage.spawn("cd "+cvsdir+"; cvs -d "+cvsroot+" login",portage.settings,free=1):
-				print "!!! cvs login error; exiting."
-				sys.exit(1)
-			if os.path.exists(cvsdir+"/gentoo-x86"):
-				print "!!! existing",cvsdir+"/gentoo-x86 directory; exiting."
-				sys.exit(1)
-			if not portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+cvsroot+" co gentoo-x86",portage.settings,free=1):
-				print "!!! cvs checkout error; exiting."
-				sys.exit(1)
-			if cvsdir!=myportdir:
-				portage.movefile(cvsdir,portage.settings["PORTDIR"])
-			sys.exit(0)
+		if "all" in myfiles:
+			repolist = overlayfile.keys()
 		else:
-			#cvs update
-			print ">>> starting cvs update with "+syncuri+"..."
-			sys.exit(portage.spawn("cd "+myportdir+"; cvs -z0 -q update -dP",portage.settings,free=1)) 
-	else:
-		print "!!! rsync setting: ",syncuri,"not recognized; exiting."
-		sys.exit(1)
-	if os.path.exists(myportdir+"/metadata/cache") and updatecache_flg:
-		print "\n>>> Updating Portage cache...  ",
-		os.umask(0002)
-		portage.db["/"]["porttree"].dbapi.flush_auxcache()
-		if os.path.exists(portage.dbcachedir):
-			portage.spawn("rm -Rf "+portage.dbcachedir,portage.settings,free=1)
-		try:
-			os.mkdir(portage.dbcachedir)
-			os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
-			os.chmod(portage.dbcachedir, 06775)
-			os.umask(002)
-		except:
-			pass
-		mynodes=portage.portdb.cp_all()
-		portage.eclassdb["modifications_limit"] = 1000
-		for x in mynodes:
-			myxsplit=x.split("/")
-			if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
-				os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
-				os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
-				os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
-			mymatches=portage.portdb.xmatch("match-all",x)
-			for y in mymatches:
-				update_spinner()
-				try:
-					ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
-				except:
-					pass
-		portage.eclassdb["modifications_limit"] = 1
-		portage.save_eclassdb(forced=1)
-		portage.spawn("chmod -R g+rw "+portage.dbcachedir, portage.settings, free=1)
-		sys.stdout.write("\b\b  ...done!\n\n")
-		sys.stdout.flush()
+			repolist = myfiles
+		for repo in repolist:
+			if repo in overlayfile.keys():
+				if len(overlayfile[repo]) < 2:
+					print yellow("*"), "Repository \"%s\" found in overlay file, but is missing elements" % repo
+					print yellow("*"), "Please check /etc/portage/overlays for errors.\n"
+					print yellow("*"), "Skipping repository \"%s\"" % repo
+					continue
+				src.append(overlayfile[repo][0])
+				dest.append(overlayfile[repo][1])
+				print ">>> adding repository \"%s\" to sync queue" % repo
+
+	for i in range(0, len(src)):
+		link = sync.connection.create(src[i], dest[i], portage.settings)
+		link.setup()
+		rValue = link.sync()
+		if rValue not in [0, 128]:
+			sys.exit(rValue)
+	
+		if os.path.exists(dest[i]+"/metadata/cache") and (rValue & 128):
+			updatecache(dest[i])
 
 	portage.portageexit()
 	reload(portage)
 	mybestpv=portage.portdb.xmatch("bestmatch-visible","sys-apps/portage")
 	mypvs=portage.best(portage.db[portage.root]["vartree"].dbapi.match("sys-apps/portage"))
 
-	if(mybestpv != mypvs):
+	if (mybestpv != mypvs):
 		print
 		print red(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended"
 		print red(" * ")+"that you update portage now, before any other packages are updated."