[patch] patch to fix all known bugs

websync.diff (text/plain), 14.28 KB, created by Marius Mauch (RETIRED) on 2003-10-29 09:35:05 UTC

--- bin/emerge.org	2003-10-13 09:43:38.000000000 +0200
+++ bin/emerge	2003-10-29 18:22:03.000000000 +0100
@@ -36,7 +36,7 @@
 merged=0
 params=["selective", "deep", "self", "recurse", "empty"]
 actions=[
-"clean", "config", "depclean", "help", "info",   "inject",  "prune",
+"clean", "config", "depclean", "help", "info",   "inject",  "ovsync",  "prune",
 "regen", "rsync",  "search",   "sync", "system", "unmerge", "world"
 ]
 options=[
@@ -1718,135 +1802,201 @@
 if "--debug" in myopts:
 	edebug=1
 
-if myaction in ["sync","rsync"] and (not "--help" in myopts):
+if myaction in ["sync", "rsync", "ovsync"] and (not "--help" in myopts):
 	if "--pretend" in myopts:
 		print "emerge: the \"sync\" action does not support \"--pretend.\""
 		sys.exit(1)
 
-	emergelog(" === rsync")
-	myportdir=portage.settings["PORTDIR"]
-	if myportdir[-1]=="/":
-		myportdir=myportdir[:-1]
-	if not os.path.exists(myportdir):
-		print ">>>",myportdir,"not found, creating it."
-		os.makedirs(myportdir,0755)	
-	syncuri=string.rstrip(portage.settings["SYNC"])
-	os.umask(0022)
-	if syncuri[:8]=="rsync://":
-		if not os.path.exists("/usr/bin/rsync"):
-			print "!!! /usr/bin/rsync does not exist, so rsync support is disabled."
-			print "!!! Type \"emerge net-misc/rsync\" to enable rsync support."
-			sys.exit(1)
-		mytimeout=180
-		if portage.settings.has_key("RSYNC_TIMEOUT"):
-			try:
-				mytimeout=int(portage.settings["RSYNC_TIMEOUT"])
-			except:
-				pass
-		mycommand="/usr/bin/rsync -rlptDvz --progress --stats --delete --delete-after --timeout="+str(mytimeout)+" --exclude='distfiles/*' --exclude='local/*' --exclude='packages/*' "
-		if portage.settings.has_key("RSYNC_EXCLUDEFROM"):
-			if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]):
-				mycommand=mycommand+" --exclude-from "+portage.settings["RSYNC_EXCLUDEFROM"]
-			else:
-				print "!!! RSYNC_EXCLUDEFROM specified, but file does not exist."
-		mycommand=mycommand+" "+syncuri+"/* "+myportdir
-		print ">>> starting rsync with "+syncuri+"..."
-		exitcode=portage.spawn(mycommand,free=1)
-		#exitcode=0
-		try:
-			maxretries=int(portage.settings["RSYNC_RETRIES"])
-		except:
-			maxretries=3 #default number of retries
-		retries=1
-		while (exitcode not in [0,1,2,3,4,11,14,20,21]) and (retries<=maxretries):
-			print "\n\n>>> Starting retry %d of %d"% (retries,maxretries)
-			time.sleep(11)
-			retries=retries+1
-			exitcode=portage.spawn(mycommand,free=1)
-		if (exitcode>0):
-			print
-			if exitcode==1:
-				print darkred("!!!")+green(" Rsync has reported that there is a syntax error. Please ensure")
-				print darkred("!!!")+green(" that your SYNC statement is proper.")
-				print darkred("!!!")+green(" SYNC="+portage.settings["SYNC"])
-			elif exitcode==11:
-				print darkred("!!!")+green(" Rsync has reported that there is a File IO error. Normally")
-				print darkred("!!!")+green(" this means your disk is full, but can be caused by corruption")
-				print darkred("!!!")+green(" on the filesystem that contains PORTDIR. Please investigate")
-				print darkred("!!!")+green(" and try again after the problem has been fixed.")
-				print darkred("!!!")+green(" PORTDIR="+portage.settings["PORTDIR"])
-			elif exitcode==20:
-				print darkred("!!!")+green(" Rsync was killed before it finished.")
-			else:
-				print darkred("!!!")+green(" Rsync has not successfully finished. It is recommended that you keep")
-				print darkred("!!!")+green(" trying or that you use the 'emerge-webrsync' option if you are unable")
-				print darkred("!!!")+green(" to use rsync due to firewall or other restrictions. This should be a")
-				print darkred("!!!")+green(" temporary problem unless complications exist with your network")
-				print darkred("!!!")+green(" (and possibly your system's filesystem) configuration.")
-			print
-			sys.exit(exitcode)
-	elif syncuri[:6]=="cvs://":
-		if not os.path.exists("/usr/bin/cvs"):
-			print "!!! /usr/bin/cvs does not exist, so rsync support is disabled."
-			print "!!! Type \"emerge dev-util/cvs\" to enable CVS support."
-			sys.exit(1)
-		cvsroot=syncuri[6:]
-		cvsdir=os.path.dirname(myportdir)
-		if not os.path.exists(myportdir+"/CVS"):
-			#initial checkout
-			print ">>> starting initial cvs checkout with "+syncuri+"..."
-			if not portage.spawn("cd "+cvsdir+"; cvs -d "+cvsroot+" login",free=1):
-				print "!!! cvs login error; exiting."
-				sys.exit(1)
-			if os.path.exists(cvsdir+"/gentoo-x86"):
-				print "!!! existing",cvsdir+"/gentoo-x86 directory; exiting."
-				sys.exit(1)
-			if not portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+cvsroot+" co gentoo-x86",free=1):
-				print "!!! cvs checkout error; exiting."
-				sys.exit(1)
-			if cvsdir!=myportdir:
-				portage.movefile(cvsdir,portage.settings["PORTDIR"])
-			sys.exit(0)
-		else:
-			#cvs update
-			print ">>> starting cvs update with "+syncuri+"..."
-			sys.exit(portage.spawn("cd "+myportdir+"; cvs -z0 -q update -dP",free=1)) 
+	emergelog(" === sync")
+	
+	if myaction == "ovsync":
+		mytrees=[]
+		overlay_dirs = portage.settings["PORTDIR_OVERLAY"].split()
+		overlay_uris = portage.settings["SYNC_OVERLAY"].split()
+		for i in range(0, min(len(overlay_dirs), len(overlay_uris))):
+			if not overlay_uris[i] in ["none", "local"]:
+				mytrees.append((overlay_dirs[i], overlay_uris[i]))
 	else:
-		print "!!! rsync setting: ",syncuri,"not recognized; exiting."
-		sys.exit(1)
-	if os.path.exists(myportdir+"/metadata/cache"):
-		print "\n>>> Updating Portage cache...  ",
-		os.umask(0002)
-		portage.db["/"]["porttree"].dbapi.flush_auxcache()
-		if os.path.exists(portage.dbcachedir):
-			portage.spawn("rm -Rf "+portage.dbcachedir,free=1)
-		try:
-			os.mkdir(portage.dbcachedir)
-			os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
-			os.chmod(portage.dbcachedir, 06775)
-			os.umask(002)
-		except:
-			pass
-		mynodes=portage.portdb.cp_all()
-		portage.eclassdb["modifications_limit"] = 1000
-		for x in mynodes:
-			myxsplit=x.split("/")
-			if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
-				os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
-				os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
-				os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
-			mymatches=portage.portdb.xmatch("match-all",x)
-			for y in mymatches:
-				update_spinner()
+		mytrees = [(portage.settings["PORTDIR"], portage.settings["SYNC"])]
+	
+	if not mytrees:
+		print
+		print red("*"), "nothing to sync"
+		print
+		sys.exit(0)
+			
+	for (myportdir, mysyncuri) in mytrees:
+		emergelog("syncing "+mysyncuri+" to "+myportdir)
+		if myportdir[-1]=="/":
+			myportdir=myportdir[:-1]
+		if not os.path.exists(myportdir):
+			print ">>>",myportdir,"not found, creating it."
+			os.makedirs(myportdir,0755)	
+		os.umask(0022)
+		if mysyncuri[:8]=="rsync://" or mysyncuri[:7] == "http://" or mysyncuri[:6] == "ftp://":
+			if not os.path.exists("/usr/bin/rsync"):
+				print "!!! /usr/bin/rsync does not exist, so rsync support is disabled."
+				print "!!! Type \"emerge net-misc/rsync\" to enable rsync support."
+				sys.exit(1)
+			mytimeout=180
+			if portage.settings.has_key("RSYNC_TIMEOUT"):
 				try:
-					ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
+					mytimeout=int(portage.settings["RSYNC_TIMEOUT"])
 				except:
 					pass
-		portage.eclassdb["modifications_limit"] = 1
-		portage.save_eclassdb(forced=1)
-		portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1)
-		sys.stdout.write("\b\b  ...done!\n\n")
-		sys.stdout.flush()
+			mycommand="/usr/bin/rsync -rlptDvz --progress --stats --delete --delete-after --timeout="+str(mytimeout)+" --exclude='distfiles/*' --exclude='local/*' --exclude='packages/*' "
+			if portage.settings.has_key("RSYNC_EXCLUDEFROM"):
+				if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]):
+					mycommand=mycommand+" --exclude-from "+portage.settings["RSYNC_EXCLUDEFROM"]
+				else:
+					print "!!! RSYNC_EXCLUDEFROM specified, but file does not exist."
+
+			# this is a rewrite of the old emerge-webrsync code
+			if mysyncuri[:7] == "http://" or mysyncuri[:6] == "ftp://":
+				attempts = 0
+				downloaded = False
+				minsize=1024*1024
+				while attempts < 40 and not downloaded:
+					mytime = time.localtime(time.time() - (attempts*24*60*60))
+					mysnapshot = "portage-"+time.strftime("%Y%m%d", mytime)+".tar.bz2"
+					syncdir=portage.settings["PORTAGE_TMPDIR"]+"/websync"
+					for m in mysyncuri.split():
+						if not os.path.exists(syncdir):
+							os.mkdir(syncdir)
+						os.chdir(syncdir)
+						myfetchcmd=portage.settings["FETCHCOMMAND"]+" "+m+"/snapshots/"+mysnapshot
+						print yellow("*"), "trying to download snapshot "+mysnapshot+" from "+m
+						if portage.spawn(myfetchcmd, free=1) == 0:
+							shutil.copy2(portage.settings["DISTDIR"]+"/"+mysnapshot, "./"+mysnapshot)
+							if os.path.exists(mysnapshot) and os.lstat(mysnapshot)[6] > minsize:
+								print green("*"), mysnapshot+" successfully downloaded"
+								downloaded = True
+								break
+					attempts += 1
+				if not downloaded:
+					print red("*"), "could not download a complete snapshot"
+					sys.exit(1)
+				
+				portage.spawn("tar xjf "+mysnapshot, free=1)
+				mycommand=mycommand+" "+syncdir+"/portage/* "+myportdir
+				print ">>> starting rsync with local portage tree ..."
+			else:
+				mycommand=mycommand+" "+mysyncuri+"/* "+myportdir
+				print ">>> starting rsync with "+mysyncuri+"..."
+			exitcode=portage.spawn(mycommand,free=1)
+			#exitcode=0
+			try:
+				maxretries=int(portage.settings["RSYNC_RETRIES"])
+			except:
+				maxretries=3 #default number of retries
+			retries=1
+			while (exitcode not in [0,1,2,3,4,11,14,20,21]) and (retries<=maxretries):
+				print "\n\n>>> Starting retry %d of %d"% (retries,maxretries)
+				time.sleep(11)
+				retries=retries+1
+				exitcode=portage.spawn(mycommand,free=1)
+			if (exitcode>0):
+				print
+				if exitcode==1:
+					print darkred("!!!")+green(" Rsync has reported that there is a syntax error. Please ensure")
+					print darkred("!!!")+green(" that your SYNC statement is proper.")
+					print darkred("!!!")+green(" SYNC="+portage.settings["SYNC"])
+				elif exitcode==11:
+					print darkred("!!!")+green(" Rsync has reported that there is a File IO error. Normally")
+					print darkred("!!!")+green(" this means your disk is full, but can be caused by corruption")
+					print darkred("!!!")+green(" on the filesystem that contains PORTDIR. Please investigate")
+					print darkred("!!!")+green(" and try again after the problem has been fixed.")
+					print darkred("!!!")+green(" PORTDIR="+portage.settings["PORTDIR"])
+				elif exitcode==20:
+					print darkred("!!!")+green(" Rsync was killed before it finished.")
+				else:
+					print darkred("!!!")+green(" Rsync has not successfully finished. It is recommended that you keep")
+					print darkred("!!!")+green(" trying or that you use the 'emerge-webrsync' option if you are unable")
+					print darkred("!!!")+green(" to use rsync due to firewall or other restrictions. This should be a")
+					print darkred("!!!")+green(" temporary problem unless complications exist with your network")
+					print darkred("!!!")+green(" (and possibly your system's filesystem) configuration.")
+				print
+				sys.exit(exitcode)
+			# cleanup
+			if mysyncuri[:7] == "http://" or mysyncuri[:6] == "ftp://":
+				shutil.rmtree(syncdir)
+		elif mysyncuri[:6]=="cvs://":
+			if not os.path.exists("/usr/bin/cvs"):
+				print "!!! /usr/bin/cvs does not exist, so rsync support is disabled."
+				print "!!! Type \"emerge dev-util/cvs\" to enable CVS support."
+				sys.exit(1)
+			cvsroot=mysyncuri[6:]
+			if cvsroot.count(":") < 2:
+				print yellow("*"), "The implicit gentoo-x86 module is deprecated"
+				print yellow("*"), "Please change your make.conf to the new"
+				print yellow("*"), "    user@server:cvsroot:cvsmodule"
+				print yellow("*"), "syntax. Thanks."
+				cvsmodule="gentoo-x86"
+			else:
+				mpos=cvsroot.rindex(":")
+				cvsmodule=cvsroot[mpos+1:]
+				cvsroot=cvsroot[:mpos]
+			cvsdir=os.path.dirname(myportdir)
+			if not os.path.exists(myportdir+"/CVS"):
+				#initial checkout
+				print ">>> starting initial cvs checkout with "+mysyncuri+"..."
+				if not portage.spawn("cd "+cvsdir+"; cvs -d "+cvsroot+" login",free=1):
+					print "!!! cvs login error; exiting."
+					sys.exit(1)
+				if os.path.exists(cvsdir+"/"+cvsmodule):
+					print "!!! existing",cvsdir+"/"+cvsmodule+" directory; exiting."
+					sys.exit(1)
+				if not portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+cvsroot+" co "+cvsmodule,free=1):
+					print "!!! cvs checkout error; exiting."
+					sys.exit(1)
+				if cvsdir!=myportdir:
+					portage.movefile(cvsdir,myportdir)
+				#sys.exit(0)
+			else:
+				#cvs update
+				print ">>> starting cvs update with "+mysyncuri+"..."
+				rValue=portage.spawn("cd "+myportdir+"; cvs -z0 -q update -dP",free=1)
+				if rValue:
+					sys.exit(rValue) 
+		else:
+			print "!!! sync setting: ",mysyncuri,"not recognized; exiting."
+			sys.exit(1)
+		if os.path.exists(myportdir+"/metadata/cache"):
+			print "\n>>> Updating Portage cache...  ",
+			os.umask(0002)
+			portage.db["/"]["porttree"].dbapi.flush_auxcache()
+			if os.path.exists(portage.dbcachedir):
+				portage.spawn("rm -Rf "+portage.dbcachedir,free=1)
+			try:
+				os.mkdir(portage.dbcachedir)
+				os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
+				os.chmod(portage.dbcachedir, 06775)
+				os.umask(002)
+			except:
+				pass
+				mynodes=portage.portdb.cp_all()
+				portage.eclassdb["modifications_limit"] = 1000
+				for x in mynodes:
+					myxsplit=x.split("/")
+					if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
+						os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
+					os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
+					os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
+				mymatches=portage.portdb.xmatch("match-all",x)
+				for y in mymatches:
+					update_spinner()
+					try:
+						ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
+					except:
+						pass
+			portage.eclassdb["modifications_limit"] = 1
+			portage.save_eclassdb(forced=1)
+			portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1)
+			sys.stdout.write("\b\b  ...done!\n\n")
+			sys.stdout.flush()
+	if "fastsearch" in portage.features:
+		print "\n>>> Building fastsearch database ..."
+		portage.spawn("/usr/lib/portage/bin/fastsearch -u", free=1)
 
 	portage.portageexit()
 	reload(portage)