Gentoo Websites Logo
Go to: Gentoo Home Documentation Forums Lists Bugs Planet Store Wiki Get Gentoo!
View | Details | Raw Unified | Return to bug 35535 | Differences between
and this patch

Collapse All | Expand All

(-)bin/emerge.org (-187 / +75 lines)
Lines 7-17 Link Here
7
os.environ["PORTAGE_CALLER"]="emerge"
7
os.environ["PORTAGE_CALLER"]="emerge"
8
sys.path = ["/usr/lib/portage/pym"]+sys.path
8
sys.path = ["/usr/lib/portage/pym"]+sys.path
9
9
10
import emergehelp,xpak,string,re,commands,time,shutil,traceback,atexit,signal,socket,types
10
import emergehelp,xpak,string,re,commands,time,traceback,atexit,signal,types
11
from stat import *
11
from stat import *
12
from output import *
12
from output import *
13
13
14
import portage
14
import portage
15
import sync
16
15
if (not sys.stdout.isatty()) or (portage.settings["NOCOLOR"] in ["yes","true"]):
17
if (not sys.stdout.isatty()) or (portage.settings["NOCOLOR"] in ["yes","true"]):
16
	nocolor()
18
	nocolor()
17
19
Lines 1826-2031 Link Here
1826
if "--debug" in myopts:
1828
if "--debug" in myopts:
1827
	edebug=1
1829
	edebug=1
1828
1830
1829
if myaction in ["sync","rsync"] and (not "--help" in myopts):
1830
	if "--pretend" in myopts:
1831
		print "emerge: the \"sync\" action does not support \"--pretend.\""
1832
		sys.exit(1)
1833
1831
1834
	emergelog(" === rsync")
1832
def updatecache(myportdir):
1835
	myportdir=portage.settings["PORTDIR"]
1833
	print "\n>>> Updating Portage cache with %s...  " % myportdir, 
1836
	if myportdir[-1]=="/":
1834
	os.umask(0002)
1837
		myportdir=myportdir[:-1]
1835
	portage.db["/"]["porttree"].dbapi.flush_auxcache()
1838
	if not os.path.exists(myportdir):
1836
	if os.path.exists(portage.dbcachedir):
1839
		print ">>>",myportdir,"not found, creating it."
1837
		portage.spawn("rm -Rf "+portage.dbcachedir,portage.settings,free=1)
1840
		os.makedirs(myportdir,0755)	
1838
	try:
1841
	syncuri=string.rstrip(portage.settings["SYNC"])
1839
		os.mkdir(portage.dbcachedir)
1842
	os.umask(0022)
1840
		os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
1843
	if syncuri[:8]=="rsync://":
1841
		os.chmod(portage.dbcachedir, 06775)
1844
		if not os.path.exists("/usr/bin/rsync"):
1842
		os.umask(002)
1845
			print "!!! /usr/bin/rsync does not exist, so rsync support is disabled."
1843
	except:
1846
			print "!!! Type \"emerge net-misc/rsync\" to enable rsync support."
1844
		pass
1847
			sys.exit(1)
1845
	mynodes=portage.portdb.cp_all()
1848
		mytimeout=180
1846
	portage.eclassdb["modifications_limit"] = 1000
1849
		if portage.settings.has_key("RSYNC_TIMEOUT"):
1847
	for x in mynodes:
1848
		myxsplit=x.split("/")
1849
		if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
1850
			os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
1851
			os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
1852
			os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
1853
		mymatches=portage.portdb.xmatch("match-all",x)
1854
		for y in mymatches:
1855
			update_spinner()
1850
			try:
1856
			try:
1851
				mytimeout=int(portage.settings["RSYNC_TIMEOUT"])
1857
				ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
1852
			except:
1858
			except:
1853
				pass
1859
				pass
1854
		rsynccommand="/usr/bin/rsync -rlptDvz --progress --stats --delete --delete-after --timeout="+str(mytimeout)+" --exclude='distfiles/*' --exclude='local/*' --exclude='packages/*' "
1860
	portage.eclassdb["modifications_limit"] = 1
1855
		if portage.settings.has_key("RSYNC_EXCLUDEFROM"):
1861
	portage.save_eclassdb(forced=1)
1856
			if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]):
1862
	portage.spawn("chmod -R g+rw "+portage.dbcachedir, portage.settings, free=1)
1857
				rsynccommand=rsynccommand+" --exclude-from "+portage.settings["RSYNC_EXCLUDEFROM"]
1863
	sys.stdout.write("\b\b  ...done!\n\n")
1858
			else:
1864
	sys.stdout.flush()
1859
				print "!!! RSYNC_EXCLUDEFROM specified, but file does not exist."
1860
		if portage.settings.has_key("RSYNC_RATELIMIT"):
1861
			rsynccommand = rsynccommand+ " --bwlimit="+portage.settings["RSYNC_RATELIMIT"]+" "
1862
1863
		servertimestampdir=portage.settings["PORTAGE_TMPDIR"]+"/sync/"
1864
1865
		content=portage.grabfile(myportdir+"/metadata/timestamp.chk")
1866
		if (content):
1867
			mytimestamp=time.mktime(time.strptime(content[0], "%a, %d %b %Y %H:%M:%S +0000"))
1868
		else:
1869
			mytimestamp=0
1870
1871
		if not os.path.exists(servertimestampdir):
1872
			os.mkdir(servertimestampdir)
1873
			os.chown(servertimestampdir, os.getuid(), portage.portage_gid)
1874
			os.chmod(servertimestampdir, 06775)
1875
1876
		if os.path.exists(servertimestampdir+"/timestamp.chk"):
1877
			os.unlink(servertimestampdir+"/timestamp.chk")
1878
1879
		#exitcode=0
1880
		try:
1881
			maxretries=int(portage.settings["RSYNC_RETRIES"])
1882
		except:
1883
			maxretries=3 #default number of retries
1884
1885
		retries=0
1886
		hostname=re.split("rsync://([^/]*)",syncuri)[1];
1887
		updatecache_flg=True
1888
		while (1):
1889
			try:
1890
				ip=socket.gethostbyname(hostname)
1891
				dosyncuri=string.replace(syncuri, "//"+hostname+"/", "//"+ip+"/", 1)
1892
			except Exception, e:
1893
				print "Notice:",str(e)
1894
				dosyncuri = syncuri
1895
1896
			if (retries==0):
1897
				print ">>> starting rsync with "+syncuri+"..."
1898
			else:
1899
				print "\n\n>>> Starting retry %d of %d"% (retries,maxretries)
1900
1901
			print ">>> checking server timestamp ..."
1902
			mycommand=rsynccommand+" "+dosyncuri+"/metadata/timestamp.chk "+servertimestampdir
1903
			exitcode=portage.spawn(mycommand,portage.settings,free=1)
1904
			if (exitcode==0):
1905
				try:
1906
					servertimestamp = time.mktime(time.strptime(portage.grabfile(servertimestampdir+"/timestamp.chk")[0], "%a, %d %b %Y %H:%M:%S +0000"))
1907
				except:
1908
					servertimestamp = 0
1909
				
1910
				if (servertimestamp != 0) and (servertimestamp == mytimestamp):
1911
					print ">>> cancelling sync because this server timestamp is the same as local timestamp"
1912
					print
1913
					updatecache_flg=False
1914
					break
1915
				elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
1916
					print ">>> skipping because this server timestamp is older than local timestamp ..."
1917
					print
1918
				elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
1919
					# actual sync
1920
					mycommand=rsynccommand+" "+dosyncuri+"/* "+myportdir
1921
					exitcode=portage.spawn(mycommand,portage.settings,free=1)
1922
					if exitcode in [0,1,2,3,4,11,14,20,21]:
1923
						break
1924
			elif exitcode in [0,1,2,3,4,11,14,20,21]:
1925
				break
1926
1927
			retries=retries+1
1928
1865
1929
			if retries<=maxretries:
1930
				print ">>> retry ..."
1931
				time.sleep(11)
1932
			else:
1933
				# over retries
1934
				# exit loop
1935
				updatecache_flg=False
1936
				break
1937
1866
1938
		if (exitcode>0):
1867
if myaction == "sync" and (not "--help" in myopts):
1939
			print
1868
	if "--pretend" in myopts:
1940
			if exitcode==1:
1869
		print "emerge: the \"sync\" action does not support \"--pretend.\""
1941
				print darkred("!!!")+green(" Rsync has reported that there is a syntax error. Please ensure")
1870
		sys.exit(1)
1942
				print darkred("!!!")+green(" that your SYNC statement is proper.")
1871
	
1943
				print darkred("!!!")+green(" SYNC="+portage.settings["SYNC"])
1872
	src = []
1944
			elif exitcode==11:
1873
	dest = []
1945
				print darkred("!!!")+green(" Rsync has reported that there is a File IO error. Normally")
1874
	
1946
				print darkred("!!!")+green(" this means your disk is full, but can be caused by corruption")
1875
	# check mode
1947
				print darkred("!!!")+green(" on the filesystem that contains PORTDIR. Please investigate")
1876
	if not myfiles or "all" in myfiles:
1948
				print darkred("!!!")+green(" and try again after the problem has been fixed.")
1877
		src.append(portage.settings["SYNC"])
1949
				print darkred("!!!")+green(" PORTDIR="+portage.settings["PORTDIR"])
1878
		dest.append(portage.settings["PORTDIR"])
1950
			elif exitcode==20:
1879
		print ">>> adding PORTDIR to sync queue"
1951
				print darkred("!!!")+green(" Rsync was killed before it finished.")
1880
	if myfiles:
1952
			else:
1881
		overlayfile = portage.grabdict("/etc/portage/overlays")
1953
				print darkred("!!!")+green(" Rsync has not successfully finished. It is recommended that you keep")
1882
		if not overlayfile:
1954
				print darkred("!!!")+green(" trying or that you use the 'emerge-webrsync' option if you are unable")
1883
			print red("*"), "Repository sync requested, but no overlay specifications found in"
1955
				print darkred("!!!")+green(" to use rsync due to firewall or other restrictions. This should be a")
1884
			print red("*"), "    /etc/portage/overlays"
1956
				print darkred("!!!")+green(" temporary problem unless complications exist with your network")
1957
				print darkred("!!!")+green(" (and possibly your system's filesystem) configuration.")
1958
			print
1959
			sys.exit(exitcode)
1960
	elif syncuri[:6]=="cvs://":
1961
		if not os.path.exists("/usr/bin/cvs"):
1962
			print "!!! /usr/bin/cvs does not exist, so rsync support is disabled."
1963
			print "!!! Type \"emerge dev-util/cvs\" to enable CVS support."
1964
			sys.exit(1)
1885
			sys.exit(1)
1965
		cvsroot=syncuri[6:]
1886
		if "all" in myfiles:
1966
		cvsdir=os.path.dirname(myportdir)
1887
			repolist = overlayfile.keys()
1967
		if not os.path.exists(myportdir+"/CVS"):
1968
			#initial checkout
1969
			print ">>> starting initial cvs checkout with "+syncuri+"..."
1970
			if not portage.spawn("cd "+cvsdir+"; cvs -d "+cvsroot+" login",portage.settings,free=1):
1971
				print "!!! cvs login error; exiting."
1972
				sys.exit(1)
1973
			if os.path.exists(cvsdir+"/gentoo-x86"):
1974
				print "!!! existing",cvsdir+"/gentoo-x86 directory; exiting."
1975
				sys.exit(1)
1976
			if not portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+cvsroot+" co gentoo-x86",portage.settings,free=1):
1977
				print "!!! cvs checkout error; exiting."
1978
				sys.exit(1)
1979
			if cvsdir!=myportdir:
1980
				portage.movefile(cvsdir,portage.settings["PORTDIR"])
1981
			sys.exit(0)
1982
		else:
1888
		else:
1983
			#cvs update
1889
			repolist = myfiles
1984
			print ">>> starting cvs update with "+syncuri+"..."
1890
		for repo in repolist:
1985
			sys.exit(portage.spawn("cd "+myportdir+"; cvs -z0 -q update -dP",portage.settings,free=1)) 
1891
			if repo in overlayfile.keys():
1986
	else:
1892
				if len(overlayfile[repo]) < 2:
1987
		print "!!! rsync setting: ",syncuri,"not recognized; exiting."
1893
					print yellow("*"), "Repository \"%s\" found in overlay file, but is missing elements" % repo
1988
		sys.exit(1)
1894
					print yellow("*"), "Please check /etc/portage/overlays for errors.\n"
1989
	if os.path.exists(myportdir+"/metadata/cache") and updatecache_flg:
1895
					print yellow("*"), "Skipping repository \"%s\"" % repo
1990
		print "\n>>> Updating Portage cache...  ",
1896
					continue
1991
		os.umask(0002)
1897
				src.append(overlayfile[repo][0])
1992
		portage.db["/"]["porttree"].dbapi.flush_auxcache()
1898
				dest.append(overlayfile[repo][1])
1993
		if os.path.exists(portage.dbcachedir):
1899
				print ">>> adding repository \"%s\" to sync queue" % repo
1994
			portage.spawn("rm -Rf "+portage.dbcachedir,portage.settings,free=1)
1900
1995
		try:
1901
	for i in range(0, len(src)):
1996
			os.mkdir(portage.dbcachedir)
1902
		link = sync.connection.create(src[i], dest[i], portage.settings)
1997
			os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
1903
		link.setup()
1998
			os.chmod(portage.dbcachedir, 06775)
1904
		rValue = link.sync()
1999
			os.umask(002)
1905
		if rValue not in [0, 128]:
2000
		except:
1906
			sys.exit(rValue)
2001
			pass
1907
	
2002
		mynodes=portage.portdb.cp_all()
1908
		if os.path.exists(dest[i]+"/metadata/cache") and (rValue & 128):
2003
		portage.eclassdb["modifications_limit"] = 1000
1909
			updatecache(dest[i])
2004
		for x in mynodes:
2005
			myxsplit=x.split("/")
2006
			if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
2007
				os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
2008
				os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
2009
				os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
2010
			mymatches=portage.portdb.xmatch("match-all",x)
2011
			for y in mymatches:
2012
				update_spinner()
2013
				try:
2014
					ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache",debug=("cachedebug" in portage.features))
2015
				except:
2016
					pass
2017
		portage.eclassdb["modifications_limit"] = 1
2018
		portage.save_eclassdb(forced=1)
2019
		portage.spawn("chmod -R g+rw "+portage.dbcachedir, portage.settings, free=1)
2020
		sys.stdout.write("\b\b  ...done!\n\n")
2021
		sys.stdout.flush()
2022
1910
2023
	portage.portageexit()
1911
	portage.portageexit()
2024
	reload(portage)
1912
	reload(portage)
2025
	mybestpv=portage.portdb.xmatch("bestmatch-visible","sys-apps/portage")
1913
	mybestpv=portage.portdb.xmatch("bestmatch-visible","sys-apps/portage")
2026
	mypvs=portage.best(portage.db[portage.root]["vartree"].dbapi.match("sys-apps/portage"))
1914
	mypvs=portage.best(portage.db[portage.root]["vartree"].dbapi.match("sys-apps/portage"))
2027
1915
2028
	if(mybestpv != mypvs):
1916
	if (mybestpv != mypvs):
2029
		print
1917
		print
2030
		print red(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended"
1918
		print red(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended"
2031
		print red(" * ")+"that you update portage now, before any other packages are updated."
1919
		print red(" * ")+"that you update portage now, before any other packages are updated."

Return to bug 35535