Attachment #22294 for bug #35934




		except OSError:
			pass
	
def flushmtimedb(record,myroot=root):
	if myroot!=None and mtimedbdict.has_key(myroot):
		if record in mytimedbdict[myroot].keys():
			del mytimedbdict[myroot][record]
			#print "mtimedb["+record+"] is cleared."
		else:
			writemsg("Invalid or unset record '"+record+"' in mtimedb.\n")
	else:
		writemsg("No mtimedb found for root "+myroot+"\n")

#grab mtimes for eclasses and upgrades
mtimedb={}
mtimedbkeys=[
"updates", "info",
"version", "starttime",
"resume"
]
mtimedbdict={}
mtimedbdict[root]={}
if root!="/":
	mtimedbdict["/"]={}
for mymtimedbroot in mtimedbdict.keys():
	mymtimedbfile=mymtimedbroot+"var/cache/edb/mtimedb"
	try:
		mypickle=cPickle.Unpickler(open(mymtimedbfile))
		mypickle.find_global=None
		mtimedbdict[mymtimedbroot]=mypickle.load()
		if mtimedbdict[mymtimedbroot].has_key("old"):
			mtimedbdict[mymtimedbroot]["updates"]=mtimedbdict[mymtimedbroot]["old"]
			del mtimedbdict[mymtimedbroot]["old"]
		if mtimedbdict[mymtimedbroot].has_key("cur"):
			del mtimedbdict[mymtimedbroot]["cur"]
	except:
		#print "!!!",e
		mtimedbdict[mymtimedbroot]={"updates":{},"version":"","starttime":0}

	for x in mtimedbdict[mymtimedbroot].keys():
		if x not in mtimedbkeys:
			writemsg("Deleting invalid "+str(mymtimedbroot)+" mtimedb key: "+str(x)+"\n")
			del mtimedbdict[mymtimedbroot][x]

# set portage.root mtimedb
# XXX BIG WARNING: this is only a reference, set for emerge compatibility... 
# XXX I've checked this vars are never reaffected, and it should be okay.
# XXX But patching emerge would be safer. Sed does it very well:
# XXX sed -i 's:portage\.mtimedb:portage.mtimedbdict[portage.root]:' bin/emerge
mtimedb=mtimedbdict[root]

#,"porttree":portagetree(root,virts),"bintree":binarytree(root,virts)}
features=settings["FEATURES"].split()

do_upgrade_packagesmessage=0
def do_upgrade(mykey,myroot="/"):
	global do_upgrade_packagesmessage
	writemsg("\n\n")
	writemsg(green("Performing Global Updates: ")+bold(mykey)+"\n")
	writemsg("(Could take a couple minutes if you have a lot of binary packages.)\n")
	# binaries updates are only for the "/" tree (no need to do them twice...)
	# XXX Why is it that way though? Shouldn't bintree be populated
	# XXX from root+settings["PKGDIR"] instead? Is it to have access
	# XXX to other existing binaries, or something like this?
	if myroot=="/":
		writemsg("  "+bold(".")+"='update pass'  "+bold("*")+"='binary update'  "+bold("@")+"='/var/db move'\n"+"  "+bold("s")+"='/var/db SLOT move' "+bold("S")+"='binary SLOT move'\n")
	else:
		writemsg("  "+bold(".")+"='update pass'  "+bold("@")+"='"+myroot+"var/db move'"+"  "+bold("s")+"='"+myroot+"var/db SLOT move'\n")
	processed=1
	#remove stale virtual entries (mappings for packages that no longer exist)
	myvirts=grabdict(myroot+"var/cache/edb/virtuals")
	
	worldlist=grabfile(myroot+"var/cache/edb/world")
	myupd=grabfile(mykey)
	db[myroot]["bintree"]=binarytree(myroot,settings["PKGDIR"],virts)
	for myline in myupd:
		mysplit=myline.split()
		if not len(mysplit):

		sys.stdout.flush()

		if mysplit[0]=="move":
			db[myroot]["vartree"].dbapi.move_ent(mysplit)
			# Do it only for "/" tree, since binaries are only there.
			if myroot=="/":
				db[myroot]["bintree"].move_ent(mysplit)
			#update world entries:
			for x in range(0,len(worldlist)):
				#update world entries, if any.

						myvirts[myvirt][mypos]=mysplit[2]

		elif mysplit[0]=="slotmove":
			db[myroot]["vartree"].dbapi.move_slot_ent(mysplit)
			# Do it only for "/" tree, since binaries are only there.
			if myroot=="/":
				db[myroot]["bintree"].move_slot_ent(mysplit,settings["PORTAGE_TMPDIR"]+"/tbz2")

	# We gotta do the brute force updates for these now.
	# Do it only for "/" tree, since binaries are only there.
	if myroot=="/":
		if (settings["PORTAGE_CALLER"] in ["fixpackages"]) or \
		   ("fixpackages" in features):
			db[myroot]["bintree"].update_ents(myupd,settings["PORTAGE_TMPDIR"]+"/tbz2")
		else:
			do_upgrade_packagesmessage = 1
	
	if processed:
		#update our internal mtime since we processed all our directives.
		mtimedbdict[myroot]["updates"][mykey]=os.stat(mykey)[ST_MTIME]
	myworld=open(myroot+"var/cache/edb/world","w")
	for x in worldlist:
		myworld.write(x+"\n")
	myworld.close()
	writedict(myvirts,myroot+"var/cache/edb/virtuals")
	print ""

def portageexit():

		for x in db.keys():
			if db[x].has_key("porttree"):
				db[x]["porttree"].dbapi.save_auxcache(x)
		# both "/" and root mtimedbs should be saved
		for mymtimedbroot in mtimedbdict.keys():
			if mtimedbdict[mymtimedbroot]:
				# Store this mtimedb
				mymfn=mymtimedbroot+"var/cache/edb/mtimedb"
				try:
					mtimedbdict[mymtimedbroot]["version"]=VERSION
					cPickle.dump(mtimedbdict[mymtimedbroot],open(mymfn,"w"))
					#print "*** Wrote out "+mymtimedbroot+" mtimedb data successfully."
					os.chown(mymfn,uid,portage_gid)
					os.chmod(mymfn,0664)
				except Exception, e:
					pass

atexit.register(portageexit)


	if settings["PORTAGE_CALLER"] in ["emerge","fixpackages"]:
		#only do this if we're root and not running repoman/ebuild digest
		updpath=os.path.normpath(settings["PORTDIR"]+"///profiles/updates")
		didupdate=0
		if not mtimedb.has_key("updates"):
			mtimedb["updates"]={}
		try:
			mylist=listdir(updpath,EmptyOnError=1)
			# resort the list
			mylist=[myfile[3:]+"-"+myfile[:2] for myfile in mylist]
			mylist.sort()
			mylist=[myfile[5:]+"-"+myfile[:4] for myfile in mylist]
		except OSError:
			#directory doesn't exist
			pass
		# both "/" and "root" trees should be upgraded
		didupdate=0
		for mymtimedbroot in mtimedbdict.keys():
			if not mtimedbdict[mymtimedbroot].has_key("updates"):
				mtimedbdict[mymtimedbroot]["updates"]={}
			for myfile in mylist:
				mykey=updpath+"/"+myfile
				if not os.path.isfile(mykey):
					continue
				if (not mtimedbdict[mymtimedbroot]["updates"].has_key(mykey)) or \
					 (mtimedbdict[mymtimedbroot]["updates"][mykey] != os.stat(mykey)[ST_MTIME]) or \
					 (settings["PORTAGE_CALLER"] == "fixpackages"):
					didupdate=1
					do_upgrade(mykey,myroot=mymtimedbroot)
					portageexit() # This lets us save state for C-c.
		# do_vartree can be done only once, it already takes care of both vartrees.
		# And do_upgrade_packagesmessage is only about the "/" tree.
			pass
		if didupdate:
			#make sure our internal databases are consistent; recreate our virts and vartree
			do_vartree()

Return to bug 35934

Lines 5820-5876 Link Here

(-)pym.orig/portage.py (-60 / +92 lines)
5820	except OSError:	5820	except OSError:
5821	pass	5821	pass
5822		5822
5823	def flushmtimedb(record):	5823	def flushmtimedb(record,myroot=root):
5824	if mtimedb:	5824	if myroot!=None and mtimedbdict.has_key(myroot):
5825	if record in mtimedb.keys():	5825	if record in mytimedbdict[myroot].keys():
5826	del mtimedb[record]	5826	del mytimedbdict[myroot][record]
5827	#print "mtimedb["+record+"] is cleared."	5827	#print "mtimedb["+record+"] is cleared."
5828	else:	5828	else:
5829	writemsg("Invalid or unset record '"+record+"' in mtimedb.\n")	5829	writemsg("Invalid or unset record '"+record+"' in mtimedb.\n")
		5830	else:
		5831	writemsg("No mtimedb found for root "+myroot+"\n")
5830		5832
5831	#grab mtimes for eclasses and upgrades	5833	#grab mtimes for eclasses and upgrades
5832	mtimedb={}
5833	mtimedbkeys=[	5834	mtimedbkeys=[
5834	"updates", "info",	5835	"updates", "info",
5835	"version", "starttime",	5836	"version", "starttime",
5836	"resume"	5837	"resume"
5837	]	5838	]
5838	mtimedbfile=root+"var/cache/edb/mtimedb"	5839	mtimedbdict={}
5839	try:	5840	mtimedbdict[root]={}
5840	mypickle=cPickle.Unpickler(open(mtimedbfile))	5841	if root!="/":
5841	mypickle.find_global=None	5842	mtimedbdict["/"]={}
5842	mtimedb=mypickle.load()	5843	for mymtimedbroot in mtimedbdict.keys():
5843	if mtimedb.has_key("old"):	5844	mymtimedbfile=mymtimedbroot+"var/cache/edb/mtimedb"
5844	mtimedb["updates"]=mtimedb["old"]	5845	try:
5845	del mtimedb["old"]	5846	mypickle=cPickle.Unpickler(open(mymtimedbfile))
5846	if mtimedb.has_key("cur"):	5847	mypickle.find_global=None
5847	del mtimedb["cur"]	5848	mtimedbdict[mymtimedbroot]=mypickle.load()
5848	except:	5849	if mtimedbdict[mymtimedbroot].has_key("old"):
5849	#print "!!!",e	5850	mtimedbdict[mymtimedbroot]["updates"]=mtimedbdict[mymtimedbroot]["old"]
5850	mtimedb={"updates":{},"version":"","starttime":0}	5851	del mtimedbdict[mymtimedbroot]["old"]
5851		5852	if mtimedbdict[mymtimedbroot].has_key("cur"):
5852	for x in mtimedb.keys():	5853	del mtimedbdict[mymtimedbroot]["cur"]
5853	if x not in mtimedbkeys:	5854	except:
5854	writemsg("Deleting invalid mtimedb key: "+str(x)+"\n")	5855	#print "!!!",e
5855	del mtimedb[x]	5856	mtimedbdict[mymtimedbroot]={"updates":{},"version":"","starttime":0}
		5857
		5858	for x in mtimedbdict[mymtimedbroot].keys():
		5859	if x not in mtimedbkeys:
		5860	writemsg("Deleting invalid "+str(mymtimedbroot)+" mtimedb key: "+str(x)+"\n")
		5861	del mtimedbdict[mymtimedbroot][x]
		5862
		5863	# set portage.root mtimedb
		5864	# XXX BIG WARNING: this is only a reference, set for emerge compatibility...
		5865	# XXX I've checked this vars are never reaffected, and it should be okay.
		5866	# XXX But patching emerge would be safer. Sed does it very well:
		5867	# XXX sed -i 's:portage\.mtimedb:portage.mtimedbdict[portage.root]:' bin/emerge
		5868	mtimedb=mtimedbdict[root]
5856		5869
5857	#,"porttree":portagetree(root,virts),"bintree":binarytree(root,virts)}	5870	#,"porttree":portagetree(root,virts),"bintree":binarytree(root,virts)}
5858	features=settings["FEATURES"].split()	5871	features=settings["FEATURES"].split()
5859		5872
5860	do_upgrade_packagesmessage=0	5873	do_upgrade_packagesmessage=0
5861	def do_upgrade(mykey):	5874	def do_upgrade(mykey,myroot="/"):
5862	global do_upgrade_packagesmessage	5875	global do_upgrade_packagesmessage
5863	writemsg("\n\n")	5876	writemsg("\n\n")
5864	writemsg(green("Performing Global Updates: ")+bold(mykey)+"\n")	5877	writemsg(green("Performing Global Updates: ")+bold(mykey)+"\n")
5865	writemsg("(Could take a couple minutes if you have a lot of binary packages.)\n")	5878	writemsg("(Could take a couple minutes if you have a lot of binary packages.)\n")
5866	writemsg(" "+bold(".")+"='update pass' "+bold("*")+"='binary update' "+bold("@")+"='/var/db move'\n"+" "+bold("s")+"='/var/db SLOT move' "+bold("S")+"='binary SLOT move'\n")	5879	# binaries updates are only for the "/" tree (no need to do them twice...)
		5880	# XXX Why is it that way though? Shouldn't bintree be populated
		5881	# XXX from root+settings["PKGDIR"] instead? Is it to have access
		5882	# XXX to other existing binaries, or something like this?
		5883	if myroot=="/":
		5884	writemsg(" "+bold(".")+"='update pass' "+bold("*")+"='binary update' "+bold("@")+"='/var/db move'\n"+" "+bold("s")+"='/var/db SLOT move' "+bold("S")+"='binary SLOT move'\n")
		5885	else:
		5886	writemsg(" "+bold(".")+"='update pass' "+bold("@")+"='"+myroot+"var/db move'"+" "+bold("s")+"='"+myroot+"var/db SLOT move'\n")
5867	processed=1	5887	processed=1
5868	#remove stale virtual entries (mappings for packages that no longer exist)	5888	#remove stale virtual entries (mappings for packages that no longer exist)
5869	myvirts=grabdict("/var/cache/edb/virtuals")	5889	myvirts=grabdict(myroot+"var/cache/edb/virtuals")
5870		5890
5871	worldlist=grabfile("/var/cache/edb/world")	5891	worldlist=grabfile(myroot+"var/cache/edb/world")
5872	myupd=grabfile(mykey)	5892	myupd=grabfile(mykey)
5873	db["/"]["bintree"]=binarytree("/",settings["PKGDIR"],virts)	5893	db[myroot]["bintree"]=binarytree(myroot,settings["PKGDIR"],virts)
5874	for myline in myupd:	5894	for myline in myupd:
5875	mysplit=myline.split()	5895	mysplit=myline.split()
5876	if not len(mysplit):	5896	if not len(mysplit):
Lines 5891-5898 Link Here
5891	sys.stdout.flush()	5911	sys.stdout.flush()
5892		5912
5893	if mysplit[0]=="move":	5913	if mysplit[0]=="move":
5894	db["/"]["vartree"].dbapi.move_ent(mysplit)	5914	db[myroot]["vartree"].dbapi.move_ent(mysplit)
5895	db["/"]["bintree"].move_ent(mysplit)	5915	# Do it only for "/" tree, since binaries are only there.
		5916	if myroot=="/":
		5917	db[myroot]["bintree"].move_ent(mysplit)
5896	#update world entries:	5918	#update world entries:
5897	for x in range(0,len(worldlist)):	5919	for x in range(0,len(worldlist)):
5898	#update world entries, if any.	5920	#update world entries, if any.
Lines 5906-5929 Link Here
5906	myvirts[myvirt][mypos]=mysplit[2]	5928	myvirts[myvirt][mypos]=mysplit[2]
5907		5929
5908	elif mysplit[0]=="slotmove":	5930	elif mysplit[0]=="slotmove":
5909	db["/"]["vartree"].dbapi.move_slot_ent(mysplit)	5931	db[myroot]["vartree"].dbapi.move_slot_ent(mysplit)
5910	db["/"]["bintree"].move_slot_ent(mysplit,settings["PORTAGE_TMPDIR"]+"/tbz2")	5932	# Do it only for "/" tree, since binaries are only there.
		5933	if myroot=="/":
		5934	db[myroot]["bintree"].move_slot_ent(mysplit,settings["PORTAGE_TMPDIR"]+"/tbz2")
5911		5935
5912	# We gotta do the brute force updates for these now.	5936	# We gotta do the brute force updates for these now.
5913	if (settings["PORTAGE_CALLER"] in ["fixpackages"]) or \	5937	# Do it only for "/" tree, since binaries are only there.
5914	("fixpackages" in features):	5938	if myroot=="/":
5915	db["/"]["bintree"].update_ents(myupd,settings["PORTAGE_TMPDIR"]+"/tbz2")	5939	if (settings["PORTAGE_CALLER"] in ["fixpackages"]) or \
5916	else:	5940	("fixpackages" in features):
5917	do_upgrade_packagesmessage = 1	5941	db[myroot]["bintree"].update_ents(myupd,settings["PORTAGE_TMPDIR"]+"/tbz2")
		5942	else:
		5943	do_upgrade_packagesmessage = 1
5918		5944
5919	if processed:	5945	if processed:
5920	#update our internal mtime since we processed all our directives.	5946	#update our internal mtime since we processed all our directives.
5921	mtimedb["updates"][mykey]=os.stat(mykey)[ST_MTIME]	5947	mtimedbdict[myroot]["updates"][mykey]=os.stat(mykey)[ST_MTIME]
5922	myworld=open("/var/cache/edb/world","w")	5948	myworld=open(myroot+"var/cache/edb/world","w")
5923	for x in worldlist:	5949	for x in worldlist:
5924	myworld.write(x+"\n")	5950	myworld.write(x+"\n")
5925	myworld.close()	5951	myworld.close()
5926	writedict(myvirts,"/var/cache/edb/virtuals")	5952	writedict(myvirts,myroot+"var/cache/edb/virtuals")
5927	print ""	5953	print ""
5928		5954
5929	def portageexit():	5955	def portageexit():
Lines 5933-5949 Link Here
5933	for x in db.keys():	5959	for x in db.keys():
5934	if db[x].has_key("porttree"):	5960	if db[x].has_key("porttree"):
5935	db[x]["porttree"].dbapi.save_auxcache(x)	5961	db[x]["porttree"].dbapi.save_auxcache(x)
5936	if mtimedb:	5962	# both "/" and root mtimedbs should be saved
5937	# Store mtimedb	5963	for mymtimedbroot in mtimedbdict.keys():
5938	mymfn=mtimedbfile	5964	if mtimedbdict[mymtimedbroot]:
5939	try:	5965	# Store this mtimedb
5940	mtimedb["version"]=VERSION	5966	mymfn=mymtimedbroot+"var/cache/edb/mtimedb"
5941	cPickle.dump(mtimedb,open(mymfn,"w"))	5967	try:
5942	#print "*** Wrote out mtimedb data successfully."	5968	mtimedbdict[mymtimedbroot]["version"]=VERSION
5943	os.chown(mymfn,uid,portage_gid)	5969	cPickle.dump(mtimedbdict[mymtimedbroot],open(mymfn,"w"))
5944	os.chmod(mymfn,0664)	5970	#print "*** Wrote out "+mymtimedbroot+" mtimedb data successfully."
5945	except Exception, e:	5971	os.chown(mymfn,uid,portage_gid)
5946	pass	5972	os.chmod(mymfn,0664)
		5973	except Exception, e:
		5974	pass
5947		5975
5948	atexit.register(portageexit)	5976	atexit.register(portageexit)
5949		5977
Lines 5951-5978 Link Here
5951	if settings["PORTAGE_CALLER"] in ["emerge","fixpackages"]:	5979	if settings["PORTAGE_CALLER"] in ["emerge","fixpackages"]:
5952	#only do this if we're root and not running repoman/ebuild digest	5980	#only do this if we're root and not running repoman/ebuild digest
5953	updpath=os.path.normpath(settings["PORTDIR"]+"///profiles/updates")	5981	updpath=os.path.normpath(settings["PORTDIR"]+"///profiles/updates")
5954	didupdate=0
5955	if not mtimedb.has_key("updates"):
5956	mtimedb["updates"]={}
5957	try:	5982	try:
5958	mylist=listdir(updpath,EmptyOnError=1)	5983	mylist=listdir(updpath,EmptyOnError=1)
5959	# resort the list	5984	# resort the list
5960	mylist=[myfile[3:]+"-"+myfile[:2] for myfile in mylist]	5985	mylist=[myfile[3:]+"-"+myfile[:2] for myfile in mylist]
5961	mylist.sort()	5986	mylist.sort()
5962	mylist=[myfile[5:]+"-"+myfile[:4] for myfile in mylist]	5987	mylist=[myfile[5:]+"-"+myfile[:4] for myfile in mylist]
		5988	except OSError:
		5989	#directory doesn't exist
		5990	pass
		5991	# both "/" and "root" trees should be upgraded
		5992	didupdate=0
		5993	for mymtimedbroot in mtimedbdict.keys():
		5994	if not mtimedbdict[mymtimedbroot].has_key("updates"):
		5995	mtimedbdict[mymtimedbroot]["updates"]={}
5963	for myfile in mylist:	5996	for myfile in mylist:
5964	mykey=updpath+"/"+myfile	5997	mykey=updpath+"/"+myfile
5965	if not os.path.isfile(mykey):	5998	if not os.path.isfile(mykey):
5966	continue	5999	continue
5967	if (not mtimedb["updates"].has_key(mykey)) or \	6000	if (not mtimedbdict[mymtimedbroot]["updates"].has_key(mykey)) or \
5968	(mtimedb["updates"][mykey] != os.stat(mykey)[ST_MTIME]) or \	6001	(mtimedbdict[mymtimedbroot]["updates"][mykey] != os.stat(mykey)[ST_MTIME]) or \
5969	(settings["PORTAGE_CALLER"] == "fixpackages"):	6002	(settings["PORTAGE_CALLER"] == "fixpackages"):
5970	didupdate=1	6003	didupdate=1
5971	do_upgrade(mykey)	6004	do_upgrade(mykey,myroot=mymtimedbroot)
5972	portageexit() # This lets us save state for C-c.	6005	portageexit() # This lets us save state for C-c.
5973	except OSError:	6006	# do_vartree can be done only once, it already takes care of both vartrees.
5974	#directory doesn't exist	6007	# And do_upgrade_packagesmessage is only about the "/" tree.
5975	pass
5976	if didupdate:	6008	if didupdate:
5977	#make sure our internal databases are consistent; recreate our virts and vartree	6009	#make sure our internal databases are consistent; recreate our virts and vartree
5978	do_vartree()	6010	do_vartree()