Go to:
Gentoo Home
Documentation
Forums
Lists
Bugs
Planet
Store
Wiki
Get Gentoo!
Gentoo's Bugzilla – Attachment 111095 Details for
Bug 147516
[PATCH] Parallel portage can reduce build times
Home
|
New
–
[Ex]
|
Browse
|
Search
|
Privacy Policy
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Fix a --fetchonly related bug
portage-2.1.2-r10-parallel.patch (text/plain), 53.29 KB, created by
devsk
on 2007-02-23 23:21:02 UTC
(
hide
)
Description:
Fix a --fetchonly related bug
Filename:
MIME Type:
Creator:
devsk
Created:
2007-02-23 23:21:02 UTC
Size:
53.29 KB
patch
obsolete
>diff -ru portage-2.1.2.orig/bin/emerge portage-2.1.2/bin/emerge >--- portage-2.1.2.orig/bin/emerge 2007-02-23 13:06:11.000000000 -0800 >+++ portage-2.1.2/bin/emerge 2007-02-23 15:18:38.000000000 -0800 >@@ -2005,6 +2005,7 @@ > ignore_priority_range.extend( > xrange(DepPriority.MIN, DepPriority.MEDIUM + 1)) > tree_mode = "--tree" in self.myopts >+ slotcount = 0 > while not mygraph.empty(): > ignore_priority = None > nodes = None >@@ -2014,11 +2015,11 @@ > if not mygraph.contains(node): > asap_nodes.remove(node) > continue >- if not mygraph.child_nodes(node, >- ignore_priority=DepPriority.SOFT): >+ if not mygraph.child_nodes(node, ignore_priority=DepPriority.SOFT): > nodes = [node] > asap_nodes.remove(node) > break >+ # if no asap node, then get the first non-null set of leaf (root if --tree) nodes in the priority range > if not nodes: > for ignore_priority in ignore_priority_range: > nodes = get_nodes(ignore_priority=ignore_priority) >@@ -2055,17 +2056,14 @@ > if node not in mergeable_nodes: > return False > selected_nodes.add(node) >- for child in mygraph.child_nodes(node, >- ignore_priority=DepPriority.SOFT): >- if not gather_deps( >- mergeable_nodes, selected_nodes, child): >+ for child in mygraph.child_nodes(node, ignore_priority=DepPriority.SOFT): >+ if not gather_deps(mergeable_nodes, selected_nodes, child): > return False > return True > mergeable_nodes = set(nodes) > for node in nodes: > selected_nodes = set() >- if gather_deps( >- mergeable_nodes, selected_nodes, node): >+ if gather_deps(mergeable_nodes, selected_nodes, node): > break > else: > selected_nodes = None >@@ -2122,7 +2120,10 @@ > sys.exit(1) > > for node in selected_nodes: >- retlist.append(list(node)) >+ node2 = list(node) >+ if len(node2) == 4: >+ node2.append(str(slotcount)) >+ retlist.append(node2) > mygraph.remove(node) > if not reversed and not circular_blocks and myblockers.contains(node): > """This node may have invalidated one or more blockers.""" >@@ -2131,6 +2132,7 @@ > if not myblockers.child_nodes(blocker): > myblockers.remove(blocker) > del self.blocker_parents[blocker] >+ slotcount += 1 > > if not reversed: > """Blocker validation does not work with reverse mode, >@@ -2333,7 +2335,7 @@ > for x in mylist: > if "blocks" == x[0]: > continue >- graph_key = tuple(x) >+ graph_key = tuple(x[:-1]) > if "--tree" in self.myopts: > depth = len(tree_nodes) > while depth and graph_key not in \ >@@ -2347,13 +2349,13 @@ > for i in xrange(len(mylist)-1,-1,-1): > if "blocks" == mylist[i][0]: > continue >- graph_key = tuple(mylist[i]) >- if mylist[i][-1] != "nomerge": >+ graph_key = tuple(mylist[i][:-1]) >+ if mylist[i][-2] != "nomerge": > last_merge_depth = node_depth[graph_key] > continue > if node_depth[graph_key] >= last_merge_depth or \ > i < len(mylist) - 1 and \ >- node_depth[graph_key] >= node_depth[tuple(mylist[i+1])]: >+ node_depth[graph_key] >= node_depth[tuple(mylist[i+1][:-1])]: > del mylist[i] > del node_depth[graph_key] > del tree_nodes >@@ -2549,7 +2551,7 @@ > if verbosity == 3: > # size verbose > mysize=0 >- if x[0] == "ebuild" and x[-1]!="nomerge": >+ if x[0] == "ebuild" and x[-2]!="nomerge": > try: > myfilesdict = portdb.getfetchsizes(pkg_key, > useflags=self.useFlags[myroot][pkg_key], >@@ -2601,7 +2603,7 @@ > oldlp=mywidth-30 > newlp=oldlp-30 > >- indent = " " * node_depth[tuple(x)] >+ indent = " " * node_depth[tuple(x[:-1])] > > if myoldbest: > myoldbest=portage.pkgsplit(myoldbest)[1]+"-"+portage.pkgsplit(myoldbest)[2] >@@ -2833,11 +2835,403 @@ > self.pkgsettings["/"] = \ > portage.config(clone=trees["/"]["vartree"].settings) > >- def merge(self, mylist, favorites, mtimedb): >+ def restart_portage(self, x, mergecount, totalcount, mtimedb): >+ xterm_titles = "notitles" not in self.settings.features >+ # don't really restart if any of these is true >+ # XXXXX - seems like redundant check, but what the hell! sky is not falling as yet. >+ if "--pretend" in self.myopts or "--fetchonly" in self.myopts or \ >+ "--fetch-all-uri" in self.myopts or "--buildpkgonly" in self.myopts: >+ return >+ >+ bad_resume_opts = set(["--ask", "--tree", "--changelog", "--skipfirst", >+ "--resume"]) >+ mysplit=portage.pkgsplit(x[2]) >+ myver=mysplit[1]+"-"+mysplit[2] >+ if myver[-3:]=='-r0': >+ myver=myver[:-3] >+ if (myver != portage.VERSION) and \ >+ "livecvsportage" not in self.settings.features: >+ if totalcount > mergecount: >+ emergelog(xterm_titles, >+ " ::: completed emerge ("+ \ >+ str(mergecount)+" of "+ \ >+ str(totalcount)+") "+ \ >+ x[2]+" to "+x[1]) >+ emergelog(xterm_titles, " *** RESTARTING " + \ >+ "emerge via exec() after change of " + \ >+ "portage version.") >+ portage.run_exitfuncs() >+ mynewargv=[sys.argv[0],"--resume"] >+ for myopt, myarg in self.myopts.iteritems(): >+ if myopt not in bad_resume_opts: >+ if myarg is True: >+ mynewargv.append(myopt) >+ else: >+ mynewargv.append(myopt +"="+ myarg) >+ # priority only needs to be adjusted on the first run >+ os.environ["PORTAGE_NICENESS"] = "0" >+ os.execv(mynewargv[0], mynewargv) >+ >+ def fork_one_emerge(self, x, mergecount, totalcount, mtimedb, favorites, mysysdict): >+ xterm_titles = "notitles" not in self.settings.features >+ myfeat = self.settings.features[:] >+ ldpath_mtimes = mtimedb["ldpath"] >+ myroot=x[1] >+ pkg_key = x[2] >+ pkg_cat = x[2].split("/")[0] >+ pkg_pf = x[2].split("/")[1] >+ pkgindex=2 >+ if x[0]=="blocks": >+ pkgindex=3 >+ >+ build_prefix=self.settings["PORTAGE_TMPDIR"]+"/portage" >+ logid_path = None >+ null_log = 0 >+ >+ if self.settings.get("PORT_LOGDIR", "") == "": >+ while "PORT_LOGDIR" in self.settings: >+ del self.settings["PORT_LOGDIR"] >+ if "PORT_LOGDIR" in self.settings: >+ port_logdir = self.settings["PORT_LOGDIR"] >+ else: >+ port_logdir = self.settings["ROOT"] + portage.DEF_LOGDIR >+ >+ try: >+ portage_util.ensure_dirs(port_logdir, uid=portage.portage_uid, >+ gid=portage.portage_gid, mode=02770) >+ except portage_exception.PortageException, e: >+ writemsg("!!! %s\n" % str(e), noiselevel=-1) >+ writemsg("!!! Permission issues with PORT_LOGDIR='%s'\n" % \ >+ self.settings["PORT_LOGDIR"], noiselevel=-1) >+ writemsg("!!! Because 'parallel' feature is enabled, you won't get any logs.\n", noiselevel=-1) >+ null_log = 1 >+ >+ if not null_log: >+ logid_path = os.path.join(build_prefix, ".logid.")+pkg_cat+"."+pkg_pf >+ if not os.path.exists(logid_path): >+ f = open(logid_path, "w") >+ f.close() >+ del f >+ logid_time = time.strftime("%Y%m%d-%H%M%S", >+ time.gmtime(os.stat(logid_path).st_mtime)) >+ logfile = os.path.join(port_logdir, "%s:%s:%s.log" % \ >+ (pkg_cat, pkg_pf, logid_time)) >+ del logid_time >+ else: >+ logfile = "/dev/null" >+ >+ if "--pretend" not in self.myopts and "--fetchonly" not in self.myopts: >+ print ">>> Emerging (" + \ >+ colorize("MERGE_LIST_PROGRESS", str(mergecount)) + " of " + \ >+ colorize("MERGE_LIST_PROGRESS", str(totalcount)) + ") " + \ >+ colorize("GOOD", x[pkgindex]) + " to " + x[1] >+ print ">>> Logfile in " + logfile >+ emergelog(xterm_titles, " >>> emerge ("+\ >+ str(mergecount)+" of "+str(totalcount)+\ >+ ") "+x[pkgindex]+" to "+x[1]) >+ >+ # need to spawn a --nodeps emerge in a separate process. >+ pkg="="+x[2] >+ merge_env = os.environ.copy() >+ merge_env["PORTAGE_INTERNAL_CALL"] = "1" >+ merge_env["FEATURES"] = merge_env.get("FEATURES", "") + " notitles -parallel" >+ merge_args = [sys.argv[0], "--nodeps", "--oneshot", pkg] >+ good_nodeps_opts = set(["--buildpkg", "--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--getbinpkg",\ >+ "--usepkg", "--usepkgonly"]) >+ fd_pipes = None >+ merge_logfd = None >+ for myopt, myarg in self.myopts.iteritems(): >+ # don't clobber the logfile at the same time as parallel fetch is >+ # all log of parallel fetch will go /var/log/emerge-fetch.log >+ # so, just leave 0,1,2 alone. >+ if "parallel-fetch" in myfeat and myopt == "--fetchonly": >+ fd_pipes = {0:0, 1:1, 2:2} >+ if myopt in good_nodeps_opts: >+ if myarg is True: >+ merge_args.append(myopt) >+ else: >+ merge_args.append(myopt +"="+ myarg) >+ if not fd_pipes: >+ merge_logfd = open(logfile, "w") >+ # put in a start message. This also makes sure that this fd is pointing to a good file on disk >+ # and hence will be used throughout the other spawns that will happen in the children. >+ merge_logfd.write("Package "+x[pkgindex]+" started at "+time.ctime()+"\n\n") >+ merge_logfd.flush() >+ fd_pipes = {0:0, 1:merge_logfd.fileno(), 2:merge_logfd.fileno()} >+ portage_util.apply_secpass_permissions(logfile, uid=portage.portage_uid, gid=portage.portage_gid, mode=0660) >+ >+ mypids = portage.portage_exec.spawn(merge_args, env=merge_env, fd_pipes=fd_pipes, returnpid=True) >+ if merge_logfd: >+ merge_logfd.close() # child has exclusive rights to it now. >+ return mypids[0] >+ >+ def do_one_emerge(self, x, mergecount, totalcount, mtimedb, favorites, mysysdict): >+ xterm_titles = "notitles" not in self.settings.features >+ myfeat = self.settings.features[:] >+ ldpath_mtimes = mtimedb["ldpath"] >+ myroot=x[1] >+ pkg_key = x[2] >+ pkg_cat = x[2].split("/")[0] >+ pkg_pf = x[2].split("/")[1] >+ pkgindex=2 >+ if x[0]=="blocks": >+ pkgindex=3 >+ >+ if "--pretend" not in self.myopts and "--fetchonly" not in self.myopts: >+ print "\n>>> Emerging (" + \ >+ colorize("MERGE_LIST_PROGRESS", str(mergecount)) + " of " + \ >+ colorize("MERGE_LIST_PROGRESS", str(totalcount)) + ") " + \ >+ colorize("GOOD", x[pkgindex]) + " to " + x[1] >+ emergelog(xterm_titles, " >>> emerge ("+\ >+ str(mergecount)+" of "+str(totalcount)+\ >+ ") "+x[pkgindex]+" to "+x[1]) >+ >+ portdb = self.trees[myroot]["porttree"].dbapi >+ bindb = self.trees[myroot]["bintree"].dbapi >+ vartree = self.trees[myroot]["vartree"] >+ pkgsettings = self.pkgsettings[myroot] >+ y = portdb.findname(pkg_key) >+ pkgsettings["EMERGE_FROM"] = x[0] >+ pkgsettings.backup_changes("EMERGE_FROM") >+ pkgsettings.reset() >+ >+ #buildsyspkg: Check if we need to _force_ binary package creation >+ issyspkg = ("buildsyspkg" in myfeat) \ >+ and x[0] != "blocks" \ >+ and mysysdict.has_key(portage.cpv_getkey(x[2])) \ >+ and "--buildpkg" not in self.myopts >+ if x[0] in ["ebuild","blocks"]: >+ if x[0] == "blocks" and "--fetchonly" not in self.myopts: >+ raise Exception, "Merging a blocker" >+ elif "--fetchonly" in self.myopts or \ >+ "--fetch-all-uri" in self.myopts: >+ if "--fetch-all-uri" in self.myopts: >+ retval = portage.doebuild(y, "fetch", myroot, >+ pkgsettings, self.edebug, >+ "--pretend" in self.myopts, fetchonly=1, >+ fetchall=1, mydbapi=portdb, tree="porttree") >+ else: >+ retval = portage.doebuild(y, "fetch", myroot, >+ pkgsettings, self.edebug, >+ "--pretend" in self.myopts, fetchonly=1, >+ mydbapi=portdb, tree="porttree") >+ if (retval is None) or retval: >+ print >+ print "!!! Fetch for",y,"failed, continuing..." >+ print >+ >+ return retval >+ >+ portage.doebuild_environment(y, "setup", myroot, >+ pkgsettings, self.edebug, 1, portdb) >+ catdir = os.path.dirname(pkgsettings["PORTAGE_BUILDDIR"]) >+ portage_util.ensure_dirs(os.path.dirname(catdir), >+ uid=portage.portage_uid, gid=portage.portage_gid, >+ mode=070, mask=0) >+ builddir_lock = None >+ catdir_lock = None >+ try: >+ catdir_lock = portage_locks.lockdir(catdir) >+ portage_util.ensure_dirs(catdir, >+ uid=portage.portage_uid, gid=portage.portage_gid, >+ mode=070, mask=0) >+ builddir_lock = portage_locks.lockdir( >+ pkgsettings["PORTAGE_BUILDDIR"]) >+ try: >+ portage_locks.unlockdir(catdir_lock) >+ finally: >+ catdir_lock = None >+ msg = " === (%s of %s) Cleaning (%s::%s)" % \ >+ (mergecount, totalcount, pkg_key, y) >+ short_msg = "emerge: (%s of %s) %s Clean" % \ >+ (mergecount, totalcount, pkg_key) >+ emergelog(xterm_titles, msg, short_msg=short_msg) >+ retval = portage.doebuild(y, "clean", myroot, >+ pkgsettings, self.edebug, cleanup=1, >+ mydbapi=portdb, tree="porttree") >+ >+ if retval != os.EX_OK: >+ return retval >+ if "--buildpkg" in self.myopts or issyspkg: >+ if issyspkg: >+ print ">>> This is a system package, " + \ >+ "let's pack a rescue tarball." >+ msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ >+ (mergecount, totalcount, pkg_key, y) >+ short_msg = "emerge: (%s of %s) %s Compile" % \ >+ (mergecount, totalcount, pkg_key) >+ emergelog(xterm_titles, msg, short_msg=short_msg) >+ self.trees[myroot]["bintree"].prevent_collision(pkg_key) >+ retval = portage.doebuild(y, "package", myroot, >+ pkgsettings, self.edebug, mydbapi=portdb, >+ tree="porttree") >+ if retval != os.EX_OK: >+ return retval >+ if "--buildpkgonly" not in self.myopts: >+ bintree = self.trees[myroot]["bintree"] >+ bintree.inject(pkg_key) >+ mytbz2 = bintree.getname(pkg_key) >+ msg = " === (%s of %s) Merging (%s::%s)" % \ >+ (mergecount, totalcount, pkg_key, y) >+ short_msg = "emerge: (%s of %s) %s Merge" % \ >+ (mergecount, totalcount, pkg_key) >+ emergelog(xterm_titles, msg, short_msg=short_msg) >+ retval = portage.merge(pkgsettings["CATEGORY"], >+ pkgsettings["PF"], pkgsettings["D"], >+ os.path.join(pkgsettings["PORTAGE_BUILDDIR"], >+ "build-info"), myroot, pkgsettings, >+ myebuild=pkgsettings["EBUILD"], >+ mytree="porttree", mydbapi=portdb, >+ vartree=vartree, prev_mtimes=ldpath_mtimes) >+ if retval != os.EX_OK: >+ return retval >+ elif "noclean" not in pkgsettings.features: >+ portage.doebuild(y, "clean", myroot, >+ pkgsettings, self.edebug, mydbapi=portdb, >+ tree="porttree") >+ else: >+ msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ >+ (mergecount, totalcount, pkg_key, y) >+ short_msg = "emerge: (%s of %s) %s Compile" % \ >+ (mergecount, totalcount, pkg_key) >+ emergelog(xterm_titles, msg, short_msg=short_msg) >+ retval = portage.doebuild(y, "merge", myroot, >+ pkgsettings, self.edebug, vartree=vartree, >+ mydbapi=portdb, tree="porttree", >+ prev_mtimes=ldpath_mtimes) >+ if retval != os.EX_OK: >+ return retval >+ finally: >+ if builddir_lock: >+ portage_locks.unlockdir(builddir_lock) >+ try: >+ if not catdir_lock: >+ # Lock catdir for removal if empty. >+ catdir_lock = portage_locks.lockdir(catdir) >+ finally: >+ if catdir_lock: >+ try: >+ os.rmdir(catdir) >+ except OSError, e: >+ if e.errno not in (errno.ENOENT, >+ errno.ENOTEMPTY, errno.EEXIST): >+ raise >+ del e >+ portage_locks.unlockdir(catdir_lock) >+ >+ elif x[0]=="binary": >+ #merge the tbz2 >+ mytbz2 = self.trees[myroot]["bintree"].getname(pkg_key) >+ if "--getbinpkg" in self.myopts: >+ tbz2_lock = None >+ try: >+ if "distlocks" in pkgsettings.features and \ >+ os.access(pkgsettings["PKGDIR"], os.W_OK): >+ portage_util.ensure_dirs(os.path.dirname(mytbz2)) >+ tbz2_lock = portage_locks.lockfile(mytbz2, >+ wantnewlockfile=1) >+ if self.trees[myroot]["bintree"].isremote(pkg_key): >+ msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ >+ (mergecount, totalcount, pkg_key, mytbz2) >+ short_msg = "emerge: (%s of %s) %s Fetch" % \ >+ (mergecount, totalcount, pkg_key) >+ emergelog(xterm_titles, msg, short_msg=short_msg) >+ if not self.trees[myroot]["bintree"].gettbz2( >+ pkg_key): >+ return 1 >+ finally: >+ if tbz2_lock: >+ portage_locks.unlockfile(tbz2_lock) >+ >+ if "--fetchonly" in self.myopts or \ >+ "--fetch-all-uri" in self.myopts: >+ return os.EX_OK >+ >+ short_msg = "emerge: ("+str(mergecount)+" of "+str(totalcount)+") "+x[pkgindex]+" Merge Binary" >+ emergelog(xterm_titles, " === ("+str(mergecount)+\ >+ " of "+str(totalcount)+") Merging Binary ("+\ >+ x[pkgindex]+"::"+mytbz2+")", short_msg=short_msg) >+ >+ retval = portage.pkgmerge(mytbz2, x[1], pkgsettings, >+ mydbapi=bindb, >+ vartree=self.trees[myroot]["vartree"], >+ prev_mtimes=ldpath_mtimes) >+ if retval != os.EX_OK: >+ return retval >+ #need to check for errors >+ if "--buildpkgonly" not in self.myopts: >+ self.trees[x[1]]["vartree"].inject(x[2]) >+ myfavkey=portage.cpv_getkey(x[2]) >+ if "--fetchonly" not in self.myopts and \ >+ "--fetch-all-uri" not in self.myopts and \ >+ myfavkey in favorites: >+ myfavs = portage.grabfile(os.path.join(myroot, portage.WORLD_FILE)) >+ myfavdict=genericdict(myfavs) >+ #don't record if already in system profile or already recorded >+ if (not mysysdict.has_key(myfavkey)) and (not myfavdict.has_key(myfavkey)): >+ #we don't have a favorites entry for this package yet; add one >+ myfavdict[myfavkey]=myfavkey >+ print ">>> Recording",myfavkey,"in \"world\" favorites file..." >+ emergelog(xterm_titles, " === ("+\ >+ str(mergecount)+" of "+\ >+ str(totalcount)+\ >+ ") Updating world file ("+x[pkgindex]+")") >+ portage.write_atomic( >+ os.path.join(myroot, portage.WORLD_FILE), >+ "\n".join(myfavdict.values())) >+ >+ if "--pretend" not in self.myopts and \ >+ "--fetchonly" not in self.myopts and \ >+ "--fetch-all-uri" not in self.myopts: >+ # Clean the old package that we have merged over top of it. >+ if pkgsettings.get("AUTOCLEAN", "yes") == "yes": >+ xsplit=portage.pkgsplit(x[2]) >+ emergelog(xterm_titles, " >>> AUTOCLEAN: " + xsplit[0]) >+ retval = unmerge(pkgsettings, self.myopts, vartree, >+ "clean", [xsplit[0]], ldpath_mtimes, autoclean=1) >+ if not retval: >+ emergelog(xterm_titles, >+ " --- AUTOCLEAN: Nothing unmerged.") >+ else: >+ portage.writemsg_stdout(colorize("WARN", "WARNING:") >+ + " AUTOCLEAN is disabled. This can cause serious" >+ + " problems due to overlapping packages.\n") >+ >+ >+ if "--pretend" not in self.myopts and \ >+ "--fetchonly" not in self.myopts and \ >+ "--fetch-all-uri" not in self.myopts: >+ if "noclean" not in self.settings.features: >+ short_msg = "emerge: (%s of %s) %s Clean Post" % \ >+ (mergecount, totalcount, x[pkgindex]) >+ emergelog(xterm_titles, (" === (%s of %s) " + \ >+ "Post-Build Cleaning (%s::%s)") % \ >+ (mergecount, totalcount, x[pkgindex], y), >+ short_msg=short_msg) >+ emergelog(xterm_titles, " ::: completed emerge ("+\ >+ str(mergecount)+" of "+str(totalcount)+") "+\ >+ x[2]+" to "+x[1]) >+ >+ return os.EX_OK >+ >+ def print_status(self, totalcount, donec, qsize, failedc): >+ print ">>> Jobs Total = "+colorize("WARN", str(totalcount))+" Done = "+\ >+ colorize("GOOD", str(donec))+" Running = "+colorize("BAD", str(qsize))+\ >+ " Failed = "+colorize("BAD", str(failedc)) >+ xtermTitle("Jobs Total = "+str(totalcount)+" Done = "+\ >+ str(donec)+" Running = "+str(qsize)+" Failed = "+str(failedc)) >+ >+ def merge(self, mylist, favorites, mtimedb, m_slots): > failed_fetches = [] > mymergelist=[] > ldpath_mtimes = mtimedb["ldpath"] > xterm_titles = "notitles" not in self.settings.features >+ parallel = "parallel" in self.settings.features >+ build_prefix=self.settings["PORTAGE_TMPDIR"]+"/portage" >+ >+ # parallel merge will be painful to watch with debug or fetchonly. So, you get only one of these...:-) >+ if self.edebug or "--fetchonly" in self.myopts: >+ parallel = False > > #check for blocking dependencies > if "--fetchonly" not in self.myopts and \ >@@ -2860,8 +3254,6 @@ > mysysdict = genericdict(getlist(self.settings, "system")) > if "--resume" in self.myopts: > # We're resuming. >- print colorize("GOOD", "*** Resuming merge...") >- emergelog(xterm_titles, " *** Resuming merge...") > mymergelist=mtimedb["resume"]["mergelist"][:] > if "--skipfirst" in self.myopts and mymergelist: > del mtimedb["resume"]["mergelist"][0] >@@ -2895,8 +3287,17 @@ > os.path.join(self.target_root, portage.WORLD_FILE), > "\n".join(myfavdict.values())) > >- mtimedb["resume"]["mergelist"]=mymergelist[:] >- mtimedb.commit() >+ if "--nodeps" not in self.myopts or len(mymergelist) > 1: >+ mtimedb["resume"]["mergelist"]=mymergelist[:] >+ mtimedb.commit() >+ >+ totalcount = len(mymergelist) >+ mergecount=1 >+ >+ if "--resume" in self.myopts and "--fetchonly" not in self.myopts: >+ # We're resuming. >+ print colorize("GOOD", "*** Resuming merge...") >+ emergelog(xterm_titles, " *** Resuming merge...") > > myfeat = self.settings.features[:] > bad_resume_opts = set(["--ask", "--tree", "--changelog", "--skipfirst", >@@ -2914,12 +3315,12 @@ > print ">>> starting parallel fetching" > fetch_log = "/var/log/emerge-fetch.log" > logfile = open(fetch_log, "w") >- fd_pipes = {1:logfile.fileno(), 2:logfile.fileno()} >+ fd_pipes = {0:0, 1:logfile.fileno(), 2:logfile.fileno()} > portage_util.apply_secpass_permissions(fetch_log, > uid=portage.portage_uid, gid=portage.portage_gid, > mode=0660) > fetch_env = os.environ.copy() >- fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs" >+ fetch_env["FEATURES"] = fetch_env.get("FEATURES", "") + " -cvs -parallel" > fetch_env["PORTAGE_NICENESS"] = "0" > fetch_args = [sys.argv[0], "--resume", "--fetchonly"] > for myopt, myarg in self.myopts.iteritems(): >@@ -2932,283 +3333,229 @@ > fd_pipes=fd_pipes, returnpid=True) > logfile.close() # belongs to the spawned process > >- mergecount=0 >- for x in mymergelist: >- mergecount+=1 >- myroot=x[1] >- pkg_key = x[2] >- pkgindex=2 >- portdb = self.trees[myroot]["porttree"].dbapi >- bindb = self.trees[myroot]["bintree"].dbapi >- vartree = self.trees[myroot]["vartree"] >- pkgsettings = self.pkgsettings[myroot] >- if x[0]=="blocks": >- pkgindex=3 >- y = portdb.findname(pkg_key) >- if "--pretend" not in self.myopts: >- print "\n>>> Emerging (" + \ >- colorize("MERGE_LIST_PROGRESS", str(mergecount)) + " of " + \ >- colorize("MERGE_LIST_PROGRESS", str(len(mymergelist))) + ") " + \ >- colorize("GOOD", x[pkgindex]) + " to " + x[1] >- emergelog(xterm_titles, " >>> emerge ("+\ >- str(mergecount)+" of "+str(len(mymergelist))+\ >- ") "+x[pkgindex]+" to "+x[1]) >+ if not parallel: >+ failed_fetches = [] >+ for x in mymergelist: >+ retcode = self.do_one_emerge(x, mergecount, totalcount, mtimedb, favorites, mysysdict) >+ mergecount += 1 >+ >+ # need to short circuit the spawn with --nodeps >+ if os.environ.get("PORTAGE_INTERNAL_CALL", "0") != "1": >+ if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: >+ continue >+ if retcode != os.EX_OK: >+ if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: >+ failed_fetches.append(x[2]) >+ continue >+ else: >+ return retcode >+ # Unsafe for parallel merges >+ del mtimedb["resume"]["mergelist"][0] >+ # Commit after each merge so that --resume may still work in >+ # in the event that portage is not allowed to exit normally >+ # due to power failure, SIGKILL, etc... >+ mtimedb.commit() >+ >+ # unlink the logid_path if any exists >+ logid_path = os.path.join(build_prefix, ".logid.")+x[2].split("/")[0]+"."+x[2].split("/")[1] >+ if os.path.exists(logid_path): >+ os.unlink(logid_path) >+ del logid_path > >- pkgsettings["EMERGE_FROM"] = x[0] >- pkgsettings.backup_changes("EMERGE_FROM") >- pkgsettings.reset() >+ # check if we need to restart portage >+ mysplit=portage.pkgsplit(x[2]) >+ if mysplit[0] == "sys-apps/portage" and x[1] == "/": >+ self.restart_portage(x, mergecount, totalcount, mtimedb) > >- #buildsyspkg: Check if we need to _force_ binary package creation >- issyspkg = ("buildsyspkg" in myfeat) \ >- and x[0] != "blocks" \ >- and mysysdict.has_key(portage.cpv_getkey(x[2])) \ >- and "--buildpkg" not in self.myopts >- if x[0] in ["ebuild","blocks"]: >- if x[0] == "blocks" and "--fetchonly" not in self.myopts: >- raise Exception, "Merging a blocker" >- elif "--fetchonly" in self.myopts or \ >- "--fetch-all-uri" in self.myopts: >- if "--fetch-all-uri" in self.myopts: >- retval = portage.doebuild(y, "fetch", myroot, >- pkgsettings, self.edebug, >- "--pretend" in self.myopts, fetchonly=1, >- fetchall=1, mydbapi=portdb, tree="porttree") >+ else: >+ if retcode != os.EX_OK: >+ sys.exit(1) > else: >- retval = portage.doebuild(y, "fetch", myroot, >- pkgsettings, self.edebug, >- "--pretend" in self.myopts, fetchonly=1, >- mydbapi=portdb, tree="porttree") >- if (retval is None) or retval: >- print >- print "!!! Fetch for",y,"failed, continuing..." >- print >- failed_fetches.append(pkg_key) >- continue >+ sys.exit(0) > >- portage.doebuild_environment(y, "setup", myroot, >- pkgsettings, self.edebug, 1, portdb) >- catdir = os.path.dirname(pkgsettings["PORTAGE_BUILDDIR"]) >- portage_util.ensure_dirs(os.path.dirname(catdir), >- uid=portage.portage_uid, gid=portage.portage_gid, >- mode=070, mask=0) >- builddir_lock = None >- catdir_lock = None >- try: >- catdir_lock = portage_locks.lockdir(catdir) >- portage_util.ensure_dirs(catdir, >- uid=portage.portage_uid, gid=portage.portage_gid, >- mode=070, mask=0) >- builddir_lock = portage_locks.lockdir( >- pkgsettings["PORTAGE_BUILDDIR"]) >- try: >- portage_locks.unlockdir(catdir_lock) >- finally: >- catdir_lock = None >- msg = " === (%s of %s) Cleaning (%s::%s)" % \ >- (mergecount, len(mymergelist), pkg_key, y) >- short_msg = "emerge: (%s of %s) %s Clean" % \ >- (mergecount, len(mymergelist), pkg_key) >- emergelog(xterm_titles, msg, short_msg=short_msg) >- retval = portage.doebuild(y, "clean", myroot, >- pkgsettings, self.edebug, cleanup=1, >- mydbapi=portdb, tree="porttree") >- if retval != os.EX_OK: >- return retval >- if "--buildpkg" in self.myopts or issyspkg: >- if issyspkg: >- print ">>> This is a system package, " + \ >- "let's pack a rescue tarball." >- msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ >- (mergecount, len(mymergelist), pkg_key, y) >- short_msg = "emerge: (%s of %s) %s Compile" % \ >- (mergecount, len(mymergelist), pkg_key) >- emergelog(xterm_titles, msg, short_msg=short_msg) >- self.trees[myroot]["bintree"].prevent_collision(pkg_key) >- retval = portage.doebuild(y, "package", myroot, >- pkgsettings, self.edebug, mydbapi=portdb, >- tree="porttree") >- if retval != os.EX_OK: >- return retval >- if "--buildpkgonly" not in self.myopts: >- bintree = self.trees[myroot]["bintree"] >- bintree.inject(pkg_key) >- mytbz2 = bintree.getname(pkg_key) >- msg = " === (%s of %s) Merging (%s::%s)" % \ >- (mergecount, len(mymergelist), pkg_key, y) >- short_msg = "emerge: (%s of %s) %s Merge" % \ >- (mergecount, len(mymergelist), pkg_key) >- emergelog(xterm_titles, msg, short_msg=short_msg) >- retval = portage.merge(pkgsettings["CATEGORY"], >- pkgsettings["PF"], pkgsettings["D"], >- os.path.join(pkgsettings["PORTAGE_BUILDDIR"], >- "build-info"), myroot, pkgsettings, >- myebuild=pkgsettings["EBUILD"], >- mytree="porttree", mydbapi=portdb, >- vartree=vartree, prev_mtimes=ldpath_mtimes) >- if retval != os.EX_OK: >- return retval >- elif "noclean" not in pkgsettings.features: >- portage.doebuild(y, "clean", myroot, >- pkgsettings, self.edebug, mydbapi=portdb, >- tree="porttree") >- else: >- msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ >- (mergecount, len(mymergelist), pkg_key, y) >- short_msg = "emerge: (%s of %s) %s Compile" % \ >- (mergecount, len(mymergelist), pkg_key) >- emergelog(xterm_titles, msg, short_msg=short_msg) >- retval = portage.doebuild(y, "merge", myroot, >- pkgsettings, self.edebug, vartree=vartree, >- mydbapi=portdb, tree="porttree", >- prev_mtimes=ldpath_mtimes) >- if retval != os.EX_OK: >- return retval >- finally: >- if builddir_lock: >- portage_locks.unlockdir(builddir_lock) >- try: >- if not catdir_lock: >- # Lock catdir for removal if empty. >- catdir_lock = portage_locks.lockdir(catdir) >- finally: >- if catdir_lock: >- try: >- os.rmdir(catdir) >- except OSError, e: >- if e.errno not in (errno.ENOENT, >- errno.ENOTEMPTY, errno.EEXIST): >- raise >- del e >- portage_locks.unlockdir(catdir_lock) >- >- elif x[0]=="binary": >- #merge the tbz2 >- mytbz2 = self.trees[myroot]["bintree"].getname(pkg_key) >- if "--getbinpkg" in self.myopts: >- tbz2_lock = None >- try: >- if "distlocks" in pkgsettings.features and \ >- os.access(pkgsettings["PKGDIR"], os.W_OK): >- portage_util.ensure_dirs(os.path.dirname(mytbz2)) >- tbz2_lock = portage_locks.lockfile(mytbz2, >- wantnewlockfile=1) >- if self.trees[myroot]["bintree"].isremote(pkg_key): >- msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ >- (mergecount, len(mymergelist), pkg_key, mytbz2) >- short_msg = "emerge: (%s of %s) %s Fetch" % \ >- (mergecount, len(mymergelist), pkg_key) >- emergelog(xterm_titles, msg, short_msg=short_msg) >- if not self.trees[myroot]["bintree"].gettbz2( >- pkg_key): >- return 1 >- finally: >- if tbz2_lock: >- portage_locks.unlockfile(tbz2_lock) >+ if "--pretend" not in self.myopts: >+ emergelog(xterm_titles, " *** Finished. Cleaning up...") > >- if "--fetchonly" in self.myopts or \ >- "--fetch-all-uri" in self.myopts: >- continue >+ # We're out of the loop... We're done. Delete the resume data. >+ if mtimedb.has_key("resume"): >+ del mtimedb["resume"] >+ mtimedb.commit() >+ >+ #by doing an exit this way, --fetchonly can continue to try to >+ #fetch everything even if a particular download fails. >+ if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: >+ if failed_fetches: >+ sys.stderr.write("\n\n!!! Some fetch errors were " + \ >+ "encountered. Please see above for details.\n\n") >+ for cpv in failed_fetches: >+ sys.stderr.write(" ") >+ sys.stderr.write(cpv) >+ sys.stderr.write("\n") >+ sys.stderr.write("\n") >+ sys.exit(1) >+ else: >+ sys.exit(0) >+ return os.EX_OK > >- short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Merge Binary" >- emergelog(xterm_titles, " === ("+str(mergecount)+\ >- " of "+str(len(mymergelist))+") Merging Binary ("+\ >- x[pkgindex]+"::"+mytbz2+")", short_msg=short_msg) >- retval = portage.pkgmerge(mytbz2, x[1], pkgsettings, >- mydbapi=bindb, >- vartree=self.trees[myroot]["vartree"], >- prev_mtimes=ldpath_mtimes) >- if retval != os.EX_OK: >- return retval >- #need to check for errors >- if "--buildpkgonly" not in self.myopts: >- self.trees[x[1]]["vartree"].inject(x[2]) >- myfavkey=portage.cpv_getkey(x[2]) >- if "--fetchonly" not in self.myopts and \ >- "--fetch-all-uri" not in self.myopts and \ >- myfavkey in favorites: >- myfavs = portage.grabfile(os.path.join(myroot, portage.WORLD_FILE)) >- myfavdict=genericdict(myfavs) >- #don't record if already in system profile or already recorded >- if (not mysysdict.has_key(myfavkey)) and (not myfavdict.has_key(myfavkey)): >- #we don't have a favorites entry for this package yet; add one >- myfavdict[myfavkey]=myfavkey >- print ">>> Recording",myfavkey,"in \"world\" favorites file..." >- emergelog(xterm_titles, " === ("+\ >- str(mergecount)+" of "+\ >- str(len(mymergelist))+\ >- ") Updating world file ("+x[pkgindex]+")") >- portage.write_atomic( >- os.path.join(myroot, portage.WORLD_FILE), >- "\n".join(myfavdict.values())) >- >- if "--pretend" not in self.myopts and \ >- "--fetchonly" not in self.myopts and \ >- "--fetch-all-uri" not in self.myopts: >- # Clean the old package that we have merged over top of it. >- if pkgsettings.get("AUTOCLEAN", "yes") == "yes": >- xsplit=portage.pkgsplit(x[2]) >- emergelog(xterm_titles, " >>> AUTOCLEAN: " + xsplit[0]) >- retval = unmerge(pkgsettings, self.myopts, vartree, >- "clean", [xsplit[0]], ldpath_mtimes, autoclean=1) >- if not retval: >- emergelog(xterm_titles, >- " --- AUTOCLEAN: Nothing unmerged.") >- else: >- portage.writemsg_stdout(colorize("WARN", "WARNING:") >- + " AUTOCLEAN is disabled. This can cause serious" >- + " problems due to overlapping packages.\n") >+ # parallel code - dirty starts here...;-) >+ one_in_slot_failed=0 >+ spawnd_pids=[] >+ >+ # dirty little trick to get number of cpus from the system >+ fd_cpuinfo = os.popen("cat /proc/cpuinfo","r") >+ cpu_count = 0 >+ for data_cpuinfo in fd_cpuinfo.readlines(): >+ if data_cpuinfo.find("cpu MHz") > -1 : >+ cpu_count += 1 >+ fd_cpuinfo.close() >+ >+ # if someone really screwed with /proc/cpuinfo output, we should not suffer >+ if cpu_count == 0: >+ cpu_count = 1 >+ >+ spawnd_pkg = {} >+ donec = 0 >+ failedc = 0 >+ failedPkgs = [] >+ logid_path = None >+ mylist = m_slots.keys() >+ mylist.sort() >+ for x in mylist: >+ # if slot is empty, go on >+ if not m_slots[x]: >+ continue > >- # Figure out if we need a restart. >- mysplit=portage.pkgsplit(x[2]) >- if mysplit[0] == "sys-apps/portage" and x[1] == "/": >- myver=mysplit[1]+"-"+mysplit[2] >- if myver[-3:]=='-r0': >- myver=myver[:-3] >- if (myver != portage.VERSION) and \ >- "livecvsportage" not in self.settings.features: >- if len(mymergelist) > mergecount: >- emergelog(xterm_titles, >- " ::: completed emerge ("+ \ >- str(mergecount)+" of "+ \ >- str(len(mymergelist))+") "+ \ >- x[2]+" to "+x[1]) >- emergelog(xterm_titles, " *** RESTARTING " + \ >- "emerge via exec() after change of " + \ >- "portage version.") >- del mtimedb["resume"]["mergelist"][0] >+ # if previous slot failed, discontinue the emerge >+ if one_in_slot_failed and not ("--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts): >+ break >+ >+ # start multiple merges in parallel mode >+ num_at_atime = cpu_count + 1 >+ >+ qsize = 0 >+ for y in m_slots[x]: >+ # these all can go in parallel, so fork one after the other >+ # but num_at_atime at most >+ if num_at_atime: >+ onepid = self.fork_one_emerge(y, mergecount, totalcount, mtimedb, favorites, mysysdict) >+ spawnd_pids.append(onepid) >+ spawnd_pkg[onepid] = (y[2], x) >+ num_at_atime -= 1 >+ mergecount += 1 >+ qsize += 1 >+ else: >+ self.print_status(totalcount, donec, qsize, failedc) >+ # let's wait for one of the jobs to finish >+ onepid = -1 >+ while onepid not in spawnd_pids: >+ onepid , retval = os.waitpid(-1, 0) >+ spawnd_pids.remove(onepid) >+ >+ pkg_compl = spawnd_pkg[onepid][0] >+ pkg_slot = spawnd_pkg[onepid][1] >+ >+ # if it failed, I need to fail next slot but continue to merge all in this slot >+ if retval: >+ one_in_slot_failed = retval >+ failedc += 1 >+ failedPkgs.append(pkg_compl) >+ else: >+ # unlink the logid_path >+ logid_path = os.path.join(build_prefix, ".logid.")+pkg_compl.split("/")[0]+"."+pkg_compl.split("/")[1] >+ if os.path.exists(logid_path): >+ os.unlink(logid_path) >+ donec += 1 >+ index = 0 >+ print ">>> Package "+colorize("GOOD", pkg_compl)+" finished emerging." >+ # we need to remove this pkg from resume DB >+ # this is the dirtiest shit I have ever written >+ for pkgs in mymergelist: >+ if pkgs[2] == pkg_compl: >+ del mymergelist[index] >+ del mtimedb["resume"]["mergelist"][index] > mtimedb.commit() >- portage.run_exitfuncs() >- mynewargv=[sys.argv[0],"--resume"] >- for myopt, myarg in self.myopts.iteritems(): >- if myopt not in bad_resume_opts: >- if myarg is True: >- mynewargv.append(myopt) >- else: >- mynewargv.append(myopt +"="+ myarg) >- # priority only needs to be adjusted on the first run >- os.environ["PORTAGE_NICENESS"] = "0" >- os.execv(mynewargv[0], mynewargv) >+ # check if we need to restart portage >+ mysplit=portage.pkgsplit(pkg_compl) >+ if mysplit[0] == "sys-apps/portage" and pkgs[1] == "/": >+ self.restart_portage(pkgs, mergecount, totalcount, mtimedb) >+ break >+ index += 1 >+ onepid = self.fork_one_emerge(y, mergecount, totalcount, mtimedb, favorites, mysysdict) >+ spawnd_pids.append(onepid) >+ spawnd_pkg[onepid] = (y[2], x) >+ mergecount += 1 >+ >+ # this slot is exhausted, so wait for all of the forks to finish >+ while spawnd_pids: >+ self.print_status(totalcount, donec, qsize, failedc) >+ onepid = -1 >+ while onepid not in spawnd_pids: >+ onepid , retval = os.waitpid(-1, 0) >+ spawnd_pids.remove(onepid) >+ pkg_compl = spawnd_pkg[onepid][0] >+ pkg_slot = spawnd_pkg[onepid][1] >+ >+ qsize -= 1 >+ if retval: >+ one_in_slot_failed = retval >+ failedc += 1 >+ failedPkgs.append(pkg_compl) >+ else: >+ # unlink the logid_path >+ logid_path = os.path.join(build_prefix, ".logid.")+pkg_compl.split("/")[0]+"."+pkg_compl.split("/")[1] >+ if os.path.exists(logid_path): >+ os.unlink(logid_path) >+ donec += 1 >+ index = 0 >+ print ">>> Package "+colorize("GOOD", pkg_compl)+" finished emerging." >+ # we need to remove this pkg from resume DB >+ # this is the dirtiest shit I have ever written >+ for pkgs in mymergelist: >+ if pkgs[2] == pkg_compl: >+ del mymergelist[index] >+ del mtimedb["resume"]["mergelist"][index] >+ mtimedb.commit() >+ # check if we need to restart portage >+ mysplit=portage.pkgsplit(pkg_compl) >+ if mysplit[0] == "sys-apps/portage" and pkgs[1] == "/": >+ self.restart_portage(pkgs, mergecount, totalcount, mtimedb) >+ break >+ index += 1 > >- if "--pretend" not in self.myopts and \ >- "--fetchonly" not in self.myopts and \ >- "--fetch-all-uri" not in self.myopts: >- if "noclean" not in self.settings.features: >- short_msg = "emerge: (%s of %s) %s Clean Post" % \ >- (mergecount, len(mymergelist), x[pkgindex]) >- emergelog(xterm_titles, (" === (%s of %s) " + \ >- "Post-Build Cleaning (%s::%s)") % \ >- (mergecount, len(mymergelist), x[pkgindex], y), >- short_msg=short_msg) >- emergelog(xterm_titles, " ::: completed emerge ("+\ >- str(mergecount)+" of "+str(len(mymergelist))+") "+\ >- x[2]+" to "+x[1]) >+ self.print_status(totalcount, donec, qsize, failedc) > >- # Unsafe for parallel merges >- del mtimedb["resume"]["mergelist"][0] >- # Commit after each merge so that --resume may still work in >- # in the event that portage is not allowed to exit normally >- # due to power failure, SIGKILL, etc... >- mtimedb.commit() >+ if one_in_slot_failed: >+ portage.writemsg_stdout(red("\nSome packages failed to emerge, summary follows:\n")) >+ >+ for pkgs in failedPkgs: >+ if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: >+ print "\n\n!!! Some fetch errors were encountered. Please see above for details.\n\n" >+ sys.exit(1) >+ >+ portage.writemsg_stdout(red("\nPackage "+pkgs+" failed to emerge\n")) >+ logfile = None >+ if "PORT_LOGDIR" in self.settings: >+ port_logdir = self.settings["PORT_LOGDIR"] >+ else: >+ port_logdir = self.settings["ROOT"] + portage.DEF_LOGDIR >+ >+ pkg_cat = pkgs.split("/")[0] >+ pkg_pf = pkgs.split("/")[1] >+ logid_path = os.path.join(build_prefix, ".logid.")+pkg_cat+"."+pkg_pf >+ if os.path.exists(logid_path): >+ logid_time = time.strftime("%Y%m%d-%H%M%S", time.gmtime(os.stat(logid_path).st_mtime)) >+ logfile = os.path.join(port_logdir, "%s:%s:%s.log" % \ >+ (pkg_cat, pkg_pf, logid_time)) >+ del logid_time >+ >+ if logfile and os.path.exists(logfile): >+ portage.portage_exec.spawn(('tail', '-n', '20', logfile), returnpid=False) >+ >+ if logfile and os.path.exists(logfile): >+ portage.writemsg_stdout(red("Please take a look at the file "+logfile+"\n")) >+ os.unlink(logid_path) >+ if one_in_slot_failed: >+ sys.exit(1) > > if "--pretend" not in self.myopts: > emergelog(xterm_titles, " *** Finished. Cleaning up...") >@@ -3218,20 +3565,9 @@ > del mtimedb["resume"] > mtimedb.commit() > >- #by doing an exit this way, --fetchonly can continue to try to >- #fetch everything even if a particular download fails. > if "--fetchonly" in self.myopts or "--fetch-all-uri" in self.myopts: >- if failed_fetches: >- sys.stderr.write("\n\n!!! Some fetch errors were " + \ >- "encountered. Please see above for details.\n\n") >- for cpv in failed_fetches: >- sys.stderr.write(" ") >- sys.stderr.write(cpv) >- sys.stderr.write("\n") >- sys.stderr.write("\n") >- sys.exit(1) >- else: >- sys.exit(0) >+ sys.exit(0) >+ > return os.EX_OK > > def unmerge(settings, myopts, vartree, unmerge_action, unmerge_files, >@@ -3672,7 +4008,7 @@ > def validate_merge_list(trees, mergelist): > """Validate the list to make sure all the packages are still available. > This is needed for --resume.""" >- for (pkg_type, myroot, pkg_key, action) in mergelist: >+ for (pkg_type, myroot, pkg_key, action, merge_slot) in mergelist: > if pkg_type == "binary" and \ > not trees[myroot]["bintree"].dbapi.match("="+pkg_key) or \ > pkg_type == "ebuild" and \ >@@ -4514,7 +4850,7 @@ > if not "--pretend" in myopts: #just check pretend, since --ask implies pretend > emergelog(xterm_titles, " >>> depclean") > >- if "--quiet" not in myopts: >+ if "--quiet" not in myopts and "--nodeps" not in myopts: > print "\nCalculating dependencies ", > > soft = 0 >@@ -4611,6 +4947,38 @@ > else: > print "Number removed: "+str(len(cleanlist)) > >+def mergelist_to_merge_slot(mergelist, myopts, printonly=False, parallel=False): >+ merge_slots = {} >+ for pkg in mergelist: >+ if pkg[0] != 'blocks' and pkg[3] == 'merge': >+ slot = int(pkg[4]) >+ try: >+ if pkg not in merge_slots[slot]: >+ merge_slots[slot].append(pkg) >+ except KeyError: >+ merge_slots[slot] = [pkg] >+ # print the merge slots >+ max_slot = 0 >+ mylist = merge_slots.keys() >+ mylist.sort() >+ for x in mylist: >+ if x > max_slot: >+ max_slot = x >+ if parallel: >+ print "Package list for slot = "+str(x) >+ for y in merge_slots[x]: >+ print " ",y >+ if printonly: >+ return >+ >+ # make one last pass at the merge_slots and initialize the missing slots to None >+ x = 0 >+ while x < max_slot: >+ if x not in merge_slots.keys(): >+ merge_slots[x] = None >+ x += 1 >+ return merge_slots >+ > def action_build(settings, trees, mtimedb, > myopts, myaction, myfiles, spinner): > ldpath_mtimes = mtimedb["ldpath"] >@@ -4658,11 +5026,13 @@ > myresumeopts[myopt] = myarg > myopts=myresumeopts > myparams = create_depgraph_params(myopts, myaction) >- if not "--quiet" in myopts: >+ if not "--quiet" in myopts and "--nodeps" not in myopts: > print "Calculating dependencies ", >+ if "--nodeps" in myopts: >+ spinner.update = spinner.update_quiet > mydepgraph = depgraph(settings, trees, > myopts, myparams, spinner) >- if not "--quiet" in myopts: >+ if not "--quiet" in myopts and "--nodeps" not in myopts: > print "\b\b... done!" > else: > if ("--resume" in myopts): >@@ -4671,19 +5041,23 @@ > > myparams = create_depgraph_params(myopts, myaction) > if myaction in ["system","world"]: >- if not ("--quiet" in myopts): >+ if not ("--quiet" in myopts) and "--nodeps" not in myopts: > print "Calculating",myaction,"dependencies ", > sys.stdout.flush() >+ if "--nodeps" in myopts: >+ spinner.update = spinner.update_quiet > mydepgraph = depgraph(settings, trees, myopts, myparams, spinner) > if not mydepgraph.xcreate(myaction): > print "!!! Depgraph creation failed." > sys.exit(1) >- if not ("--quiet" in myopts): >+ if not ("--quiet" in myopts) and "--nodeps" not in myopts: > print "\b\b... done!" > else: >- if not ("--quiet" in myopts): >+ if not ("--quiet" in myopts) and "--nodeps" not in myopts: > print "Calculating dependencies ", > sys.stdout.flush() >+ if "--nodeps" in myopts: >+ spinner.update = spinner.update_quiet > mydepgraph = depgraph(settings, trees, myopts, myparams, spinner) > try: > retval, favorites = mydepgraph.select_files(myfiles) >@@ -4692,7 +5066,7 @@ > sys.exit(1) > if not retval: > sys.exit(1) >- if not ("--quiet" in myopts): >+ if not ("--quiet" in myopts) and "--nodeps" not in myopts: > print "\b\b... done!" > > if ("--usepkgonly" in myopts) and mydepgraph.missingbins: >@@ -4713,13 +5087,15 @@ > if len(mymergelist) == 0: > print colorize("INFORM", "emerge: It seems we have nothing to resume...") > sys.exit(0) >+ mergelist_to_merge_slot(mymergelist, myopts, True, "parallel" in settings.features) > mydepgraph.display(mymergelist) > prompt="Would you like to resume merging these packages?" > else: >- mydepgraph.display( >- mydepgraph.altlist(reversed=("--tree" in myopts))) >+ mymergelist = mydepgraph.altlist(reversed=("--tree" in myopts)) >+ mergelist_to_merge_slot(mymergelist, myopts, True, "parallel" in settings.features) >+ mydepgraph.display(mymergelist) > mergecount=0 >- for x in mydepgraph.altlist(): >+ for x in mymergelist: > if x[0] != "blocks" and x[3] != "nomerge": > mergecount+=1 > #check for blocking dependencies >@@ -4764,10 +5140,13 @@ > if len(mymergelist) == 0: > print colorize("INFORM", "emerge: It seems we have nothing to resume...") > sys.exit(0) >+ mergelist_to_merge_slot(mymergelist, myopts, True, "parallel" in settings.features) > mydepgraph.display(mymergelist) > else: >- mydepgraph.display( >- mydepgraph.altlist(reversed=("--tree" in myopts))) >+ # mydepgraph.digraph.debug_print() >+ mymergelist = mydepgraph.altlist(reversed=("--tree" in myopts)) >+ mergelist_to_merge_slot(mymergelist, myopts, True, "parallel" in settings.features) >+ mydepgraph.display(mymergelist) > else: > if ("--buildpkgonly" in myopts): > if not mydepgraph.digraph.hasallzeros(ignore_priority=DepPriority.MEDIUM): >@@ -4783,23 +5162,27 @@ > it to write the mtimedb""" > mtimedb.filename = None > time.sleep(3) # allow the parent to have first fetch >+ mymergelist = mtimedb["resume"]["mergelist"] >+ merge_slots = mergelist_to_merge_slot(mymergelist, myopts, False, False) > del mydepgraph >- retval = mergetask.merge( >- mtimedb["resume"]["mergelist"], favorites, mtimedb) >+ retval = mergetask.merge(mymergelist, favorites, mtimedb, merge_slots) > if retval != os.EX_OK: > sys.exit(retval) > else: >- if "resume" in mtimedb and \ >- "mergelist" in mtimedb["resume"] and \ >- len(mtimedb["resume"]["mergelist"]) > 1: >- mtimedb["resume_backup"] = mtimedb["resume"] >- del mtimedb["resume"] >- mtimedb.commit() >- mtimedb["resume"]={} >- # XXX: Stored as a list for backward compatibility. >- mtimedb["resume"]["myopts"] = \ >- [k for k in myopts if myopts[k] is True] >- mtimedb["resume"]["favorites"]=favorites >+ mymergelist = mydepgraph.altlist() >+ merge_slots = mergelist_to_merge_slot(mymergelist, myopts, False, False) >+ if "--nodeps" not in myopts or len(mymergelist) > 1: >+ if "resume" in mtimedb and \ >+ "mergelist" in mtimedb["resume"] and \ >+ len(mtimedb["resume"]["mergelist"]) > 1: >+ mtimedb["resume_backup"] = mtimedb["resume"] >+ del mtimedb["resume"] >+ mtimedb.commit() >+ mtimedb["resume"]={} >+ # XXX: Stored as a list for backward compatibility. >+ mtimedb["resume"]["myopts"] = \ >+ [k for k in myopts if myopts[k] is True] >+ mtimedb["resume"]["favorites"]=favorites > if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts): > for pkgline in mydepgraph.altlist(): > if pkgline[0]=="ebuild" and pkgline[3]=="merge": >@@ -4815,18 +5198,18 @@ > tree="porttree") > if "--fetchonly" in myopts or "--fetch-all-uri" in myopts: > pkglist = [] >- for pkg in mydepgraph.altlist(): >+ for pkg in mymergelist: > if pkg[0] != "blocks": > pkglist.append(pkg) > else: >- pkglist = mydepgraph.altlist() >+ pkglist = mymergelist > del mydepgraph > mergetask = MergeTask(settings, trees, myopts) >- retval = mergetask.merge(pkglist, favorites, mtimedb) >+ retval = mergetask.merge(pkglist, favorites, mtimedb, merge_slots) > if retval != os.EX_OK: > sys.exit(retval) > >- if mtimedb.has_key("resume"): >+ if mtimedb.has_key("resume") and ("--nodeps" not in myopts or len(mymergelist) > 1): > del mtimedb["resume"] > if settings["AUTOCLEAN"] and "yes"==settings["AUTOCLEAN"]: > print ">>> Auto-cleaning packages..." >diff -ru portage-2.1.2.orig/pym/portage.py portage-2.1.2/pym/portage.py >--- portage-2.1.2.orig/pym/portage.py 2007-02-23 13:06:11.000000000 -0800 >+++ portage-2.1.2/pym/portage.py 2007-02-23 13:07:42.000000000 -0800 >@@ -64,7 +64,7 @@ > from output import bold, colorize, green, red, yellow > > import portage_const >- from portage_const import VDB_PATH, PRIVATE_PATH, CACHE_PATH, DEPCACHE_PATH, \ >+ from portage_const import VDB_PATH, PRIVATE_PATH, DEF_LOGDIR, CACHE_PATH, DEPCACHE_PATH, \ > USER_CONFIG_PATH, MODULES_FILE_PATH, CUSTOM_PROFILE_PATH, PORTAGE_BASE_PATH, \ > PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, PROFILE_PATH, LOCALE_DATA_PATH, \ > EBUILD_SH_BINARY, SANDBOX_BINARY, BASH_BINARY, \ >@@ -2185,19 +2185,32 @@ > mypids = [] > pw = None > if logfile: >- del keywords["logfile"] >+ tee_good = 1 > fd_pipes = keywords.get("fd_pipes") > if fd_pipes is None: > fd_pipes = {0:0, 1:1, 2:2} > elif 1 not in fd_pipes or 2 not in fd_pipes: > raise ValueError(fd_pipes) >- pr, pw = os.pipe() >- mypids.extend(portage_exec.spawn(('tee', '-i', '-a', logfile), >- returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) >- os.close(pr) >- fd_pipes[1] = pw >- fd_pipes[2] = pw >- keywords["fd_pipes"] = fd_pipes >+ try: >+ statinfo1 = os.stat(logfile) >+ statinfo2 = os.fstat(fd_pipes[1]) >+ statinfo3 = os.fstat(fd_pipes[2]) >+ # if they are pointing to same file as logfile, no 'tee' is required. >+ if statinfo1 == statinfo2 and statinfo2 == statinfo3: >+ tee_good = 0 >+ except: >+ tee_good = 1 >+ >+ if tee_good: >+ >+ del keywords["logfile"] >+ pr, pw = os.pipe() >+ mypids.extend(portage_exec.spawn(('tee', '-i', '-a', logfile), >+ returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) >+ os.close(pr) >+ fd_pipes[1] = pw >+ fd_pipes[2] = pw >+ keywords["fd_pipes"] = fd_pipes > > features = mysettings.features > # XXX: Negative RESTRICT word >@@ -3210,6 +3223,10 @@ > if mysettings.get("PORT_LOGDIR", "") == "": > while "PORT_LOGDIR" in mysettings: > del mysettings["PORT_LOGDIR"] >+ >+ if not "PORT_LOGDIR" in mysettings and "parallel" in mysettings.features: >+ mysettings["PORT_LOGDIR"] = mysettings["ROOT"] + portage.DEF_LOGDIR >+ > if "PORT_LOGDIR" in mysettings: > try: > portage_util.ensure_dirs(mysettings["PORT_LOGDIR"], >@@ -3222,7 +3239,7 @@ > while "PORT_LOGDIR" in mysettings: > del mysettings["PORT_LOGDIR"] > if "PORT_LOGDIR" in mysettings: >- logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid") >+ logid_path = os.path.join(mysettings["BUILD_PREFIX"], ".logid.")+mysettings["CATEGORY"]+"."+ mysettings["PF"] > if not os.path.exists(logid_path): > f = open(logid_path, "w") > f.close() >diff -ru portage-2.1.2.orig/pym/portage_const.py portage-2.1.2/pym/portage_const.py >--- portage-2.1.2.orig/pym/portage_const.py 2007-02-23 13:06:11.000000000 -0800 >+++ portage-2.1.2/pym/portage_const.py 2007-02-23 13:07:42.000000000 -0800 >@@ -12,6 +12,7 @@ > > VDB_PATH = "var/db/pkg" > PRIVATE_PATH = "var/lib/portage" >+DEF_LOGDIR = "var/log/portage" > CACHE_PATH = "/var/cache/edb" > DEPCACHE_PATH = CACHE_PATH+"/dep" > >diff -ru portage-2.1.2.orig/pym/portage_exec.py portage-2.1.2/pym/portage_exec.py >--- portage-2.1.2.orig/pym/portage_exec.py 2007-01-11 20:42:51.000000000 -0800 >+++ portage-2.1.2/pym/portage_exec.py 2007-02-23 13:07:42.000000000 -0800 >@@ -19,6 +19,8 @@ > > if os.path.isdir("/proc/%i/fd" % os.getpid()): > def get_open_fds(): >+ # there is a race here - fd used by listdir may be in the list but closed >+ # before this method returns. > return map(int, [fd for fd in os.listdir("/proc/%i/fd" % os.getpid()) if fd.isdigit()]) > else: > def get_open_fds(): >@@ -167,27 +169,41 @@ > # mypids will hold the pids of all processes created. > mypids = [] > >+ pw = None > if logfile: > # Using a log file requires that stdout and stderr > # are assigned to the process we're running. > if 1 not in fd_pipes or 2 not in fd_pipes: > raise ValueError(fd_pipes) > >- # Create a pipe >- (pr, pw) = os.pipe() >- >- # Create a tee process, giving it our stdout and stderr >- # as well as the read end of the pipe. >- mypids.extend(spawn(('tee', '-i', '-a', logfile), >- returnpid=True, fd_pipes={0:pr, >- 1:fd_pipes[1], 2:fd_pipes[2]})) >- >- # We don't need the read end of the pipe, so close it. >- os.close(pr) >- >- # Assign the write end of the pipe to our stdout and stderr. >- fd_pipes[1] = pw >- fd_pipes[2] = pw >+ tee_good = 1 >+ try: >+ statinfo1 = os.stat(logfile) >+ statinfo2 = os.fstat(fd_pipes[1]) >+ statinfo3 = os.fstat(fd_pipes[2]) >+ # if they are pointing to same file as logfile, no 'tee' is required. >+ if statinfo1 == statinfo2 and statinfo2 == statinfo3: >+ tee_good = 0 >+ except: >+ tee_good = 1 >+ >+ if tee_good: >+ >+ # Create a pipe >+ (pr, pw) = os.pipe() >+ >+ # Create a tee process, giving it our stdout and stderr >+ # as well as the read end of the pipe. >+ mypids.extend(spawn(('tee', '-i', '-a', logfile), >+ returnpid=True, fd_pipes={0:pr, >+ 1:fd_pipes[1], 2:fd_pipes[2]})) >+ >+ # We don't need the read end of the pipe, so close it. >+ os.close(pr) >+ >+ # Assign the write end of the pipe to our stdout and stderr. >+ fd_pipes[1] = pw >+ fd_pipes[2] = pw > > pid = os.fork() > >@@ -209,7 +225,7 @@ > > # If we started a tee process the write side of the pipe is no > # longer needed, so close it. >- if logfile: >+ if logfile and pw: > os.close(pw) > > # If the caller wants to handle cleaning up the processes, we tell
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
Attachments on
bug 147516
:
96922
|
96923
|
97020
|
97056
|
111065
|
111081
|
111086
|
111095
|
111106
|
111136
|
111434
|
111724
|
114444
|
114501
|
115212
|
115395
|
115986
|
121259
|
121347
|
121467
|
126839
|
140211
|
142083
|
142841
|
143576
|
144689
|
160298
|
160299
|
160301