Attachment #386860 for bug #525718

Lines 57-63 from portage.util._async.run_main_scheduler import run_main_scheduler Link Here

(-)a/bin/egencache (-2 / +41 lines)
57	from portage.util._eventloop.global_event_loop import global_event_loop	57	from portage.util._eventloop.global_event_loop import global_event_loop
58	from portage import cpv_getkey	58	from portage import cpv_getkey
59	from portage.dep import Atom, isjustname	59	from portage.dep import Atom, isjustname
60	from portage.versions import pkgsplit, vercmp	60	from portage.versions import pkgsplit, vercmp, _pkg_str
61		61
62	try:	62	try:
63	from xml.etree import ElementTree	63	from xml.etree import ElementTree
Lines 91-96 def parse_args(args): Link Here
91	actions.add_argument("--update-changelogs",	91	actions.add_argument("--update-changelogs",
92	action="store_true",	92	action="store_true",
93	help="update the ChangeLog files from SCM logs")	93	help="update the ChangeLog files from SCM logs")
		94	actions.add_argument("--update-pkg-desc-index",
		95	action="store_true",
		96	help="update package description index")
94	actions.add_argument("--update-manifests",	97	actions.add_argument("--update-manifests",
95	action="store_true",	98	action="store_true",
96	help="update manifests")	99	help="update manifests")
Lines 451-456 class GenCache(object): Link Here
451	if hasattr(trg_cache, '_prune_empty_dirs'):	454	if hasattr(trg_cache, '_prune_empty_dirs'):
452	trg_cache._prune_empty_dirs()	455	trg_cache._prune_empty_dirs()
453		456
		457	class GenPkgDescIndex(object):
		458	def __init__(self, portdb, output_file):
		459	self.returncode = os.EX_OK
		460	self._portdb = portdb
		461	self._output_file = output_file
		462
		463	def run(self):
		464
		465	portage.util.ensure_dirs(os.path.dirname(self._output_file))
		466	f = portage.util.atomic_ofstream(self._output_file,
		467	encoding=_encodings["repo.content"])
		468
		469	portdb = self._portdb
		470	for cp in portdb.cp_all():
		471	pkgs = portdb.cp_list(cp)
		472	if not pkgs:
		473	continue
		474	desc, = portdb.aux_get(pkgs[-1], ["DESCRIPTION"])
		475
		476	if len(pkgs) == 1:
		477	output = "%s: %s\n" % (pkgs[0], desc)
		478	else:
		479	output = "%s,%s: %s\n" % (pkgs[0],
		480	",".join(_pkg_str(cpv).version
		481	for cpv in pkgs[1:]), desc)
		482	f.write(output)
		483
		484	f.close()
		485
454	class GenUseLocalDesc(object):	486	class GenUseLocalDesc(object):
455	def __init__(self, portdb, output=None,	487	def __init__(self, portdb, output=None,
456	preserve_comments=False):	488	preserve_comments=False):
Lines 893-899 def egencache_main(args): Link Here
893	local_config=False, env=env)	925	local_config=False, env=env)
894		926
895	if not (options.update or options.update_use_local_desc or	927	if not (options.update or options.update_use_local_desc or
896	options.update_changelogs or options.update_manifests):	928	options.update_changelogs or options.update_manifests or
		929	options.update_pkg_desc_index):
897	parser.error('No action specified')	930	parser.error('No action specified')
898	return 1	931	return 1
899		932
Lines 1057-1062 def egencache_main(args): Link Here
1057	else:	1090	else:
1058	ret.append(scheduler.returncode)	1091	ret.append(scheduler.returncode)
1059		1092
		1093	if options.update_pkg_desc_index:
		1094	gen_index = GenPkgDescIndex(portdb, os.path.join(
		1095	repo_config.location, "metadata", "pkg_desc_index"))
		1096	gen_index.run()
		1097	ret.append(gen_index.returncode)
		1098
1060	if options.update_use_local_desc:	1099	if options.update_use_local_desc:
1061	gen_desc = GenUseLocalDesc(portdb,	1100	gen_desc = GenUseLocalDesc(portdb,
1062	output=options.uld_output,	1101	output=options.uld_output,

Lines 19-24 for the details on package atom syntax. Link Here

(-)a/man/egencache.1 (+4 lines)
19	.BR "\-\-update\-changelogs"	19	.BR "\-\-update\-changelogs"
20	Update the ChangeLog files from SCM logs (supported only in git repos).	20	Update the ChangeLog files from SCM logs (supported only in git repos).
21	.TP	21	.TP
		22	.BR "\-\-update\-pkg\-desc\-index"
		23	Update the package description index which is located at
		24	\fImetadata/pkg_desc_index\fR in the repository.
		25	.TP
22	.BR "\-\-update\-use\-local\-desc"	26	.BR "\-\-update\-use\-local\-desc"
23	Update the \fIprofiles/use.local.desc\fR file from metadata.xml.	27	Update the \fIprofiles/use.local.desc\fR file from metadata.xml.
24	.TP	28	.TP

Lines 75-80 user\-defined package sets Link Here

(-)a/man/portage.5 (+6 lines)
75	.BR /usr/portage/metadata/	75	.BR /usr/portage/metadata/
76	.nf	76	.nf
77	layout.conf	77	layout.conf
		78	pkg_desc_index
78	.fi	79	.fi
79	.TP	80	.TP
80	.BR /usr/portage/profiles/	81	.BR /usr/portage/profiles/
Lines 1110-1115 cache\-formats = md5-dict pms Link Here
1110	profile\-formats = portage-2	1111	profile\-formats = portage-2
1111	.fi	1112	.fi
1112	.RE	1113	.RE
		1114	.TP
		1115	.BR pkg_desc_index
		1116	This is an index of packages and descriptions which may be generated
		1117	by \fBegencache\fR(1) in order to optimize \fBemerge\fR(1) search
		1118	actions.
1113	.RE	1119	.RE
1114	.TP	1120	.TP
1115	.BR /usr/portage/profiles/	1121	.BR /usr/portage/profiles/

Lines 3-15 Link Here

(-)a/pym/_emerge/search.py (-16 / +181 lines)
3		3
4	from __future__ import print_function	4	from __future__ import print_function
5		5
		6	import io
6	import re	7	import re
7	import portage	8	import portage
8	from portage import os	9	from portage import os, _encodings
9	from portage.dbapi.porttree import _parse_uri_map	10	from portage.dbapi.porttree import _parse_uri_map
		11	from portage.dep import Atom
		12	from portage.exception import InvalidData
10	from portage.localization import localized_size	13	from portage.localization import localized_size
11	from portage.output import bold, bold as white, darkgreen, green, red	14	from portage.output import bold, bold as white, darkgreen, green, red
12	from portage.util import writemsg_stdout	15	from portage.util import writemsg_stdout
		16	from portage.versions import _pkg_str
13		17
14	from _emerge.Package import Package	18	from _emerge.Package import Package
15		19
Lines 30-36 class search(object): Link Here
30	The list of available and installed packages is created at object instantiation.	34	The list of available and installed packages is created at object instantiation.
31	This makes successive searches faster."""	35	This makes successive searches faster."""
32	self.settings = root_config.settings	36	self.settings = root_config.settings
33	self.vartree = root_config.trees["vartree"]
34	self.spinner = spinner	37	self.spinner = spinner
35	self.verbose = verbose	38	self.verbose = verbose
36	self.searchdesc = searchdesc	39	self.searchdesc = searchdesc
Lines 41-49 class search(object): Link Here
41		44
42	self._dbs = []	45	self._dbs = []
43		46
44	portdb = root_config.trees["porttree"].dbapi	47	portdb = IndexedPortdb(root_config.trees["porttree"].dbapi)
45	bindb = root_config.trees["bintree"].dbapi	48	bindb = root_config.trees["bintree"].dbapi
46	vardb = root_config.trees["vartree"].dbapi	49	vardb = IndexedVardb(root_config.trees["vartree"].dbapi)
47		50
48	if not usepkgonly and portdb._have_root_eclass_dir:	51	if not usepkgonly and portdb._have_root_eclass_dir:
49	self._dbs.append(portdb)	52	self._dbs.append(portdb)
Lines 53-58 class search(object): Link Here
53		56
54	self._dbs.append(vardb)	57	self._dbs.append(vardb)
55	self._portdb = portdb	58	self._portdb = portdb
		59	self._vardb = vardb
56		60
57	def _spinner_update(self):	61	def _spinner_update(self):
58	if self.spinner:	62	if self.spinner:
Lines 97-103 class search(object): Link Here
97	return {}	101	return {}
98		102
99	def _visible(self, db, cpv, metadata):	103	def _visible(self, db, cpv, metadata):
100	installed = db is self.vartree.dbapi	104	installed = db is self._vardb
101	built = installed or db is not self._portdb	105	built = installed or db is not self._portdb
102	pkg_type = "ebuild"	106	pkg_type = "ebuild"
103	if installed:	107	if installed:
Lines 208-213 class search(object): Link Here
208	masked=1	212	masked=1
209	self.matches["pkg"].append([package,masked])	213	self.matches["pkg"].append([package,masked])
210	elif self.searchdesc: # DESCRIPTION searching	214	elif self.searchdesc: # DESCRIPTION searching
		215	# Check for DESCRIPTION match first, so that we can skip
		216	# the expensive visiblity check if it doesn't match.
		217	full_package = portage.best(
		218	self._xmatch("match-all", package))
		219	try:
		220	full_desc = self._aux_get(
		221	full_package, ["DESCRIPTION"])[0]
		222	except KeyError:
		223	portage.writemsg(
		224	"emerge: search: aux_get() failed, skipping\n",
		225	noiselevel=-1)
		226	continue
		227	if not self.searchre.search(full_desc):
		228	continue
211	full_package = self._xmatch("bestmatch-visible", package)	229	full_package = self._xmatch("bestmatch-visible", package)
212	if not full_package:	230	if not full_package:
213	#no match found; we don't want to query description	231	#no match found; we don't want to query description
Lines 217-230 class search(object): Link Here
217	continue	235	continue
218	else:	236	else:
219	masked=1	237	masked=1
220	try:	238
221	full_desc = self._aux_get(	239	self.matches["desc"].append((full_package, masked))
222	full_package, ["DESCRIPTION"])[0]
223	except KeyError:
224	print("emerge: search: aux_get() failed, skipping")
225	continue
226	if self.searchre.search(full_desc):
227	self.matches["desc"].append([full_package,masked])
228		240
229	self.sdict = self.setconfig.getSets()	241	self.sdict = self.setconfig.getSets()
230	for setname in self.sdict:	242	for setname in self.sdict:
Lines 262-268 class search(object): Link Here
262	bold(self.searchkey) + " ]\n")	274	bold(self.searchkey) + " ]\n")
263	msg.append("[ Applications found : " + \	275	msg.append("[ Applications found : " + \
264	bold(str(self.mlen)) + " ]\n\n")	276	bold(str(self.mlen)) + " ]\n\n")
265	vardb = self.vartree.dbapi	277	vardb = self._vardb
266	metadata_keys = set(Package.metadata_keys)	278	metadata_keys = set(Package.metadata_keys)
267	metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"])	279	metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"])
268	metadata_keys = tuple(metadata_keys)	280	metadata_keys = tuple(metadata_keys)
Lines 372-378 class search(object): Link Here
372	# private interface	384	# private interface
373	#	385	#
374	def getInstallationStatus(self,package):	386	def getInstallationStatus(self,package):
375	installed_package = self.vartree.dep_bestmatch(package)	387	installed_package = self._vardb.match(package)
		388	if installed_package:
		389	installed_package = installed_package[-1]
		390	else:
		391	installed_package = ""
376	result = ""	392	result = ""
377	version = self.getVersion(installed_package,search.VERSION_RELEASE)	393	version = self.getVersion(installed_package,search.VERSION_RELEASE)
378	if len(version) > 0:	394	if len(version) > 0:
Lines 392-394 class search(object): Link Here
392	result = ""	408	result = ""
393	return result	409	return result
394		410
395	-	411
		412	class IndexedPortdb(object):
		413	"""
		414	A portdbapi interface that uses a package description index to
		415	improve performance. If the description index is missing for a
		416	particular repository, then all metadata for that repository is
		417	obtained using the normal pordbapi.aux_get method.
		418	"""
		419	def __init__(self, portdb):
		420	self._portdb = portdb
		421	self.cpv_exists = portdb.cpv_exists
		422	self.getFetchMap = portdb.getFetchMap
		423	self.findname = portdb.findname
		424	self._aux_cache_keys = portdb._aux_cache_keys
		425	self._have_root_eclass_dir = portdb._have_root_eclass_dir
		426	self._cpv_sort_ascending = portdb._cpv_sort_ascending
		427	self._desc_cache = None
		428	self._cp_map = None
		429
		430	def _init_index(self):
		431	cp_map = {}
		432	desc_cache = {}
		433	for repo_path in self._portdb.porttrees:
		434	outside_repo = os.path.join(self._portdb.depcachedir,
		435	repo_path.lstrip(os.sep))
		436	for parent_dir in (repo_path, outside_repo):
		437	file_path = os.path.join(parent_dir,
		438	"metadata", "pkg_desc_index")
		439
		440	try:
		441	with io.open(file_path,
		442	encoding=_encodings["repo.content"]) as f:
		443	for line in f:
		444	pkgs, desc = line.split(":", 1)
		445	desc = desc.strip()
		446	pkgs = pkgs.split(",")
		447	if not pkgs[0]:
		448	continue
		449	try:
		450	pkg = _pkg_str(pkgs[0])
		451	except InvalidData:
		452	continue
		453	cp_list = cp_map.get(pkg.cp)
		454	if cp_list is None:
		455	cp_list = []
		456	cp_map[pkg.cp] = cp_list
		457	cp_list.append(pkg)
		458	for ver in pkgs[1:]:
		459	try:
		460	cp_list.append(
		461	_pkg_str(pkg.cp + "-" + ver))
		462	except InvalidData:
		463	pass
		464	for cpv in cp_list:
		465	desc_cache[cpv] = desc
		466	except IOError:
		467	pass
		468	else:
		469	break
		470	else:
		471	# No descriptions index was found, so populate
		472	# cp_map the slow way.
		473	for cp in self._portdb.cp_all(trees=[repo_path]):
		474	cp_list = cp_map.get(cp)
		475	if cp_list is None:
476	cp_list = []
477	cp_map[cp] = cp_list
478	for cpv in self._portdb.cp_list(cp, mytree=repo_path):
479	if cpv not in cp_list:
480	cp_list.append(_pkg_str(cpv))
481
482	self._desc_cache = desc_cache
483	self._cp_map = cp_map
484
485	def cp_all(self):
486	if self._cp_map is None:
487	self._init_index()
488	return list(self._cp_map)
489
490	def match(self, atom):
491	if not isinstance(atom, Atom):
492	atom = Atom(atom)
493	cp_list = self._cp_map.get(atom.cp)
494	if cp_list is None:
495	return []
496	self._portdb._cpv_sort_ascending(cp_list)
497	return portage.match_from_list(atom, cp_list)
498
499	def aux_get(self, cpv, attrs, myrepo = None):
500	if len(attrs) == 1 and attrs[0] == "DESCRIPTION":
501	try:
502	return [self._desc_cache[cpv]]
503	except KeyError:
504	pass
505	return self._portdb.aux_get(cpv, attrs)
506
507
508	class IndexedVardb(object):
509	"""
510	A vardbapi interface that sacrifices validation in order to
511	improve performance. It takes advantage of vardbdbapi._aux_cache,
512	which is backed by vardb_metadata.pickle. Since _aux_cache is
513	not updated for every single merge/unmerge (see
514	_aux_cache_threshold), the list of packages is obtained directly
515	from the real vardbapi instance. If a package is missing from
516	_aux_cache, then its metadata is obtained using the normal
517	(validated) vardbapi.aux_get method.
518	"""
519	def __init__(self, vardb):
520	self._vardb = vardb
521	self._aux_cache_keys = vardb._aux_cache_keys
522	self._cpv_sort_ascending = vardb._cpv_sort_ascending
523	self._cp_map = {}
524	self.cpv_exists = vardb.cpv_exists
525
526	def cp_all(self):
527	if self._cp_map:
528	return list(self._cp_map)
529	cp_map = self._cp_map
530	for cpv in self._vardb.cpv_all():
531	cp = portage.cpv_getkey(cpv)
532	if cp is not None:
533	cp_list = cp_map.get(cp)
534	if cp_list is None:
535	cp_list = []
536	cp_map[cp] = cp_list
537	cp_list.append(_pkg_str(cpv))
538	return list(cp_map)
539
540	def match(self, atom):
541	if not isinstance(atom, Atom):
542	atom = Atom(atom)
543	cp_list = self._cp_map.get(atom.cp)
544	if cp_list is None:
545	return []
546	self._vardb._cpv_sort_ascending(cp_list)
547	return portage.match_from_list(atom, cp_list)
548
549	def aux_get(self, cpv, attrs, myrepo = None):
550	pkg_data = self._vardb._aux_cache["packages"].get(cpv)
551	if not isinstance(pkg_data, tuple) or \
552	len(pkg_data) != 2 or \
553	not isinstance(pkg_data[1], dict):
554	pkg_data = None
555	if pkg_data is None:
556	# It may be missing from _aux_cache due to
557	# _aux_cache_threshold.
558	return self._vardb.aux_get(cpv, attrs)
559	metadata = pkg_data[1]
560	return [metadata.get(k, "") for k in attrs]

Return to bug 525718