Attachment #386988 for bug #525718

Lines 57-63 from portage.util._async.run_main_scheduler import run_main_scheduler Link Here

(-)a/bin/egencache (-2 / +37 lines)
57	from portage.util._eventloop.global_event_loop import global_event_loop	57	from portage.util._eventloop.global_event_loop import global_event_loop
58	from portage import cpv_getkey	58	from portage import cpv_getkey
59	from portage.dep import Atom, isjustname	59	from portage.dep import Atom, isjustname
60	from portage.versions import pkgsplit, vercmp	60	from portage.versions import pkgsplit, vercmp, _pkg_str
61		61
62	try:	62	try:
63	from xml.etree import ElementTree	63	from xml.etree import ElementTree
Lines 91-96 def parse_args(args): Link Here
91	actions.add_argument("--update-changelogs",	91	actions.add_argument("--update-changelogs",
92	action="store_true",	92	action="store_true",
93	help="update the ChangeLog files from SCM logs")	93	help="update the ChangeLog files from SCM logs")
		94	actions.add_argument("--update-pkg-desc-index",
		95	action="store_true",
		96	help="update package description index")
94	actions.add_argument("--update-manifests",	97	actions.add_argument("--update-manifests",
95	action="store_true",	98	action="store_true",
96	help="update manifests")	99	help="update manifests")
Lines 451-456 class GenCache(object): Link Here
451	if hasattr(trg_cache, '_prune_empty_dirs'):	454	if hasattr(trg_cache, '_prune_empty_dirs'):
452	trg_cache._prune_empty_dirs()	455	trg_cache._prune_empty_dirs()
453		456
		457	class GenPkgDescIndex(object):
		458	def __init__(self, portdb, output_file):
		459	self.returncode = os.EX_OK
		460	self._portdb = portdb
		461	self._output_file = output_file
		462
		463	def run(self):
		464
		465	portage.util.ensure_dirs(os.path.dirname(self._output_file))
		466	f = portage.util.atomic_ofstream(self._output_file,
		467	encoding=_encodings["repo.content"])
		468
		469	portdb = self._portdb
		470	for cp in portdb.cp_all():
		471	pkgs = portdb.cp_list(cp)
		472	if not pkgs:
		473	continue
		474	desc, = portdb.aux_get(pkgs[-1], ["DESCRIPTION"])
		475
		476	f.write("%s %s: %s\n" % (cp,
		477	" ".join(_pkg_str(cpv).version
		478	for cpv in pkgs), desc))
		479
		480	f.close()
		481
454	class GenUseLocalDesc(object):	482	class GenUseLocalDesc(object):
455	def __init__(self, portdb, output=None,	483	def __init__(self, portdb, output=None,
456	preserve_comments=False):	484	preserve_comments=False):
Lines 893-899 def egencache_main(args): Link Here
893	local_config=False, env=env)	921	local_config=False, env=env)
894		922
895	if not (options.update or options.update_use_local_desc or	923	if not (options.update or options.update_use_local_desc or
896	options.update_changelogs or options.update_manifests):	924	options.update_changelogs or options.update_manifests or
		925	options.update_pkg_desc_index):
897	parser.error('No action specified')	926	parser.error('No action specified')
898	return 1	927	return 1
899		928
Lines 1057-1062 def egencache_main(args): Link Here
1057	else:	1086	else:
1058	ret.append(scheduler.returncode)	1087	ret.append(scheduler.returncode)
1059		1088
		1089	if options.update_pkg_desc_index:
		1090	gen_index = GenPkgDescIndex(portdb, os.path.join(
		1091	repo_config.location, "metadata", "pkg_desc_index"))
		1092	gen_index.run()
		1093	ret.append(gen_index.returncode)
		1094
1060	if options.update_use_local_desc:	1095	if options.update_use_local_desc:
1061	gen_desc = GenUseLocalDesc(portdb,	1096	gen_desc = GenUseLocalDesc(portdb,
1062	output=options.uld_output,	1097	output=options.uld_output,

Lines 19-24 for the details on package atom syntax. Link Here

(-)a/man/egencache.1 (+4 lines)
19	.BR "\-\-update\-changelogs"	19	.BR "\-\-update\-changelogs"
20	Update the ChangeLog files from SCM logs (supported only in git repos).	20	Update the ChangeLog files from SCM logs (supported only in git repos).
21	.TP	21	.TP
		22	.BR "\-\-update\-pkg\-desc\-index"
		23	Update the package description index which is located at
		24	\fImetadata/pkg_desc_index\fR in the repository.
		25	.TP
22	.BR "\-\-update\-use\-local\-desc"	26	.BR "\-\-update\-use\-local\-desc"
23	Update the \fIprofiles/use.local.desc\fR file from metadata.xml.	27	Update the \fIprofiles/use.local.desc\fR file from metadata.xml.
24	.TP	28	.TP

Lines 790-795 If ebuilds using EAPIs which \fIdo not\fR support \fBHDEPEND\fR are built in Link Here

(-)a/man/emerge.1 (+8 lines)
790	the same \fBemerge\fR run as those using EAPIs which \fIdo\fR support	790	the same \fBemerge\fR run as those using EAPIs which \fIdo\fR support
791	\fBHDEPEND\fR, this option affects only the former.	791	\fBHDEPEND\fR, this option affects only the former.
792	.TP	792	.TP
		793	.BR "\-\-search\-index < y \| n >"
		794	Enable or disable indexed search for search actions. This option is
		795	enabled by default. The search index needs to be regenerated by
		796	\fBegencache\fR(1) after changes are made to a repository (see the
		797	\fB\-\-update\-pkg\-desc\-index\fR action). This setting can be added
		798	to \fBEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) and later
		799	overridden via the command line.
		800	.TP
793	.BR "\-\-select [ y \| n ] (\-w short option)"	801	.BR "\-\-select [ y \| n ] (\-w short option)"
794	Add specified packages to the world set (inverse of	802	Add specified packages to the world set (inverse of
795	\fB\-\-oneshot\fR). This is useful if you want to	803	\fB\-\-oneshot\fR). This is useful if you want to




.BR /usr/portage/metadata/
.nf
layout.conf
pkg_desc_index
.fi
.TP
.BR /usr/portage/profiles/

profile\-formats = portage-2
.fi
.RE
.TP
.BR pkg_desc_index
This is an index of package names, versions, and descriptions which
may be generated by \fBegencache\fR(1) in order to optimize
\fBemerge\fR(1) search actions.

.I Example:
.nf
sys-apps/sed 4.2 4.2.1 4.2.1-r1 4.2.2: Super-useful stream editor
sys-apps/usleep 0.1: A wrapper for usleep
.fi
.RE
.TP
.BR /usr/portage/profiles/

Lines 2015-2021 def action_search(root_config, myopts, myfiles, spinner): Link Here

(-)a/pym/_emerge/actions.py (-1 / +2 lines)
2015	searchinstance = search(root_config,	2015	searchinstance = search(root_config,
2016	spinner, "--searchdesc" in myopts,	2016	spinner, "--searchdesc" in myopts,
2017	"--quiet" not in myopts, "--usepkg" in myopts,	2017	"--quiet" not in myopts, "--usepkg" in myopts,
2018	"--usepkgonly" in myopts)	2018	"--usepkgonly" in myopts,
		2019	search_index = myopts.get("--search-index", "y") != "n")
2019	for mysearch in myfiles:	2020	for mysearch in myfiles:
2020	try:	2021	try:
2021	searchinstance.execute(mysearch)	2022	searchinstance.execute(mysearch)

Lines 616-621 def parse_opts(tmpcmdline, silent=False): Link Here

(-)a/pym/_emerge/main.py (+5 lines)
616	"choices" :("True", "rdeps")	616	"choices" :("True", "rdeps")
617	},	617	},
618		618
		619	"--search-index": {
		620	"help": "Enable or disable indexed search (enabled by default)",
		621	"choices": y_or_n
		622	},
		623
619	"--select": {	624	"--select": {
620	"shortopt" : "-w",	625	"shortopt" : "-w",
621	"help" : "add specified packages to the world set " + \	626	"help" : "add specified packages to the world set " + \





from __future__ import print_function

import io
import re
import portage
from portage import os, _encodings
from portage.dbapi.porttree import _parse_uri_map
from portage.dep import Atom
from portage.exception import InvalidAtom, InvalidData
from portage.localization import localized_size
from portage.output import  bold, bold as white, darkgreen, green, red
from portage.util import writemsg_stdout
from portage.versions import _pkg_str

from _emerge.Package import Package


	# public interface
	#
	def __init__(self, root_config, spinner, searchdesc,
		verbose, usepkg, usepkgonly, search_index = True):
		"""Searches the available and installed packages for the supplied search key.
		The list of available and installed packages is created at object instantiation.
		This makes successive searches faster."""
		self.settings = root_config.settings
		self.vartree = root_config.trees["vartree"]
		self.spinner = spinner
		self.verbose = verbose
		self.searchdesc = searchdesc

		bindb = root_config.trees["bintree"].dbapi
		vardb = root_config.trees["vartree"].dbapi

		if search_index:
			portdb = IndexedPortdb(portdb)
			vardb = IndexedVardb(vardb)

		if not usepkgonly and portdb._have_root_eclass_dir:
			self._dbs.append(portdb)



		self._dbs.append(vardb)
		self._portdb = portdb
		self._vardb = vardb

	def _spinner_update(self):
		if self.spinner:

		return {}

	def _visible(self, db, cpv, metadata):
		installed = db is self._vardb
		built = installed or db is not self._portdb
		pkg_type = "ebuild"
		if installed:

					masked=1
				self.matches["pkg"].append([package,masked])
			elif self.searchdesc: # DESCRIPTION searching
				# Check for DESCRIPTION match first, so that we can skip
				# the expensive visiblity check if it doesn't match.
				full_package = self._xmatch("match-all", package)
				if not full_package:
					continue
				full_package = full_package[-1]
				try:
					full_desc = self._aux_get(
						full_package, ["DESCRIPTION"])[0]
				except KeyError:
					portage.writemsg(
						"emerge: search: aux_get() failed, skipping\n",
						noiselevel=-1)
					continue
				if not self.searchre.search(full_desc):
					continue
				full_package = self._xmatch("bestmatch-visible", package)
				if not full_package:
					#no match found; we don't want to query description

						continue
					else:
						masked=1

				self.matches["desc"].append((full_package, masked))
						full_package, ["DESCRIPTION"])[0]
				except KeyError:
					print("emerge: search: aux_get() failed, skipping")
					continue
				if self.searchre.search(full_desc):
					self.matches["desc"].append([full_package,masked])

		self.sdict = self.setconfig.getSets()
		for setname in self.sdict:

			bold(self.searchkey) + " ]\n")
		msg.append("[ Applications found : " + \
			bold(str(self.mlen)) + " ]\n\n")
		vardb = self._vardb
		metadata_keys = set(Package.metadata_keys)
		metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"])
		metadata_keys = tuple(metadata_keys)

	# private interface
	#
	def getInstallationStatus(self,package):
		installed_package = self._vardb.match(package)
		if installed_package:
			installed_package = installed_package[-1]
		else:
			installed_package = ""
		result = ""
		version = self.getVersion(installed_package,search.VERSION_RELEASE)
		if len(version) > 0:

			result = ""
		return result


class IndexedPortdb(object):
	"""
	A portdbapi interface that uses a package description index to
	improve performance. If the description index is missing for a
	particular repository, then all metadata for that repository is
	obtained using the normal pordbapi.aux_get method.
	"""
	def __init__(self, portdb):
		self._portdb = portdb
		self.cpv_exists = portdb.cpv_exists
		self.getFetchMap = portdb.getFetchMap
		self.findname = portdb.findname
		self._aux_cache_keys = portdb._aux_cache_keys
		self._have_root_eclass_dir = portdb._have_root_eclass_dir
		self._cpv_sort_ascending = portdb._cpv_sort_ascending
		self._desc_cache = None
		self._cp_map = None

	def _init_index(self):
		cp_map = {}
		desc_cache = {}
		for repo_path in self._portdb.porttrees:
			outside_repo = os.path.join(self._portdb.depcachedir,
				repo_path.lstrip(os.sep))
			for parent_dir in (repo_path, outside_repo):
				file_path = os.path.join(parent_dir,
					"metadata", "pkg_desc_index")

				try:
					with io.open(file_path,
						encoding=_encodings["repo.content"]) as f:
						for line in f:
							try:
								pkgs, desc = line.split(":", 1)
							except ValueError:
								continue
							desc = desc.strip()
							try:
								cp, pkgs = pkgs.split(" ", 1)
							except ValueError:
								continue
							if not cp:
								continue
							try:
								atom = Atom(cp)
							except InvalidAtom:
								continue
							if cp != atom.cp:
								continue
							cp_list = cp_map.get(cp)
							if cp_list is None:
								cp_list = []
								cp_map[cp] = cp_list
							for ver in pkgs.split():
								try:
									cpv = _pkg_str(cp + "-" + ver)
								except InvalidData:
									pass
								else:
									cp_list.append(cpv)
									desc_cache[cpv] = desc
				except IOError:
					pass
				else:
					break
			else:
				# No descriptions index was found, so populate
				# cp_map the slow way.
				for cp in self._portdb.cp_all(trees=[repo_path]):
					cp_list = cp_map.get(cp)
					if cp_list is None:
						cp_list = []
						cp_map[cp] = cp_list
					for cpv in self._portdb.cp_list(cp, mytree=repo_path):
						if cpv not in cp_list:
							cp_list.append(_pkg_str(cpv))

		self._desc_cache = desc_cache
		self._cp_map = cp_map

	def cp_all(self):
		if self._cp_map is None:
			self._init_index()
		return list(self._cp_map)

	def match(self, atom):
		if not isinstance(atom, Atom):
			atom = Atom(atom)
		cp_list = self._cp_map.get(atom.cp)
		if cp_list is None:
			return []
		self._portdb._cpv_sort_ascending(cp_list)
		return portage.match_from_list(atom, cp_list)

	def aux_get(self, cpv, attrs, myrepo = None):
		if len(attrs) == 1 and attrs[0] == "DESCRIPTION":
			try:
				return [self._desc_cache[cpv]]
			except KeyError:
				pass
		return self._portdb.aux_get(cpv, attrs)


class IndexedVardb(object):
	"""
	A vardbapi interface that sacrifices validation in order to
	improve performance. It takes advantage of vardbdbapi._aux_cache,
	which is backed by vdb_metadata.pickle. Since _aux_cache is
	not updated for every single merge/unmerge (see
	_aux_cache_threshold), the list of packages is obtained directly
	from the real vardbapi instance. If a package is missing from
	_aux_cache, then its metadata is obtained using the normal
	(validated) vardbapi.aux_get method.
	"""
	def __init__(self, vardb):
		self._vardb = vardb
		self._aux_cache_keys = vardb._aux_cache_keys
		self._cpv_sort_ascending = vardb._cpv_sort_ascending
		self._cp_map = {}
		self.cpv_exists = vardb.cpv_exists

	def cp_all(self):
		if self._cp_map:
			return list(self._cp_map)
		cp_map = self._cp_map
		for cpv in self._vardb.cpv_all():
			cp = portage.cpv_getkey(cpv)
			if cp is not None:
				cp_list = cp_map.get(cp)
				if cp_list is None:
					cp_list = []
					cp_map[cp] = cp_list
				cp_list.append(_pkg_str(cpv))
		return list(cp_map)

	def match(self, atom):
		if not isinstance(atom, Atom):
			atom = Atom(atom)
		cp_list = self._cp_map.get(atom.cp)
		if cp_list is None:
			return []
		self._vardb._cpv_sort_ascending(cp_list)
		return portage.match_from_list(atom, cp_list)

	def aux_get(self, cpv, attrs, myrepo = None):
		pkg_data = self._vardb._aux_cache["packages"].get(cpv)
		if not isinstance(pkg_data, tuple) or \
			len(pkg_data) != 2 or \
			not isinstance(pkg_data[1], dict):
			pkg_data = None
		if pkg_data is None:
			# It may be missing from _aux_cache due to
			# _aux_cache_threshold.
			return self._vardb.aux_get(cpv, attrs)
		metadata = pkg_data[1]
		return [metadata.get(k, "") for k in attrs]

Return to bug 525718

Lines 75-80 user\-defined package sets Link Here

(-)a/man/portage.5 (+12 lines)
75	.BR /usr/portage/metadata/	75	.BR /usr/portage/metadata/
76	.nf	76	.nf
77	layout.conf	77	layout.conf
		78	pkg_desc_index
78	.fi	79	.fi
79	.TP	80	.TP
80	.BR /usr/portage/profiles/	81	.BR /usr/portage/profiles/
Lines 1110-1115 cache\-formats = md5-dict pms Link Here
1110	profile\-formats = portage-2	1111	profile\-formats = portage-2
1111	.fi	1112	.fi
1112	.RE	1113	.RE
		1114	.TP
		1115	.BR pkg_desc_index
		1116	This is an index of package names, versions, and descriptions which
		1117	may be generated by \fBegencache\fR(1) in order to optimize
		1118	\fBemerge\fR(1) search actions.
		1119
		1120	.I Example:
		1121	.nf
		1122	sys-apps/sed 4.2 4.2.1 4.2.1-r1 4.2.2: Super-useful stream editor
		1123	sys-apps/usleep 0.1: A wrapper for usleep
		1124	.fi
1113	.RE	1125	.RE
1114	.TP	1126	.TP
1115	.BR /usr/portage/profiles/	1127	.BR /usr/portage/profiles/

Lines 3-15 Link Here

(-)a/pym/_emerge/search.py (-15 / +193 lines)
3		3
4	from __future__ import print_function	4	from __future__ import print_function
5		5
		6	import io
6	import re	7	import re
7	import portage	8	import portage
8	from portage import os	9	from portage import os, _encodings
9	from portage.dbapi.porttree import _parse_uri_map	10	from portage.dbapi.porttree import _parse_uri_map
		11	from portage.dep import Atom
		12	from portage.exception import InvalidAtom, InvalidData
10	from portage.localization import localized_size	13	from portage.localization import localized_size
11	from portage.output import bold, bold as white, darkgreen, green, red	14	from portage.output import bold, bold as white, darkgreen, green, red
12	from portage.util import writemsg_stdout	15	from portage.util import writemsg_stdout
		16	from portage.versions import _pkg_str
13		17
14	from _emerge.Package import Package	18	from _emerge.Package import Package
15		19
Lines 25-36 class search(object): Link Here
25	# public interface	29	# public interface
26	#	30	#
27	def __init__(self, root_config, spinner, searchdesc,	31	def __init__(self, root_config, spinner, searchdesc,
28	verbose, usepkg, usepkgonly):	32	verbose, usepkg, usepkgonly, search_index = True):
29	"""Searches the available and installed packages for the supplied search key.	33	"""Searches the available and installed packages for the supplied search key.
30	The list of available and installed packages is created at object instantiation.	34	The list of available and installed packages is created at object instantiation.
31	This makes successive searches faster."""	35	This makes successive searches faster."""
32	self.settings = root_config.settings	36	self.settings = root_config.settings
33	self.vartree = root_config.trees["vartree"]
34	self.spinner = spinner	37	self.spinner = spinner
35	self.verbose = verbose	38	self.verbose = verbose
36	self.searchdesc = searchdesc	39	self.searchdesc = searchdesc
Lines 45-50 class search(object): Link Here
45	bindb = root_config.trees["bintree"].dbapi	48	bindb = root_config.trees["bintree"].dbapi
46	vardb = root_config.trees["vartree"].dbapi	49	vardb = root_config.trees["vartree"].dbapi
47		50
		51	if search_index:
		52	portdb = IndexedPortdb(portdb)
		53	vardb = IndexedVardb(vardb)
		54
48	if not usepkgonly and portdb._have_root_eclass_dir:	55	if not usepkgonly and portdb._have_root_eclass_dir:
49	self._dbs.append(portdb)	56	self._dbs.append(portdb)
50		57
Lines 53-58 class search(object): Link Here
53		60
54	self._dbs.append(vardb)	61	self._dbs.append(vardb)
55	self._portdb = portdb	62	self._portdb = portdb
		63	self._vardb = vardb
56		64
57	def _spinner_update(self):	65	def _spinner_update(self):
58	if self.spinner:	66	if self.spinner:
Lines 97-103 class search(object): Link Here
97	return {}	105	return {}
98		106
99	def _visible(self, db, cpv, metadata):	107	def _visible(self, db, cpv, metadata):
100	installed = db is self.vartree.dbapi	108	installed = db is self._vardb
101	built = installed or db is not self._portdb	109	built = installed or db is not self._portdb
102	pkg_type = "ebuild"	110	pkg_type = "ebuild"
103	if installed:	111	if installed:
Lines 208-213 class search(object): Link Here
208	masked=1	216	masked=1
209	self.matches["pkg"].append([package,masked])	217	self.matches["pkg"].append([package,masked])
210	elif self.searchdesc: # DESCRIPTION searching	218	elif self.searchdesc: # DESCRIPTION searching
		219	# Check for DESCRIPTION match first, so that we can skip
		220	# the expensive visiblity check if it doesn't match.
		221	full_package = self._xmatch("match-all", package)
		222	if not full_package:
		223	continue
		224	full_package = full_package[-1]
		225	try:
		226	full_desc = self._aux_get(
		227	full_package, ["DESCRIPTION"])[0]
		228	except KeyError:
		229	portage.writemsg(
		230	"emerge: search: aux_get() failed, skipping\n",
		231	noiselevel=-1)
		232	continue
		233	if not self.searchre.search(full_desc):
		234	continue
211	full_package = self._xmatch("bestmatch-visible", package)	235	full_package = self._xmatch("bestmatch-visible", package)
212	if not full_package:	236	if not full_package:
213	#no match found; we don't want to query description	237	#no match found; we don't want to query description
Lines 217-230 class search(object): Link Here
217	continue	241	continue
218	else:	242	else:
219	masked=1	243	masked=1
220	try:	244
221	full_desc = self._aux_get(	245	self.matches["desc"].append((full_package, masked))
222	full_package, ["DESCRIPTION"])[0]
223	except KeyError:
224	print("emerge: search: aux_get() failed, skipping")
225	continue
226	if self.searchre.search(full_desc):
227	self.matches["desc"].append([full_package,masked])
228		246
229	self.sdict = self.setconfig.getSets()	247	self.sdict = self.setconfig.getSets()
230	for setname in self.sdict:	248	for setname in self.sdict:
Lines 262-268 class search(object): Link Here
262	bold(self.searchkey) + " ]\n")	280	bold(self.searchkey) + " ]\n")
263	msg.append("[ Applications found : " + \	281	msg.append("[ Applications found : " + \
264	bold(str(self.mlen)) + " ]\n\n")	282	bold(str(self.mlen)) + " ]\n\n")
265	vardb = self.vartree.dbapi	283	vardb = self._vardb
266	metadata_keys = set(Package.metadata_keys)	284	metadata_keys = set(Package.metadata_keys)
267	metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"])	285	metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"])
268	metadata_keys = tuple(metadata_keys)	286	metadata_keys = tuple(metadata_keys)
Lines 372-378 class search(object): Link Here
372	# private interface	390	# private interface
373	#	391	#
374	def getInstallationStatus(self,package):	392	def getInstallationStatus(self,package):
375	installed_package = self.vartree.dep_bestmatch(package)	393	installed_package = self._vardb.match(package)
		394	if installed_package:
		395	installed_package = installed_package[-1]
		396	else:
		397	installed_package = ""
376	result = ""	398	result = ""
377	version = self.getVersion(installed_package,search.VERSION_RELEASE)	399	version = self.getVersion(installed_package,search.VERSION_RELEASE)
378	if len(version) > 0:	400	if len(version) > 0:
Lines 392-394 class search(object): Link Here
392	result = ""	414	result = ""
393	return result	415	return result
394		416
395	-	417
		418	class IndexedPortdb(object):
		419	"""
		420	A portdbapi interface that uses a package description index to
		421	improve performance. If the description index is missing for a
		422	particular repository, then all metadata for that repository is
		423	obtained using the normal pordbapi.aux_get method.
		424	"""
		425	def __init__(self, portdb):
		426	self._portdb = portdb
		427	self.cpv_exists = portdb.cpv_exists
		428	self.getFetchMap = portdb.getFetchMap
		429	self.findname = portdb.findname
		430	self._aux_cache_keys = portdb._aux_cache_keys
		431	self._have_root_eclass_dir = portdb._have_root_eclass_dir
		432	self._cpv_sort_ascending = portdb._cpv_sort_ascending
		433	self._desc_cache = None
		434	self._cp_map = None
		435
		436	def _init_index(self):
		437	cp_map = {}
		438	desc_cache = {}
		439	for repo_path in self._portdb.porttrees:
		440	outside_repo = os.path.join(self._portdb.depcachedir,
		441	repo_path.lstrip(os.sep))
		442	for parent_dir in (repo_path, outside_repo):
		443	file_path = os.path.join(parent_dir,
		444	"metadata", "pkg_desc_index")
		445
		446	try:
		447	with io.open(file_path,
		448	encoding=_encodings["repo.content"]) as f:
		449	for line in f:
		450	try:
		451	pkgs, desc = line.split(":", 1)
		452	except ValueError:
		453	continue
		454	desc = desc.strip()
		455	try:
		456	cp, pkgs = pkgs.split(" ", 1)
		457	except ValueError:
		458	continue
		459	if not cp:
		460	continue
		461	try:
		462	atom = Atom(cp)
		463	except InvalidAtom:
		464	continue
		465	if cp != atom.cp:
		466	continue
		467	cp_list = cp_map.get(cp)
		468	if cp_list is None:
		469	cp_list = []
		470	cp_map[cp] = cp_list
		471	for ver in pkgs.split():
		472	try:
		473	cpv = _pkg_str(cp + "-" + ver)
		474	except InvalidData:
		475	pass
		476	else:
		477	cp_list.append(cpv)
		478	desc_cache[cpv] = desc
		479	except IOError:
		480	pass
		481	else:
482	break
483	else:
484	# No descriptions index was found, so populate
485	# cp_map the slow way.
486	for cp in self._portdb.cp_all(trees=[repo_path]):
487	cp_list = cp_map.get(cp)
488	if cp_list is None:
489	cp_list = []
490	cp_map[cp] = cp_list
491	for cpv in self._portdb.cp_list(cp, mytree=repo_path):
492	if cpv not in cp_list:
493	cp_list.append(_pkg_str(cpv))
494
495	self._desc_cache = desc_cache
496	self._cp_map = cp_map
497
498	def cp_all(self):
499	if self._cp_map is None:
500	self._init_index()
501	return list(self._cp_map)
502
503	def match(self, atom):
504	if not isinstance(atom, Atom):
505	atom = Atom(atom)
506	cp_list = self._cp_map.get(atom.cp)
507	if cp_list is None:
508	return []
509	self._portdb._cpv_sort_ascending(cp_list)
510	return portage.match_from_list(atom, cp_list)
511
512	def aux_get(self, cpv, attrs, myrepo = None):
513	if len(attrs) == 1 and attrs[0] == "DESCRIPTION":
514	try:
515	return [self._desc_cache[cpv]]
516	except KeyError:
517	pass
518	return self._portdb.aux_get(cpv, attrs)
519
520
521	class IndexedVardb(object):
522	"""
523	A vardbapi interface that sacrifices validation in order to
524	improve performance. It takes advantage of vardbdbapi._aux_cache,
525	which is backed by vdb_metadata.pickle. Since _aux_cache is
526	not updated for every single merge/unmerge (see
527	_aux_cache_threshold), the list of packages is obtained directly
528	from the real vardbapi instance. If a package is missing from
529	_aux_cache, then its metadata is obtained using the normal
530	(validated) vardbapi.aux_get method.
531	"""
532	def __init__(self, vardb):
533	self._vardb = vardb
534	self._aux_cache_keys = vardb._aux_cache_keys
535	self._cpv_sort_ascending = vardb._cpv_sort_ascending
536	self._cp_map = {}
537	self.cpv_exists = vardb.cpv_exists
538
539	def cp_all(self):
540	if self._cp_map:
541	return list(self._cp_map)
542	cp_map = self._cp_map
543	for cpv in self._vardb.cpv_all():
544	cp = portage.cpv_getkey(cpv)
545	if cp is not None:
546	cp_list = cp_map.get(cp)
547	if cp_list is None:
548	cp_list = []
549	cp_map[cp] = cp_list
550	cp_list.append(_pkg_str(cpv))
551	return list(cp_map)
552
553	def match(self, atom):
554	if not isinstance(atom, Atom):
555	atom = Atom(atom)
556	cp_list = self._cp_map.get(atom.cp)
557	if cp_list is None:
558	return []
559	self._vardb._cpv_sort_ascending(cp_list)
560	return portage.match_from_list(atom, cp_list)
561
562	def aux_get(self, cpv, attrs, myrepo = None):
563	pkg_data = self._vardb._aux_cache["packages"].get(cpv)
564	if not isinstance(pkg_data, tuple) or \
565	len(pkg_data) != 2 or \
566	not isinstance(pkg_data[1], dict):
567	pkg_data = None
568	if pkg_data is None:
569	# It may be missing from _aux_cache due to
570	# _aux_cache_threshold.
571	return self._vardb.aux_get(cpv, attrs)
572	metadata = pkg_data[1]
573	return [metadata.get(k, "") for k in attrs]