[patch] Improves porttree.auxget timings from 11msec down to 5.8msec

faster_porttree_auxget.patch (text/plain), 4.30 KB, created by Marat Radchenko on 2009-09-23 18:20:46 UTC

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 918c935..0f87136 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -9,6 +9,7 @@ from portage.cache import cache_errors
 import errno
 import stat
 import sys
+import os as _os
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -32,33 +33,32 @@ class database(fs_template.FsBased):
 			self._ensure_dirs()
 
 	def _getitem(self, cpv):
-		fp = os.path.join(self.location, cpv)
+		fp = self.location + _os.sep + cpv
 		try:
 			myf = codecs.open(_unicode_encode(fp,
 				encoding=_encodings['fs'], errors='strict'),
 				mode='r', encoding=_encodings['repo.content'],
 				errors='replace')
 			try:
-				d = self._parse_data(myf.readlines(), cpv)
+				d = self._parse_data(myf.read().split("\n"), cpv)
 				if '_mtime_' not in d:
 					# Backward compatibility with old cache
 					# that uses mtime mangling.
-					d['_mtime_'] = long(os.fstat(myf.fileno()).st_mtime)
+					d['_mtime_'] = long(_os.fstat(myf.fileno()).st_mtime)
 				return d
 			finally:
 				myf.close()
 		except (IOError, OSError) as e:
 			if e.errno != errno.ENOENT:
 				raise cache_errors.CacheCorruption(cpv, e)
-			raise KeyError(cpv)
+			raise KeyError(cpv, e)
 
 	def _parse_data(self, data, cpv):
 		try:
-			d = dict(map(lambda x:x.rstrip("\n").split("=", 1), data))
+			return dict( x.split("=", 1) for x in data )
 		except ValueError as e:
 			# If a line is missing an "=", the split length is 1 instead of 2.
 			raise cache_errors.CacheCorruption(cpv, e)
-		return d
 
 	def _setitem(self, cpv, values):
 #		import pdb;pdb.set_trace()
@@ -101,7 +101,6 @@ class database(fs_template.FsBased):
 			os.remove(fp)
 			raise cache_errors.CacheCorruption(cpv, e)
 
-
 	def _delitem(self, cpv):
 #		import pdb;pdb.set_trace()
 		try:
@@ -112,11 +111,9 @@ class database(fs_template.FsBased):
 			else:
 				raise cache_errors.CacheCorruption(cpv, e)
 
-
 	def __contains__(self, cpv):
 		return os.path.exists(os.path.join(self.location, cpv))
 
-
 	def __iter__(self):
 		"""generator for walking the dir struct"""
 		dirs = [self.location]
@@ -140,4 +137,3 @@ class database(fs_template.FsBased):
 					continue
 				yield p[len_base+1:]
 			dirs.pop(0)
-
diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index 81b1f17..2704bd5 100644
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -52,7 +52,7 @@ class database(flat_hash.database):
 				d.clear()
 				try:
 					for i, key in enumerate(self.auxdbkey_order):
-						d[key] = data[i].rstrip("\n")
+						d[key] = data[i]
 				except IndexError:
 					pass
 				break
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index b073e42..200d6e7 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -379,9 +379,6 @@ class portdbapi(dbapi):
 			"PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
 			"RESTRICT", "SLOT"])
 
-		# Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict
-		# initialization until the first aux_get call.
-		self._aux_cache_slot_dict = None
 		self._aux_cache = {}
 		self._broken_ebuilds = set()
 
@@ -548,7 +545,7 @@ class portdbapi(dbapi):
 	def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
 
 		try:
-			st = os.stat(ebuild_path)
+			st = _os.stat(_unicode_encode(ebuild_path, encoding=portage._encodings['fs']))
 			emtime = st[stat.ST_MTIME]
 		except OSError:
 			writemsg(_("!!! aux_get(): ebuild for " \
@@ -668,8 +665,7 @@ class portdbapi(dbapi):
 				mydata["_eclasses_"] = {}
 
 		# do we have a origin repository name for the current package
-		mydata["repository"] = self._repository_map.get(
-			os.path.sep.join(myebuild.split(os.path.sep)[:-3]), "")
+		mydata["repository"] = self._repository_map.get(mylocation, "")
 
 		mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
 		mydata["_mtime_"] = long(st.st_mtime)
@@ -687,10 +683,7 @@ class portdbapi(dbapi):
 		returnme = [mydata.get(x, "") for x in mylist]
 
 		if cache_me:
-			if self._aux_cache_slot_dict is None:
-				self._aux_cache_slot_dict = \
-					slot_dict_class(self._aux_cache_keys)
-			aux_cache = self._aux_cache_slot_dict()
+			aux_cache = {}
 			for x in self._aux_cache_keys:
 				aux_cache[x] = mydata.get(x, "")
 			self._aux_cache[mycpv] = aux_cache