Go to:
Gentoo Home
Documentation
Forums
Lists
Bugs
Planet
Store
Wiki
Get Gentoo!
Gentoo's Bugzilla – Attachment 37914 Details for
Bug 51704
MySQL cache backend for portage
Home
|
New
–
[Ex]
|
Browse
|
Search
|
Privacy Policy
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
new module
portage_db_mysql.py (text/plain), 10.82 KB, created by
Ahmed Farid
on 2004-08-22 01:36:02 UTC
(
hide
)
Description:
new module
Filename:
MIME Type:
Creator:
Ahmed Farid
Created:
2004-08-22 01:36:02 UTC
Size:
10.82 KB
patch
obsolete
>import MySQLdb, portage_db_template >__author__ = "Ahmed Farid" >__version__ = "0.8" > >ex = MySQLdb.escape_string >class database(portage_db_template.database): > #ought to do ref counting on the number of open connections. course that's worthless till portage calls database.close(). > connections = {} > path_map = {} > categories = {} > tables = [] > defaultOptions = {'host':'localhost', 'port': 3306, 'user': 'portage', 'password': "", 'db': 'portage'} > last_haskey = last_value = None > global ex > call_count = {'get': 0, 'set' : 0, 'keys': 0, 'del': 0} > def module_init(self): > options = database.defaultOptions.copy() # default options in case the user doesnt pass any options > if self.config: > for x in self.config.keys(): > options[x] = self.config[x] > self.config = options; del options > path = self.path.replace('//', '/') > self.constr = self.config['host'] + ':' + str(self.config['port']) + '/' + self.config['db'] > if not database.connections.has_key(self.constr): > try: > con = [0, MySQLdb.connect(host=self.config['host'], port=self.config['port'], user=self.config['user'], passwd=self.config['password'])] > database.connections.setdefault(self.constr, con) > except Exception, e: > raise Exception, "Error connecting to Database using self.config=(%s): exception=(%s)" % (str(self.config), str(e)) > > self.con = database.connections[self.constr][1] # connection object > self.db = self.con.cursor() > database.connections[self.constr][0] += 1 > if database.connections[self.constr][0] == 1: # set to autocommit to false (so the sync function can be used) and check the database/init it > self.db.execute('set AUTOCOMMIT=0') > self.check_database() > > try: > if len(database.categories.keys()) == 0: > self.db.execute('select name,c_id from `category_table`') > for x, y in self.db.fetchall(): > database.categories[x] = y > except Exception, e: > self.check_exception(e) > raise Exception, "Database Error: failed pulling tables using self.config=(%s), exception=(%s)" % (str(self.config), str(e)) > if not database.categories.has_key(self.category) : > try: > self.db.execute('INSERT INTO `category_table` (`name`) VALUES("%s")' % ex(self.category)) > if self.db.insert_id(): > database.categories[self.category] = self.db.insert_id() > else: > print "no insert_id() in category_table, wth?" > > except Exception, e: > self.check_exception(e) > raise Exception, "Database error: failed creation table for path('%s'), category('%s'), exception=(%s)" % (self.path, self.category, str(e)) > self.cat_id = database.categories[self.category] > > try: > if len( database.path_map.keys()) == 0: > if self.db.execute('SELECT name, p_id FROM `path_table` ') > 0: > for x,y in self.db.fetchall(): > database.path_map[x] = y > except Exception, e: > self.check_exception(e) > raise Exception, "Database Error: failed pulling tables using self.config=(%s), exception=(%s)" % (str(self.config), str(e)) > > if not database.path_map.has_key(path): > try: > self.db.execute('INSERT INTO `path_table`(`name`) VALUES("%s")' % ex(path)) > if self.db.insert_id(): > database.path_map[path] = self.db.insert_id() > else: > print "no insert_id() in path_map, wth?" > except Exception, e: > self.check_exception(e) > raise Exception, "Database error: failed creation table for path('%s'), category('%s'), exception=(%s)" % (self.path, self.category, str(e)) > self.p_id = database.path_map[path] > > def check_exception(self, e): > # this should be removed later, i had to use it because it was a bug in reiser4. > if "Got error 127 from table handler" in str(e): > if self.db.execute('check table `package_name`') > 1: > print '`package_name` got corrupted some how, trying to fix' > self.db.execute('repair table `package_name`') > check = self.db.execute('check table `package_name`') > status = self.db.fetchall() > if ( check == 1 and status[0][3] == "OK"): > raise Exception, "Database was corrupt be we were able to repair it, please rerun the emerge command" > else: > raise Exception, "Database is corrupt, exception=(%s)" % str(e) > > def has_key(self,key): > self.check_key(key) > self.call_count['get'] += 1 > try : > self.last_haskey = key > return self.db.execute("SELECT data_id FROM `package_name` WHERE `name` = '%s' AND `cat_id` = '%d' AND `path_id` = '%d'" % (ex(key), self.cat_id, self.p_id )) > 0 > #return self.db.execute("SELECT t2.* FROM `package_name` AS t1, `items_table` as t2 WHERE t1.name = '%s' AND t1.cat_id = '%d' AND t1.path_id = '%d' AND t1.data_id = t2.i_id" % ( ex(key), self.cat_id, self.p_id )) > 0 > except Exception, e: > self.check_exception(e) > print "exception in has_key for Key (%s), Error (%s) " % (key, str(e)) > return False > > def keys(self): > print 'in keys' > ks = [] > self.call_count['keys'] += 1 > try: > #self.db.execute("SELECT t1.name, t2.* FROM `package_name` AS t1, `items_table` as t2 WHERE t1.cat_id ='%d' AND t1.path_id = '%d' AND t1.data_id = t2.i_id " % (self.cat_id, self.p_id )) > self.db.execute("SELECT name FROM `package_name` WHERE `cat_id` = '%d' AND `path_id` = '%d'" % (self.cat_id, self.p_id )) > for x in self.db.fetchall(): > ks += [x[0]] > return ks > except Exception, e: > self.check_exception(e) > raise KeyError, "Keys are dead :( (%s)" % str(e) > > #def __getitem__(self, key): > # self.has_key(key) > # return self.get_values(key) > #def __setitem(self, key, val): > # return self.set_values(key,val) > > def get_values(self,key): > self.check_key(key) > self.call_count['get'] += 1 > try: > #if self.last_haskey != key: self.has_key(key) > self.db.execute("SELECT t2.* FROM `package_name` AS t1, `items_table` as t2 WHERE t1.name = '%s' AND t1.cat_id = '%d' AND t1.path_id = '%d' AND t1.data_id = t2.i_id" % ( ex(key), self.cat_id, self.p_id )) > one = self.db.fetchone() > if len(one) > 0 and one[0] != None: > return self.table2dict(one[1:]) > except Exception, e: > self.check_exception(e) > raise ValueError, "Value error (%s)" % str(e) > return None > > def xget_values(self, key): > return sqldict(None, self.db.fetchone()[0], self.db) > > def set_values(self,key,val): > self.check_key(key) > self.call_count['set'] += 1 > try: > val = self.dict2table(val) > if self.db.execute("SELECT data_id FROM `package_name` WHERE `name` = '%s' AND `cat_id` = '%d' AND `path_id` = '%d'" % (ex(key), self.cat_id, self.p_id )) > 0: > self.db.execute(" REPLACE INTO `items_table` (`i_id`, `%s`) VALUES ( '%d', '%s')" % ( '`,`'.join(val.keys()), int(self.db.fetchone()[0]), "','".join(val.values()) )) > else: > self.db.execute("replace INTO `items_table` (`%s`) VALUES ('%s') " % ( '`,`'.join(val.keys()), "','".join(val.values()) ) ) > self.db.execute("replace INTO `package_name` (`name`,`cat_id`, `data_id`, `path_id`) VALUES ('%s', '%d', '%d', '%d')" % (ex(key), self.cat_id, self.db.insert_id(), self.p_id ) ) > except Exception, e: > self.check_exception(e) > raise Exception, "Error inserting/updating the database (%s)." % str(e) > > def del_key(self,key): > self.call_count['del'] += 1 > try: > return (self.db.execute('DELETE `package_name`, `items_table` FROM `package_name` AS t1, `items_table` as t2 WHERE t1.cat_id = "%d" AND t1.name= "%s" AND t1.path_id = "%d" AND t1.data_id = t2.i_id' % (self.cat_id, key, self.p_id)) > 0) > except Exception, e: > self.check_exception(e) > print "Error deleting key=(%s), exception= (%s)" % (key,str(e)) > return False > > def sync(self): > try: > self.db.execute('commit') > except Exception, e: > self.check_exception(e) > > def close(self): > print self.call_count > if database.connections.get(self.constr, [0])[0] == 0: return 0 # make sure that .close() doesn't get called twice > try: > #print 'in self.db.close(), instcount=%d' % database.connections[self.constr][0] > database.connections[self.constr][0] -= 1 > self.db.close() > if database.connections[self.constr][0] == 0: > database.connections[self.constr][1].close() > del database.connections[self.constr] > return 1 > except Exception, e: > self.check_exception(e) > print "Exception in self.db.close() == (%s), ignored." % str(e) > > def check_database(self): > try: > self.db.execute('use %s' % self.config['db']) > self.tables = [] > if self.db.execute('show tables') > 0: > for name in self.db.fetchall(): self.tables.append(name[0]) > self.setup_tables() > except Exception, e: > self.check_exception(e) > # e.args[0] == 1049 means the actual db is missing, too bad :-/ > if e.args[0] == 1049: > raise Exception("Please create the database (%s), and assign a user to it.") > else: > raise e > > > def table2dict(self, d, rid=1): # private and internal call, and very ugly... > rows = [ row[0] for row in self.db.description[rid:] ] > items = d #+ ('', '', '', '', '', '', '', '') > return dict(zip(rows, items)) > > def dict2table(self, d): # really ugly way to clean the dictionary passed to the class by portage, this is only a private internal call > td = d.copy() > for i in range(1,9): > del td['UNUSED_0%d' % i] > for v in td: > td[v] = ex(str(td[v])) > return td > > def setup_tables(self): > > tbls = { 'items_table': """ > CREATE TABLE `items_table` ( > i_id int PRIMARY KEY AUTO_INCREMENT, > DEPEND text, > RDEPEND text, > SLOT varchar(255), > SRC_URI text, > `RESTRICT` varchar(255), > HOMEPAGE varchar(255), > LICENSE varchar(255), > DESCRIPTION text, > KEYWORDS varchar(255), > INHERITED varchar(255), > IUSE text, > CDEPEND text, > PDEPEND text, > PROVIDE varchar(255), > _mtime_ varchar(255) > );""", 'path_table': """ > CREATE TABLE `path_table` ( > p_id int PRIMARY KEY AUTO_INCREMENT, > name varchar(255) NOT NULL UNIQUE > );""", 'category_table': """ > CREATE TABLE `category_table` ( > c_id int PRIMARY KEY AUTO_INCREMENT, > name varchar(255) NOT NULL UNIQUE > ) ;""", 'package_name' : """ > CREATE TABLE `package_name` ( > name varchar(255) NOT NULL, > cat_id int NOT NULL, > path_id int NOT NULL, > data_id int NOT NULL, > FOREIGN KEY (cat_id) REFERENCES category_table (c_id) ON DELETE CASCADE, > FOREIGN KEY (path_id) REFERENCES path_table (p_id) ON DELETE CASCADE, > primary key (name, cat_id, path_id) > ) MAX_ROWS=99999999;""" } > # loop for each table so if a table already exists it wouldnt stop creating the rest > for tn in tbls: > try: > if not tn in self.tables: > self.db.execute(tbls[tn]) > except Exception, e: > self.check_exception(e) > # e.args[0] == 1050 means table already exists, so just ignore that exception. > if not e.args[0] == 1050: print 'Died in setup_tables(), e == ', e > >class sqldict: > options = {} > data_id = 0 > db = None > def __init__(self, options, data_id, db): > self.options = options > self.data_id = data_id > self.db = db > def has_key(self, key): > return self.db.execute('SELECT %s FROM `items_table` WHERE i_id = "%d"' % (ex(key), self.data_id)) == 1 > > def __getitem__(self, key): > try: > return self.db.fetchone()[0] > except Exception, e: print e >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 51704
:
31813
|
31814
|
31815
|
32679
|
32680
| 37914 |
37916