Attachment #162768 for bug #208376




#!/usr/bin/env python
"""Packages2 CherryPy Application and launcher"""

import os, sys, math, string, re, types

import cherrypy


    """Packages2 CherryPy Application"""

    database = None
    regexp_atomic = None

    def __init__(self, db):
        self.database = db
        self.regexp_atomic = re.compile(r"(>|>=|=|<=|<)?([a-z0-9-]*/)?([a-z-]*)(-[0-9.]*)?(-r[0-9])?$", re.IGNORECASE)


    @property


    @cherrypy.expose
    @template.expire_on_30_min()
    @template.output('index.html', method='xhtml')
    def search(self, *args, **kwds):
        """Search for ebuilds"""
        return self._search(*args, **kwds)

    def _search(self, *args, **kwds):
        """Search for ebuilds"""
        sstring = string.strip(kwds["searchstring"])

        db = self.database
        entry_filter = filters.EntryFilters(db)
        
        sinfo = {
            "comparator": None,
            "categories": [],
            "pkgname": None,
            "version": None,
            "anyfield": None,
            }

        # process search string
        match = self.regexp_atomic.match(sstring)
        if match:
            [comparator, category, pkgname, version, revision] =  match.groups()
            sinfo["pkgname"] = pkgname
            
            # regexp matches single word too, so a single word must not be a package name
            if pkgname and not (category or version):
                sinfo["anyfield"] = pkgname
                sinfo["pkgname"] = None

            if version: # remove leading divis
                version = version[1:]
                if revision:
                    sinfo["version"] = version+revision
                else:
                    sinfo["version"] = version
                if not comparator:
                    sinfo["comparator"] = "="
                else:
                    sinfo["comparator"] = comparator
            if category:
                sinfo["categories"].append(category[:len(category)-1]) #remove tailing slash
        else:
            sinfo["anyfield"] = sstring

        limit = filters.limit_centercount(kwds)

        search_entries = entry_filter.search_filter(sinfo, limit=None)
        center_pkgs = build_centerpkg_list(search_entries,
                db.get_package_details_cpv, None)

        pagetitle = "/search"

        arches = filters.limit_arches(kwds)
        kwds = sanitize_query_string(kwds)
        latest_entries = self.cache_latest
        left_daycount = filters.limit_leftcount(kwds)
        day_list = latest_per_day(latest_entries, left_daycount)
        left_daycount = filters.limit_leftcount(kwds)
        arches = filters.limit_arches(kwds)
        latest_entry = entry_filter.latest_entry()

        db.close_mc()
        return template.render(arches = arches,
            daylist = day_list, center_pkgs = center_pkgs,
            lastupdate = latest_entry, safeqs = kwds,
            pagetitle = pagetitle)
        

    @cherrypy.expose
    @template.expire_on_30_min()
    @template.output('index.xml', method='xml')
    def feed(self, *args, **kwds):
        """Render the /feed/<someurl> page as Atom"""

            return self._category(*args, **kwds)
        elif base == "package":
            return self._package(*args, **kwds)
        elif base == "search":
            return self._search(*args, **kwds)
        elif base == "verbump":
            return self._verbump(*args, **kwds)
        elif base == "newpackage":

Lines 82-87 class EntryFilters(object): Link Here

(-)a/web/lib/filters.py (+8 lines)
82	return []	82	return []
83	return self.package_source.get_latest_cpvs_by_pkgname(pkgname, limit)	83	return self.package_source.get_latest_cpvs_by_pkgname(pkgname, limit)
84		84
		85	def search_filter(self, sinfo, limit=None):
		86	"""we don't use cache here"""
		87	return self._search_filter(sinfo, limit)
		88
		89	def _search_filter(self, sinfo, limit=None):
		90	"""filter packages by search criteria (uncached)"""
		91	return self.package_source.get_cpvs_by_searchcriteria(sinfo, limit)
		92
85	def category_package_filter(self, category, pkgname, limit=None):	93	def category_package_filter(self, category, pkgname, limit=None):
86	"""Skip packages not matching pkgname and category"""	94	"""Skip packages not matching pkgname and category"""
87	key = 'category_package_filter_%r%r%r' % (category, pkgname, limit)	95	key = 'category_package_filter_%r%r%r' % (category, pkgname, limit)

Lines 102-107 def create_rel(path): Link Here

(-)a/web/lib/query_filter.py (+2 lines)
102	t = '%s %s' % (m[3], m[2])	102	t = '%s %s' % (m[3], m[2])
103	elif path.startswith('/verbump'):	103	elif path.startswith('/verbump'):
104	t = 'Version bumps'	104	t = 'Version bumps'
		105	elif path.startswith('/search'):
		106	t = 'Search'
105	elif path.startswith('/newpackage'):	107	elif path.startswith('/newpackage'):
106	t = 'New packages'	108	t = 'New packages'
107	elif path.startswith('/faq'):	109	elif path.startswith('/faq'):




from time import localtime, strftime, time
import datetime
import re, sys, types
import operator
import cherrypy


    columns_category_pn_pv = None
    sql = {}
    mc = None
    valid_modes = ['+', '~', 'M', 'M+', 'M~', '']

    def __init__(self, config=None):
        # Do not complain about correct usage of ** magic

        cursor.close()
        return entries

    sql['SELECT_get_cpvs_by_searchcriteria'] = """
    SELECT __CPV__, versions.mtime
    FROM versions
        JOIN packages USING (cp)
        JOIN categories USING (c)
        __JOINS__
    WHERE __WHERE__
    ORDER by versions.mtime DESC
    LIMIT 0, ?
    """
    def get_cpvs_by_searchcriteria(self, sinfo, limit=None):
        """return cpvs matching the search criteria"""
        criteria = []
        params = []
        joins = []

        # strrep produces strings like "(?,?,?,?)"
        def strrep(chr, delimiter, count):
            ret = chr
            for i in range(count-1):
                ret += delimiter
                ret += chr
            return ret

        def add_criterion(arg, sql, join_info):
            if type(arg)==types.ListType:
                if replacer:
                    sql = sql % replacer
                criteria.append(sql)
                params.extend(arg)
            else:
                criteria.append(sql)
                params.append(arg)
            if join_info:
                if join_info not in joins:
                    joins.append(join_info)
            return

        sql_criteria = {
            "pkgname": "pn LIKE "+self.syntax_placeholder,
            "categories": "category IN (%s)",
            "category": "category LIKE "+self.syntax_placeholder,
            "version_eq": "pv = "+self.syntax_placeholder,
            "version_gt": "pv > "+self.syntax_placeholder,
            "version_ge": "pv >= "+self.syntax_placeholder,
            "version_lt": "pv < "+self.syntax_placeholder,
            "version_le": "pv <= "+self.syntax_placeholder,
            "anyfield": "((MATCH (description, changelog) AGAINST ("+self.syntax_placeholder+")) OR (MATCH (pn) AGAINST("+self.syntax_placeholder+")))"
            }
        
        if sinfo["pkgname"]:
            add_criterion("%"+sinfo["pkgname"]+"%", sql_criteria["pkgname"], None)

        if sinfo["categories"]:
            if len(sinfo["categories"]) > 1:
                add_criterion(sinfo["categories"], sql_criteria["categories"], None, strrep(self.syntax_placeholder, ",", len(sinfo["categories"])))
            else:
                add_criterion("%"+sinfo["categories"][0]+"%", sql_criteria["category"], None)

        if sinfo["version"]:
            available_comparators = {
                ">":  sql_criteria["version_gt"],
                ">=": sql_criteria["version_ge"],
                "=":  sql_criteria["version_eq"],
                "<=": sql_criteria["version_le"],
                "<":  sql_criteria["version_lt"]
            }
            if not sinfo["comparator"]:
                sinfo["comparator"] = "="
            if sinfo["comparator"] not in available_comparators.keys():
                return []
            add_criterion(sinfo["version"], available_comparators[sinfo["comparator"]], None)

        if sinfo["anyfield"]:
            add_criterion(sinfo["anyfield"], sql_criteria["anyfield"], ["metadata", "cp"])
            params.append(sinfo["anyfield"])
        
        if limit is None:
            limit = 1000
        try:
            limit = int(limit)
        except ValueError:
            return []
        params.append(limit)


        # build sql query string
        criteria_str = ""
        i = 0
        for c in criteria:
            if (i > 0) and i < len(criteria):
                criteria_str += " AND "
            criteria_str += c
            i += 1
        joins_str = ""
        for j in joins:
            joins_str += "JOIN %s USING (%s) " % (j[0], j[1])
            print joins_str

        sql = self.sql['SELECT_get_cpvs_by_searchcriteria']

        reps = []
        reps.append(('__JOINS__', joins_str))
        reps.append(('__WHERE__', criteria_str))
        # Only used once
        spacematch = re.compile(r'(\s+|\n)')
        for o, n in reps:
            sql = sql.replace(o, n)
            sql = spacematch.sub(' ', sql)


        cursor = self.cursor()
        cursor.execute(sql, params)
        entries = cursor.fetchall()
        cursor.close()
        return entries

    sql['SELECT_get_latest_cpvs_by_arch_mode'] = """
    SELECT __CPV__, versions.mtime
    FROM arches

    def get_latest_cpvs_by_arch(self, arch, mode, limit=None):
        """return modified cpvs limited by arch and mode"""

        valid_modes = self.valid_modes
        params = ()
        if mode not in valid_modes or arch not in self.arches:
            return []

Lines 212-217 a:visited { Link Here

(-)a/web/static/style.css (+4 lines)
212	background: #dddaec;	212	background: #dddaec;
213	}	213	}
214		214
		215	#rightcontent input.search {
		216	width: 8em;
		217	}
		218
215	#rightcontent p, #rightcontent ul {	219	#rightcontent p, #rightcontent ul {
216	/* margin: 1em; */	220	/* margin: 1em; */
217	}	221	}




      <p class="update">
        Last update:<br />${HTML(lastmodified_rightcontent(lastupdate))}
      </p>

      <div>
          <span class="left-legend">Search</span>
	  
	  <form action="search" method="get">
	    <input class="search" type="text" name="searchstring" />
	    <input type="submit" value="Go!" />
	  </form>
      </div>

      <div>
          <span class="left-legend">Legend</span>
          <ul class="legend left-bar-list" 

Return to bug 208376

Lines 1-7 Link Here

(-)a/web/controller.py (-1 / +77 lines)
1	#!/usr/bin/env python	1	#!/usr/bin/env python
2	"""Packages2 CherryPy Application and launcher"""	2	"""Packages2 CherryPy Application and launcher"""
3		3
4	import os, sys, math	4	import os, sys, math, string, re, types
5		5
6	import cherrypy	6	import cherrypy
7		7
Lines 20-28 class Root(object): Link Here
20	"""Packages2 CherryPy Application"""	20	"""Packages2 CherryPy Application"""
21		21
22	database = None	22	database = None
		23	regexp_atomic = None
23		24
24	def __init__(self, db):	25	def __init__(self, db):
25	self.database = db	26	self.database = db
		27	self.regexp_atomic = re.compile(r"(>\|>=\|=\|<=\|<)?([a-z0-9-]/)?([a-z-])(-[0-9.]*)?(-r[0-9])?$", re.IGNORECASE)
26		28
27		29
28	@property	30	@property
Lines 409-414 class Root(object): Link Here
409		411
410	@cherrypy.expose	412	@cherrypy.expose
411	@template.expire_on_30_min()	413	@template.expire_on_30_min()
		414	@template.output('index.html', method='xhtml')
		415	def search(self, args, *kwds):
		416	"""Search for ebuilds"""
		417	return self._search(args, *kwds)
		418
		419	def _search(self, args, *kwds):
		420	"""Search for ebuilds"""
		421	sstring = string.strip(kwds["searchstring"])
		422
		423	db = self.database
		424	entry_filter = filters.EntryFilters(db)
		425
		426	sinfo = {
		427	"comparator": None,
		428	"categories": [],
		429	"pkgname": None,
		430	"version": None,
		431	"anyfield": None,
		432	}
		433
		434	# process search string
		435	match = self.regexp_atomic.match(sstring)
		436	if match:
		437	[comparator, category, pkgname, version, revision] = match.groups()
		438	sinfo["pkgname"] = pkgname
		439
		440	# regexp matches single word too, so a single word must not be a package name
		441	if pkgname and not (category or version):
		442	sinfo["anyfield"] = pkgname
		443	sinfo["pkgname"] = None
		444
		445	if version: # remove leading divis
		446	version = version[1:]
		447	if revision:
		448	sinfo["version"] = version+revision
		449	else:
		450	sinfo["version"] = version
		451	if not comparator:
		452	sinfo["comparator"] = "="
		453	else:
		454	sinfo["comparator"] = comparator
		455	if category:
		456	sinfo["categories"].append(category[:len(category)-1]) #remove tailing slash
		457	else:
		458	sinfo["anyfield"] = sstring
		459
		460	limit = filters.limit_centercount(kwds)
		461
		462	search_entries = entry_filter.search_filter(sinfo, limit=None)
		463	center_pkgs = build_centerpkg_list(search_entries,
		464	db.get_package_details_cpv, None)
		465
		466	pagetitle = "/search"
		467
		468	arches = filters.limit_arches(kwds)
		469	kwds = sanitize_query_string(kwds)
		470	latest_entries = self.cache_latest
		471	left_daycount = filters.limit_leftcount(kwds)
		472	day_list = latest_per_day(latest_entries, left_daycount)
		473	left_daycount = filters.limit_leftcount(kwds)
		474	arches = filters.limit_arches(kwds)
		475	latest_entry = entry_filter.latest_entry()
		476
		477	db.close_mc()
478	return template.render(arches = arches,
479	daylist = day_list, center_pkgs = center_pkgs,
480	lastupdate = latest_entry, safeqs = kwds,
481	pagetitle = pagetitle)
482
483
484	@cherrypy.expose
485	@template.expire_on_30_min()
412	@template.output('index.xml', method='xml')	486	@template.output('index.xml', method='xml')
413	def feed(self, args, *kwds):	487	def feed(self, args, *kwds):
414	"""Render the /feed/<someurl> page as Atom"""	488	"""Render the /feed/<someurl> page as Atom"""
Lines 427-432 class Root(object): Link Here
427	return self._category(args, *kwds)	501	return self._category(args, *kwds)
428	elif base == "package":	502	elif base == "package":
429	return self._package(args, *kwds)	503	return self._package(args, *kwds)
		504	elif base == "search":
		505	return self._search(args, *kwds)
430	elif base == "verbump":	506	elif base == "verbump":
431	return self._verbump(args, *kwds)	507	return self._verbump(args, *kwds)
432	elif base == "newpackage":	508	elif base == "newpackage":

Lines 1-6 Link Here

(-)a/web/model.py (-2 / +120 lines)
1	from time import localtime, strftime, time	1	from time import localtime, strftime, time
2	import datetime	2	import datetime
3	import re, sys	3	import re, sys, types
4	import operator	4	import operator
5	import cherrypy	5	import cherrypy
6		6
Lines 82-87 class PackageDB(object): Link Here
82	columns_category_pn_pv = None	82	columns_category_pn_pv = None
83	sql = {}	83	sql = {}
84	mc = None	84	mc = None
		85	valid_modes = ['+', '~', 'M', 'M+', 'M~', '']
85		86
86	def __init__(self, config=None):	87	def __init__(self, config=None):
87	# Do not complain about correct usage of ** magic	88	# Do not complain about correct usage of ** magic
Lines 348-353 class PackageDB(object): Link Here
348	cursor.close()	349	cursor.close()
349	return entries	350	return entries
350		351
		352	sql['SELECT_get_cpvs_by_searchcriteria'] = """
		353	SELECT __CPV__, versions.mtime
		354	FROM versions
		355	JOIN packages USING (cp)
		356	JOIN categories USING (c)
		357	__JOINS__
		358	WHERE __WHERE__
		359	ORDER by versions.mtime DESC
		360	LIMIT 0, ?
		361	"""
		362	def get_cpvs_by_searchcriteria(self, sinfo, limit=None):
		363	"""return cpvs matching the search criteria"""
		364	criteria = []
		365	params = []
		366	joins = []
		367
		368	# strrep produces strings like "(?,?,?,?)"
		369	def strrep(chr, delimiter, count):
		370	ret = chr
		371	for i in range(count-1):
		372	ret += delimiter
		373	ret += chr
		374	return ret
		375
		376	def add_criterion(arg, sql, join_info):
		377	if type(arg)==types.ListType:
		378	if replacer:
		379	sql = sql % replacer
		380	criteria.append(sql)
		381	params.extend(arg)
		382	else:
		383	criteria.append(sql)
		384	params.append(arg)
		385	if join_info:
		386	if join_info not in joins:
		387	joins.append(join_info)
		388	return
		389
		390	sql_criteria = {
		391	"pkgname": "pn LIKE "+self.syntax_placeholder,
		392	"categories": "category IN (%s)",
		393	"category": "category LIKE "+self.syntax_placeholder,
		394	"version_eq": "pv = "+self.syntax_placeholder,
		395	"version_gt": "pv > "+self.syntax_placeholder,
		396	"version_ge": "pv >= "+self.syntax_placeholder,
		397	"version_lt": "pv < "+self.syntax_placeholder,
		398	"version_le": "pv <= "+self.syntax_placeholder,
		399	"anyfield": "((MATCH (description, changelog) AGAINST ("+self.syntax_placeholder+")) OR (MATCH (pn) AGAINST("+self.syntax_placeholder+")))"
		400	}
		401
		402	if sinfo["pkgname"]:
		403	add_criterion("%"+sinfo["pkgname"]+"%", sql_criteria["pkgname"], None)
		404
		405	if sinfo["categories"]:
		406	if len(sinfo["categories"]) > 1:
		407	add_criterion(sinfo["categories"], sql_criteria["categories"], None, strrep(self.syntax_placeholder, ",", len(sinfo["categories"])))
		408	else:
		409	add_criterion("%"+sinfo["categories"][0]+"%", sql_criteria["category"], None)
		410
		411	if sinfo["version"]:
		412	available_comparators = {
		413	">": sql_criteria["version_gt"],
		414	">=": sql_criteria["version_ge"],
		415	"=": sql_criteria["version_eq"],
416	"<=": sql_criteria["version_le"],
417	"<": sql_criteria["version_lt"]
418	}
419	if not sinfo["comparator"]:
420	sinfo["comparator"] = "="
421	if sinfo["comparator"] not in available_comparators.keys():
422	return []
423	add_criterion(sinfo["version"], available_comparators[sinfo["comparator"]], None)
424
425	if sinfo["anyfield"]:
426	add_criterion(sinfo["anyfield"], sql_criteria["anyfield"], ["metadata", "cp"])
427	params.append(sinfo["anyfield"])
428
429	if limit is None:
430	limit = 1000
431	try:
432	limit = int(limit)
433	except ValueError:
434	return []
435	params.append(limit)
436
437
438	# build sql query string
439	criteria_str = ""
440	i = 0
441	for c in criteria:
442	if (i > 0) and i < len(criteria):
443	criteria_str += " AND "
444	criteria_str += c
445	i += 1
446	joins_str = ""
447	for j in joins:
448	joins_str += "JOIN %s USING (%s) " % (j[0], j[1])
449	print joins_str
450
451	sql = self.sql['SELECT_get_cpvs_by_searchcriteria']
452
453	reps = []
454	reps.append(('__JOINS__', joins_str))
455	reps.append(('__WHERE__', criteria_str))
456	# Only used once
457	spacematch = re.compile(r'(\s+\|\n)')
458	for o, n in reps:
459	sql = sql.replace(o, n)
460	sql = spacematch.sub(' ', sql)
461
462
463	cursor = self.cursor()
464	cursor.execute(sql, params)
465	entries = cursor.fetchall()
466	cursor.close()
467	return entries
468
351	sql['SELECT_get_latest_cpvs_by_arch_mode'] = """	469	sql['SELECT_get_latest_cpvs_by_arch_mode'] = """
352	SELECT __CPV__, versions.mtime	470	SELECT __CPV__, versions.mtime
353	FROM arches	471	FROM arches
Lines 373-379 class PackageDB(object): Link Here
373	def get_latest_cpvs_by_arch(self, arch, mode, limit=None):	491	def get_latest_cpvs_by_arch(self, arch, mode, limit=None):
374	"""return modified cpvs limited by arch and mode"""	492	"""return modified cpvs limited by arch and mode"""
375		493
376	valid_modes = ['+', '~', 'M', 'M+', 'M~', '']	494	valid_modes = self.valid_modes
377	params = ()	495	params = ()
378	if mode not in valid_modes or arch not in self.arches:	496	if mode not in valid_modes or arch not in self.arches:
379	return []	497	return []

Lines 56-62 def alpha_url(baseurl): Link Here

(-)a/web/templates/layout.html (-1 / +10 lines)
56	<p class="update">	56	<p class="update">
57	Last update:<br />${HTML(lastmodified_rightcontent(lastupdate))}	57	Last update:<br />${HTML(lastmodified_rightcontent(lastupdate))}
58	</p>	58	</p>
59		59
		60	<div>
		61	<span class="left-legend">Search</span>
		62
		63	<form action="search" method="get">
		64	<input class="search" type="text" name="searchstring" />
		65	<input type="submit" value="Go!" />
		66	</form>
		67	</div>
		68
60	<div>	69	<div>
61	<span class="left-legend">Legend</span>	70	<span class="left-legend">Legend</span>
62	<ul class="legend left-bar-list"	71	<ul class="legend left-bar-list"