bw-script v1

bw-script.py (text/plain), 11.25 KB, created by Sebastian Luther (few) on 2009-10-06 15:00:48 UTC

#!/usr/bin/python

import calendar
import os
import sys
import time
import xml.etree.cElementTree as ET

from bugz.bugzilla import Bugz
import portage
from gentoolkit.equery.meta import get_package_directory, get_herd_email, get_maitainer


####gentoolkit stuff ############

def get_herd(xml_tree):
	#Modified version of the gentoolkit.equery.meta function
	
	result = []
	for elem in xml_tree.findall("herd"):
		herd_mail = get_herd_email(elem.text)
		result.append(herd_mail)

	return result
####end of gentoolkit stuff ############


#make pybugz.Bugz happy
class my_bugz(Bugz):
	def __init__(self):
		Bugz.__init__(self, "https://bugs.gentoo.org/")
		
	def get_input(self, prompt):
		return raw_input(prompt)

#Initialize the Bugz bugzilla Interface
bugz = my_bugz()


def get_pv_from_word(word):
	"""Checks if a given word corresponds to a existing package
	and returns this package"""
	
	#substitute commenly used wrong package names
	if word == "firefox":
		word = "mozilla-firefox"
	elif word == "thunderbird":
		word = "mozilla-thunderbird"
	elif word == "apache2":
		word = "apache"
	elif word in ("oobase", "oocalc", "oodraw", "ooffice", "ooimpress", \
					"oomath", "ooweb", "oowriter"):
		word = "openoffice"
	
	#filter words that often generate false positives
	if word.lower() in ("at", "binary", "build", "check", "dialog", \
		"emerge", "enable", "fetch", "file", "find", "gentoo", "icon", "init", \
		"instead", "javascript", "less", "locale", "man", "more", "patch", \
		"player", "portage", "png", "rpm", "sandbox", "screen", \
		"scripts", "session", "ssh", "window" , "version", "when" ):
		return
	
	#strip the version information
	pkg_name = portage.pkgsplit(word)
		
	#in case there is no version information, use the word
	if pkg_name is None:
		pkg_name = [ word ]
			
	try:
		pkgs = portage.db['/']['porttree'].dbapi.xmatch("match-all", pkg_name[0])
		
		if pkgs:
			return pkg_name[0]

	except portage.exception.InvalidAtom:
		pass
	except portage.exception.AmbiguousPackageName:
		pass
	except UnicodeDecodeError:
		pass
			
	return

def get_cpv_from_word(word):
	"""Checks if a given word corresponds to an existing cat/pkg
	and returns the package. Is assumes that 'word' contains at
	least one occurence of '/'"""
	
	splitted_word = word.split("/")
	if len(splitted_word) > 2:
		return
			
	try:
		dep_ex = portage.dep_expand(word)
		atom_cp = dep_ex.cp
		pkg = portage.db['/']['porttree'].dbapi.xmatch("match-all", atom_cp)
			
		if pkg:
			return atom_cp
				
	except portage.exception.InvalidAtom:
		pass

	return
	
def out_bug(bug, reason):
	print bug["bugid"], " ", bug["desc"]
	if reason:
		print "not done:", reason

def get_packages_from_summary(summary):
	"""Takes a summary line and return two lists of packe names.
	The first list contains occurences of cat/pkg, the second occurences
	of a package anme without it's category."""
	
	cpv_matches = []
	pv_matches = []
	
	for split_word in summary.split(" "):
		#strip useless stuff, sometimes added by users
		word = split_word.strip("(),\":.")
		if len(word) > 2 and word[-2:] == "'s":
			word = word[:-2]
		
		if not word:
			continue
		
		if "/" in word:
			new_pkg = get_cpv_from_word(word)
			if new_pkg and new_pkg not in cpv_matches:
				cpv_matches.append(new_pkg)
		else:
			new_pkg = get_pv_from_word(word)
			if new_pkg is None:
				new_pkg = get_pv_from_word(word.lower())
			if new_pkg:
				if new_pkg and new_pkg not in pv_matches:
						pv_matches.append(new_pkg)
	
	return cpv_matches, pv_matches

def get_bug_features(bug, bug_data, cpv_list, pv_list):
	"""This function returns a list of bug_features found in a bug.
	Possible bug_features are stable_req, keyword_req, version_bump,
	attachment, emerge_info."""
	
	bug_features = []

	keywords = None
	
	#FIXME: I don't understand why the .find() approach used for
	#"reporter" isn't working here
	search_iter = bug_data.getiterator("keywords")
	for i in search_iter:
		keywords = i.text

	if any( x in bug["desc"].lower() for x in ["stable request", \
		"mark stable"] ) or ( keywords and "STABLEREQ" in keywords ):
		bug_features.append("stable_req")
		
	if ( "keyword" in bug["desc"].lower() and len(cpv_list) == 1 ) or \
		( keywords and "KEYWORDREQ" in keywords ):
		bug_features.append("keyword_req")
	
	if any( x in bug["desc"].lower() for x in ["new version", \
		"released", "version bump"] ) and len(cpv_list) == 1:
		bug_features.append("version_bump")
		
	if bug_data.findall('//attachment'):
		bug_features.append("attachment")
	
	#Scan the comments for something that looks like emerge --info.
	#For me len(emerge --info) is ~11500. I saw emerge --info with <5000
	for bug_comment in bug_data.findall('//long_desc'):
		comment = bug_comment.find("thetext").text
		if len(comment) < 2000:
			continue
		
		#check if a random collect of stuff from emerge --info is present
		
		if all( x in comment for x in ["CFLAGS", "CXXFLAGS", "CHOST", \
			"ACCEPT_KEYWORDS"] ):
			bug_features.append("emerge_info")
			break

	return bug_features

		
def main():
	
	do_it = False	#really assign the bug
	debug = False
	
	
	cmd_line = sys.argv[1:]
	if "--do-it" in cmd_line:
		cmd_line.remove("--do-it")
		do_it = True
	
	if "--debug" in cmd_line:
		cmd_line.remove("--debug")
		debug = True
	
	#get the bugs
	if cmd_line:
		bug_list = bugz.search( " ".join(cmd_line))
	else:
		bug_list = bugz.search( "", assigned_to = "[e-mail]",
			status = [ "NEW" ] )
	
	print "bugs fetched: ", len(bug_list)
	
	assigned = 0
	unassigned = 0
	
	for bug in bug_list:
		
		print "----------------"
		summary = bug["desc"]
		bug_data = bugz.get(bug["bugid"])
		
		#check for multi submission
		reporter = bug_data.find('//%s' % "reporter").text
		bug_copy_list = bugz.search( summary, reporter=reporter )
		
		if len(bug_copy_list) > 1:
			if bug_copy_list[0]["bugid"] != bug["bugid"]:
				out_bug(bug, "")
				print "duplicate of bug " + bug_copy_list[0]["bugid"]
				if do_it:
					bugz.modify( bug["bugid"], resolution = "DUPLICATE", \
						duplicate = bug_copy_list[0]["bugid"] )
				assigned += 1
				continue
	
		#TODO: handle overlay bugs
		#one could search for [$OVERLAY overlay] and use layman to get
		#the correct assignee for the overlay
		if "overlay" in summary:
			out_bug(bug, "I don't handle overlay bugs")
			unassigned += 1
			continue
		
		
		#Is the bug older than 30 minutes?
		#Give the reporter time to add stuff and others time to 
		#add comments.
		creation_time = time.strptime( bug_data.find('//%s' % \
			"creation_ts").text, "%Y-%m-%d %H:%M 0000" )
		time_sice_submission = calendar.timegm(time.gmtime()) - \
			calendar.timegm(creation_time)
		if time_sice_submission < 1800:
			out_bug(bug, "I only handle bugs, older than 30 minutes.")
			unassigned += 1
			continue

		#Has the bug been touched by a person with edit rights?
		#Prevent us from doing edit wars in case the bot assigns to
		#the wrong person and this person reassignes to bw@g.o
		
		for node in bug_data.getiterator():
			if node.tag == "bugzilla":
				my_email = node.attrib["exporter"]
				break

		dont_touch_it = False
		for bug_comment in bug_data.findall('//long_desc'):
			commenter = bug_comment.find("who").text

			if any( x in commenter for x in [my_email, "@gentoo.org"] ):
				dont_touch_it = True
				break

		if dont_touch_it:
			out_bug(bug, "A @g.o person or myself commented on the bug.")
			unassigned += 1
			continue
		
		#Parse the summary line and get a list of packages.
		cpv_list, pv_list = get_packages_from_summary(summary)
		
		#Prever cpv matches over cp match. The assumption is,
		#that if a user used a cpv for a package once, he will do 
		#for every important package
		if cpv_list:
			pkg_list = cpv_list
		else:
			pkg_list = pv_list
		
		if not pkg_list:
			out_bug(bug, "No package found in summary.")
			unassigned += 1
			continue
		
		#Convert the package list to email adresses.
		#Assign to the first maintainer ot the first package in the list,
		#CC the rest. Only exception is when the assignee would be
		#m-needed@g.o.
		
		assignee = None
		CC = set()
		
		for pkg in pkg_list:
			
			#The following lines are based on stuff from gentoolkit.equery.meta
			package_dir = get_package_directory(pkg)
			if not package_dir:
				raise errors.GentoolkitNoMatches(pkg)
			metadata_path = os.path.join(package_dir, "metadata.xml")
			
			try:
				xml_tree = ET.parse(metadata_path)
			except IOError:
				print("No metadata available")
				return
			
			herd_list = get_herd(xml_tree)
			maintainer_list = get_maitainer(xml_tree)

			if maintainer_list:
				for maintainer in maintainer_list:
					if not "@" in maintainer:
						#maintainer can contain comments
						#feel free to find better way to check for a
						#valid email but be aware that you can't check 
						#for @g.o, since some packages have proxy-
						#maintainers
						continue
						
					if not assignee or assignee == \
						"[e-mail]":
						assignee = maintainer
					else:
						if assignee != maintainer:
							CC.add(maintainer)
			
			if herd_list:
				for herd in herd_list:
					if herd == None:
						continue
					if not assignee or assignee == \
						"[e-mail]":
						assignee = herd
					else:
						if assignee != herd:
							CC.add(herd)
		
		if debug:
			print "pkg_list:", pkg_list
			print "maintainer =", maintainer_list
			print "herd =", herd_list
		
		if not assignee:
			print "error: metadat.xml contains no maintainer"
			unassigned += 1
			continue
		
		#Find out if we really want to assign this bug
		#Does it have emerge --info?
		#Are there attachments? (most probably a patch/log)
		#Can we find out if it is a stable/keyword request?
		#Other ideas how to validate the bug?
		#Check total size of comments?
		
		#get bug features (emerge_info attachment keyword-req stable-req
		#version_bump)
		bug_features = get_bug_features(bug, bug_data, cpv_list, pv_list)
		
		if not bug_features:
			out_bug(bug, "No bug features found (e.g. no emerge --info, ...)")
			unassigned += 1
			continue
		
		out_bug(bug, "")
		
		print "bug_features:",
		for feature in bug_features:
			print feature,
		print ""
		
		print "assignee:    ", assignee
		if CC:
			print "CC:          ",
			for email in CC:
				print email,
			print ""
			
		#Submit the changes to bugzilla
		
		comment = "This bug was automatically assigned. If you think it hit the wrong\n"
		comment += "person or isn't ready for assignment, re-assign to [e-mail]\n"
		comment += "and CC [e-mail] or comlain in #gentoo-bugs on irc.\n"
		if len(pkg_list) > 1:
			comment += "The following packages were found in the summary:\n"
		else:
			comment += "The following package was found in the summary:\n"
		comment += "     "
		for pkg in pkg_list:
			comment += " " + str(pkg)
		comment += "\n"
		
		comment += "bug features:"
		for feature in bug_features:
			comment += " " + str(feature)
		comment += "\n"
		
		if not cpv_list:
			comment += "Warning: No cat/pkg found. Please use the full packge name in your\n"
			comment += "         next bug report, if possible."
		
		if debug:
			print comment
		
		if do_it:
			if CC:
				bugz.modify( bug["bugid"], comment = comment, \
					assigned_to = assignee, add_cc = list(CC) )
			else:
				bugz.modify( bug["bugid"], comment = comment, \
					assigned_to = assignee )
		
		assigned += 1
		
	print "assigned:  ", assigned
	print "unassigned:", unassigned
	print "total:     ", len(bug_list)

main()

#reference the test bug in the bug