#!/usr/bin/env python
# Copyright 2003-2005 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: $

# THANKS:
# The starting point ideas were found here:
# http://forums.gentoo.org/viewtopic.php?t=3011
# Thanks to eswanson and far for their contributions, and to wolf31o2 for his
# support.  Thanks also to karltk, some of this code was at some point inspired
# by his "equery" tool.  And thanks to people who had a look on bug #33877: 
# Benjamin Braatz, fuzzyray, genone, etc.

# TODO:
# - exclusion file syntax could be improved (maybe it should support real
#   dep-atoms, or wildcards, etc.)
# - some policy to keep the X latest versions of a package (in each of its 
#   SLOT maybe) would be really cool...
# - add an option to protect system binary packages
#   => yup, but later... (needs some portage modifications to be done right)
# - add actions for PORT_LOGDIR and/or /var/tmp/portage cleaning?
#   => bah, don't know... imho tmpreaper or find+rm onliners are enough here
# - cleanup of DISTDIR/cvs-src when action=distfiles
#   => i never use cvs ebuilds, i should check what it does exactly
# - rewrite for a decent Portage API if there ever is one

# ChangeLog:
# 0.4 --> 0.4.1
# - added support for some "eclean-dist" and "eclean-pkg" symlinks on "eclean"
#   (and thus refactored command-line parsing and help screen code)
# - accept file names in exclude files for specific distfiles protection
#   (useful to protect the OOo i18n files for instance, which are not in
#   $SRC_URI but put there manually)
# - minor rewrite of some findDistfiles() code
# - added /usr/lib/portage/pym python path, just to be sure it comes first
#   (after all, "ouput" is a pretty generic name for a python module...)
# 0.2 --> 0.4
# - added exclusion files support
# - added time limit option
# - added size limit option (for distfiles only)
# - added fetch-restricted distfile optionnal protection
# - added --package-names option for protection of all versions of installed
#   packages
# - removed support of multiple actions on command-line. That would have been
#   hell with action-specific options.

# BUGS:
# No real bug found yet, see TODO for stuffs which could be improved/modified.


###############################################################################
# Meta:
__author__ = "Thomas de Grenier de Latour (tgl)"
__email__ = "degrenier@easyconnect.fr"
__version__ = "0.4.1"
__productname__ = "eclean"
__description__ = "A cleaning tool for Gentoo distfiles and binaries."


###############################################################################
# Python imports:
import sys
import os, stat
import string, re
import time
import getopt
import fpformat
import signal
sys.path.insert(0,'/usr/lib/portage/pym')
import portage
from output import *

# Ignominous CVS / non-CVS compatibility hacks:
if portage.VERSION[:6] == '2.0.51': listdir = portage.listdir
# else, assume it's a reasonably recent CVS (or future 2.1)
else: from portage_file import listdir


###############################################################################
# Misc. shortcuts to some portage stuff:
port_settings = portage.settings
distdir = port_settings["DISTDIR"]
pkgdir = port_settings["PKGDIR"]


###############################################################################
# printVersion:
def printVersion():
	print "%s (version %s) - %s" \
			% (__productname__, __version__, __description__)
	print "Author: %s <%s>" % (__author__,__email__)
	print "Copyright 2003-2005 Gentoo Foundation"
	print "Distributed under the terms of the GNU General Public License v2"


###############################################################################
# printUsage: print help message. May also print partial help to stderr if an
# error from {'options','actions'} is specified.
def printUsage(error=None,help=None):
	out = sys.stdout
	if error: out = sys.stderr
	if not error in ('actions', 'global-options', \
			'packages-options', 'distfiles-options', \
			'merged-packages-options', 'merged-distfiles-options', \
			'time', 'size'):
		error = None
	if not error and not help: help = 'all'
	if error == 'time':
		eerror("Wrong time specification")
		print >>out, "Time specification should be an integer followed by a"+ \
				" single letter unit."
		print >>out, "Available units are: y (years), m (months), w (weeks), "+ \
				"d (days) and h (hours)."
		print >>out, "For instance: \"1y\" is \"one year\", \"2w\" is \"two"+ \
				" weeks\", etc. "
		return
	if error == 'size':
		eerror("Wrong size specification")
		print >>out, "Size specification should be an integer followed by a"+ \
				" single letter unit."
		print >>out, "Available units are: G, M, K and B."
		print >>out, "For instance: \"10M\" is \"ten megabytes\", \"200K\" "+ \
				"is \"two hundreds kilobytes\", etc."
		return
	if error in ('global-options', 'packages-options', 'distfiles-options', \
			'merged-packages-options', 'merged-distfiles-options',):
		eerror("Wrong option on command line.")
		print >>out
	elif error == 'actions':
		eerror("Wrong or missing action name on command line.")
		print >>out
	print >>out, white("Usage:")
	if error in ('actions','global-options', 'packages-options', \
	'distfiles-options') or help == 'all':
		print >>out, " "+turquoise(__productname__), \
			yellow("[global-option] ..."), \
			green("<action>"), \
			yellow("[action-option] ...")
	if error == 'merged-distfiles-options' or help in ('all','distfiles'):
		print >>out, " "+turquoise(__productname__+'-dist'), \
			yellow("[global-option, distfiles-option] ...")
	if error == 'merged-packages-options' or help in ('all','packages'):
		print >>out, " "+turquoise(__productname__+'-pkg'), \
			yellow("[global-option, packages-option] ...")
	if error in ('global-options', 'actions'):
		print >>out, " "+turquoise(__productname__), \
			yellow("[--help, --version]")
	if help == 'all':
		print >>out, " "+turquoise(__productname__+"(-dist,-pkg)"), \
			yellow("[--help, --version]")
	if error == 'merged-packages-options' or help == 'packages':
		print >>out, " "+turquoise(__productname__+'-pkg'), \
			yellow("[--help, --version]")
	if error == 'merged-distfiles-options' or help == 'distfiles':
		print >>out, " "+turquoise(__productname__+'-dist'), \
			yellow("[--help, --version]")
	print >>out
	if error in ('global-options', 'merged-packages-options', \
	'merged-distfiles-options') or help:
		print >>out, "Available global", yellow("options")+":"
		print >>out, yellow(" -C, --nocolor")+ \
			"             - turn off colors on output"
		print >>out, yellow(" -d, --destructive")+ \
			"         - only keep the minimum for a reinstallation"
		print >>out, yellow(" -e, --exclude-file=<path>")+ \
			" - path to the exclusion file"
		print >>out, yellow(" -i, --interactive")+ \
			"         - ask confirmation before deletions"
		print >>out, yellow(" -n, --package-names")+ \
			"       - protect all versions (when --destructive)"
		print >>out, yellow(" -p, --pretend")+ \
			"             - only display what would be cleaned"
		print >>out, yellow(" -q, --quiet")+ \
			"               - be as quiet as possible"
		print >>out, yellow(" -t, --time-limit=<time>")+ \
			"   - don't delete files modified since "+yellow("<time>")
		print >>out, "   "+yellow("<time>"), "is a duration: \"1y\" is"+ \
				" \"one year\", \"2w\" is \"two weeks\", etc. "
		print >>out, "   "+"Units are: y (years), m (months), w (weeks), "+ \
				"d (days) and h (hours)."
		print >>out, yellow(" -h, --help")+ \
			"                - display the help screen"
		print >>out, yellow(" -V, --version")+ \
			"             - display version info"
		print >>out
	if error == 'actions' or help == 'all':
		print >>out, "Available", green("actions")+":"
		print >>out, green(" packages")+ \
			"      - clean outdated binary packages from:"
		print >>out, "                  ",teal(pkgdir)
		print >>out, green(" distfiles")+ \
			"     - clean outdated packages sources files from:"
		print >>out, "                  ",teal(distdir)
		print >>out
	if error in ('packages-options','merged-packages-options') \
	or help in ('all','packages'):
		print >>out, "Available", yellow("options"),"for the", \
				green("packages"),"action:"
		print >>out, yellow(" NONE  :)")
		print >>out
	if error in ('distfiles-options', 'merged-distfiles-options') \
	or help in ('all','distfiles'):
		print >>out, "Available", yellow("options"),"for the", \
				green("distfiles"),"action:"
		print >>out, yellow(" -f, --fetch-restricted")+ \
			"   - protect fetch-restricted files (when --destructive)"
		print >>out, yellow(" -s, --size-limit=<size>")+ \
			"  - don't delete disfiles bigger than "+yellow("<size>")
		print >>out, "   "+yellow("<size>"), "is a size specification: "+ \
				"\"10M\" is \"ten megabytes\", \"200K\" is"
		print >>out, "   "+"\"two hundreds kilobytes\", etc.  Units are: "+ \
				"G, M, K and B."
		print >>out
	print >>out, "More detailed instruction can be found in", \
			turquoise("`man %s`" % __productname__)


###############################################################################
# einfo: display an info message depending on a color mode
def einfo(message="", nocolor=False):
	if not nocolor: prefix = " "+green('*')
	else: prefix = ">>>"
	print prefix,message


###############################################################################
# eerror: display an error depending on a color mode
def eerror(message="", nocolor=False):
	if not nocolor: prefix = " "+red('*')
	else: prefix = "!!!"
	print >>sys.stderr,prefix,message


###############################################################################
# eprompt: display a user question depending on a color mode.
def eprompt(message, nocolor=False):
	if not nocolor: prefix = " "+red('>')+" "
	else: prefix = "??? "
	sys.stdout.write(prefix+message)
	sys.stdout.flush()


###############################################################################
# prettySize: integer -> byte/kilo/mega/giga converter. Optionnally justify the
# result. Output is a string.
def prettySize(size,justify=False):
	units = [" G"," M"," K"," B"]
	approx = 0
	while len(units) and size >= 1000:
		approx = 1
		size = size / 1024.
		units.pop()
	sizestr = fpformat.fix(size,approx)+units[-1]
	if justify: 
		sizestr = " " + blue("[ ") + " "*(7-len(sizestr)) \
		          + green(sizestr) + blue(" ]")
	return sizestr


###############################################################################
# yesNoAllPrompt: print a prompt until user answer in yes/no/all. Return a 
# boolean for answer, and also may affect the 'accept_all' option.
# Note: i gave up with getch-like functions, to much bugs in case of escape
# sequences. Back to raw_input.
def yesNoAllPrompt(myoptions,message="Do you want to proceed?"):
	user_string="xxx"
	while not user_string.lower() in ["","y","n","a","yes","no","all"]:
		eprompt(message+" [Y/n/a]: ", myoptions['nocolor'])
		user_string = raw_input()
	if user_string.lower() in ["a","all"]:
		myoptions['accept_all'] = True
	myanswer = user_string.lower() in ["","y","a","yes","all"]
	return myanswer


###############################################################################
# ParseArgsException: for parseArgs() -> main() communication
class ParseArgsException(Exception):
	def __init__(self, value):
		self.value = value
	def __str__(self):
		return repr(self.value)


###############################################################################
# parseSize: convert a file size "Xu" ("X" is an integer, and "u" in [G,M,K,B])
# into an integer (file size in Bytes). Raises ParseArgsException('size') in
# case of failure.
def parseSize(size):
	myunits = { \
		'G': (1024**3), \
		'M': (1024**2), \
		'K': 1024, \
		'B': 1 \
	}
	try:
		mymatch = re.match(r"^(?P<value>\d+)(?P<unit>[GMKBgmkb])?$",size)
		mysize = int(mymatch.group('value'))
		if mymatch.group('unit'):
			mysize *= myunits[string.capitalize(mymatch.group('unit'))]
	except:
		raise ParseArgsException('size')
	return mysize


###############################################################################
# parseTime: convert a duration "Xu" ("X" is an int, and "u" a time unit in 
# [Y,M,W,D,H]) into an integer which is a past EPOCH date. 
# Raises ParseArgsException('time') in case of failure.
# (yep, big approximations inside... who cares?)
def parseTime(timespec):
	myunits = {'H' : (60 * 60)}
	myunits['D'] = myunits['H'] * 24
	myunits['W'] = myunits['D'] * 7
	myunits['M'] = myunits['D'] * 30
	myunits['Y'] = myunits['D'] * 365
	try:
		# parse the time specification
		mymatch = re.match(r"^(?P<value>\d+)(?P<unit>[YMWDHymwdh])?$",timespec)
		myvalue = int(mymatch.group('value'))
		if not mymatch.group('unit'): myunit = 'D'
		else: myunit = string.capitalize(mymatch.group('unit'))
	except: raise ParseArgsException('time')
	# calculate the limit EPOCH date
	mytime = time.time() - (myvalue * myunits[myunit])
	return mytime


###############################################################################
# parseCmdLine: parse the command line arguments. Raise exceptions on errors or
# non-action modes (help/version). Returns an action, and affect the options
# dict.
def parseArgs(myoptions={}):

	# local function for interpreting command line options 
	# and setting myoptions accordingly
	def optionSwitch(myoption,opts,action=None):
		return_code = True
		for o, a in opts:
			if o in ("-h", "--help"):
				if action: raise ParseArgsException('help-'+action)
				else: raise ParseArgsException('help')
			elif o in ("-V", "--version"):
				raise ParseArgsException('version')
			elif o in ("-C", "--nocolor"):
				myoptions['nocolor'] = True
				nocolor()
			elif o in ("-d", "--destructive"):
				myoptions['destructive'] = True
			elif o in ("-i", "--interactive") and not myoptions['pretend']:
				myoptions['interactive'] = True
			elif o in ("-p", "--pretend"):
				myoptions['pretend'] = True
				myoptions['interactive'] = False
			elif o in ("-q", "--quiet"):
				myoptions['quiet'] = True
			elif o in ("-t", "--time-limit"):
				myoptions['time-limit'] = parseTime(a)
			elif o in ("-e", "--exclude-file"):
				myoptions['exclude-file'] = a
			elif o in ("-n", "--package-names"):
				myoptions['package-names'] = True
			elif o in ("-f", "--fetch-restricted"):
				myoptions['fetch-restricted'] = True
			elif o in ("-s", "--size-limit"):
				myoptions['size-limit'] = parseSize(a)
			else: return_code = False
		# sanity check of --destructive only options:
		for myopt in ('fetch-restricted', 'package-names'):
			if (not myoptions['destructive']) and myoptions[myopt]:
				if not myoptions['quiet']:	
					eerror("--%s only makes sense in --destructive mode." \
							% myopt, myoptions['nocolor'])
				myoptions[myopt] = False
		return return_code

	# here are the different allowed command line options (getopt args)
	getopt_options = {'short':{}, 'long':{}}
	getopt_options['short']['global'] = "Cdipqe::t::nhV"
	getopt_options['long']['global'] = ["nocolor", "destructive", \
			"interactive", "pretend", "quiet", "exclude-file", "time-limit", \
			"package-names", "help", "version"]
	getopt_options['short']['distfiles'] = "fs::"
	getopt_options['long']['distfiles'] = ["fetch-restricted", "size-limit"]
	getopt_options['short']['packages'] = ""
	getopt_options['long']['packages'] = [""]
	# set default options, except 'nocolor', which is set in main()
	myoptions['interactive'] = False
	myoptions['pretend'] = False
	myoptions['quiet'] = False
	myoptions['accept_all'] = False
	myoptions['destructive'] = False
	myoptions['time-limit'] = 0
	myoptions['package-names'] = False
	myoptions['fetch-restricted'] = False
	myoptions['size-limit'] = 0
	# if called by a well-named symlink, set the acction accordingly:
	myaction = None
	if os.path.basename(sys.argv[0]) in \
			(__productname__+'-pkg', __productname__+'-packages'):
		myaction = 'packages'
	elif os.path.basename(sys.argv[0]) in \
			(__productname__+'-dist', __productname__+'-distfiles'):
		myaction = 'distfiles'
	# prepare for the first getopt
	if myaction:
		short_opts = getopt_options['short']['global'] \
				+ getopt_options['short'][myaction]
		long_opts = getopt_options['long']['global'] \
				+ getopt_options['long'][myaction]
		opts_mode = 'merged-'+myaction
	else:
		short_opts = getopt_options['short']['global']
		long_opts = getopt_options['long']['global']
		opts_mode = 'global'
	# apply getopts to command line, show partial help on failure
	try: opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts)
	except: raise ParseArgsException(opts_mode+'-options')
	# set myoptions accordingly
	optionSwitch(myoptions,opts,action=myaction)
	# if action was already set, there should be no more args
	if myaction and len(args): raise ParseArgsException(opts_mode+'-options')
	# if action was set, there is nothing left to do
	if myaction: return myaction
	# So, we are in "eclean --foo action --bar" mode. Parse remaining args...
	# Only two actions are allowed: 'packages' and 'distfiles'.
	if not len(args) or not args[0] in ('packages','distfiles'):
		raise ParseArgsException('actions')
	myaction = args.pop(0)
	# parse the action specific options
	try: opts, args = getopt.getopt(args, \
			getopt_options['short'][myaction], \
			getopt_options['long'][myaction])
	except: raise ParseArgsException(myaction+'-options')
	# set myoptions again, for action-specific options
	optionSwitch(myoptions,opts,action=myaction)
	# any remaning args? Then die!
	if len(args): raise ParseArgsException(myaction+'-options')
	# returns the action. Options dictionary is modified by side-effect.
	return myaction

###############################################################################
# isValidCP: check wether a string is a valid cat/pkg-name
# This is for 2.0.51 vs. CVS HEAD compatibility, i've not found any function
# for that which would exists in both. Weird...
def isValidCP(cp):
	if not '/' in cp: return False
	try: portage.cpv_getkey(cp+"-0")
	except: return False
	else: return True


###############################################################################
# ParseExcludeFileException: for parseExcludeFile() -> main() communication
class ParseExcludeFileException(Exception):
	def __init__(self, value):
		self.value = value
	def __str__(self):
		return repr(self.value)


###############################################################################
# parseExcludeFile: parses an exclusion file, returns an exclusion dictionnary
# Raises ParseExcludeFileException in case of fatal error.
def parseExcludeFile(filepath):
	excl_dict = { \
			'categories':{}, \
			'packages':{}, \
			'anti-packages':{}, \
			'garbage':{} }
	try: file = open(filepath,"r")
	except IOError:
		raise ParseExcludeFileException("Could not open exclusion file.")
	filecontents = file.readlines()
	file.close()
	cat_re = re.compile('^(?P<cat>[a-zA-Z0-9]+-[a-zA-Z0-9]+)(/\*)?$')
	cp_re = re.compile('^(?P<cp>[-a-zA-Z0-9_]+/[-a-zA-Z0-9_]+)$')	
	for line in filecontents:
		line = line.strip()
		if not len(line): continue
		if line[0] == '#': continue
		try: mycat = cat_re.match(line).group('cat')
		except: pass
		else: 
			if not mycat in portage.settings.categories:
				raise ParseExcludeFileException("Invalid category: "+mycat)
			excl_dict['categories'][mycat] = None
			continue
		dict_key = 'packages'
		if line[0] == '!':
			dict_key = 'anti-packages'
			line = line[1:]
		try:
			mycp = cp_re.match(line).group('cp')
			if isValidCP(mycp):
				excl_dict[dict_key][mycp] = None
				continue
			else: raise ParseExcludeFileException("Invalid cat/pkg: "+mycp)
		except: pass
		#raise ParseExcludeFileException("Invalid line: "+line)
		excl_dict['garbage'][line] = None
	return excl_dict


###############################################################################
# exclDictExpand: returns a dictionary of all CP from porttree which match
# the exclusion dictionary
def exclDictExpand(excl_dict):
	mydict = {}
	if 'categories' in excl_dict:
		# XXX: i smell an access to something which is really out of API...
		for mytree in portage.portdb.porttrees:
			for mycat in excl_dict['categories']:
				for mypkg in listdir(os.path.join(mytree,mycat),ignorecvs=1):
					mydict[mycat+'/'+mypkg] = None
	if 'packages' in excl_dict:
		for mycp in excl_dict['packages']:
			mydict[mycp] = None
	if 'anti-packages' in excl_dict:
		for mycp in excl_dict['anti-packages']:
			if mycp in mydict:
				del mydict[mycp]
	return mydict


###############################################################################
# exclDictMatch: checks whether a CP matches the exclusion rules
def exclDictMatch(excl_dict,pkg):
	if 'anti-packages' in excl_dict \
	   and pkg in excl_dict['anti-packages']:
		return False
	if 'packages' in excl_dict \
	   and pkg in excl_dict['packages']:
		return True
	mycat = pkg.split('/')[0]
	if 'categories' in excl_dict \
	   and mycat in excl_dict['categories']:
		return True
	return False


###############################################################################
# findDistfiles: find all obsolete distfiles.
# XXX: what about cvs ebuilds? i should install some to see where it goes...
def findDistfiles( \
		exclude_dict={}, \
		destructive=False,\
		fetch_restricted=False, \
		package_names=False, \
		time_limit=0, \
		size_limit=0):
	# this regexp extracts files names from SRC_URI. It is not very precise,
	# but we don't care (may return empty strings, etc.), since it is fast.
	file_regexp = re.compile('([a-zA-Z0-9_,\.\-\+]*)[\s\)]')
	clean_dict = {}
	keep = []
	pkg_dict = {}

	# create a big CPV->SRC_URI dict of packages whose distfiles should be kept
	if (not destructive) or fetch_restricted:
		# list all CPV from portree (yeah, that takes time...)
		for package in portage.portdb.cp_all():
			for my_cpv in portage.portdb.cp_list(package):
				# get SRC_URI and RESTRICT from aux_get
				try: (src_uri,restrict) = \
					portage.portdb.aux_get(my_cpv,["SRC_URI","RESTRICT"])
				except KeyError: continue
				# keep either all or fetch-restricted only
				if (not destructive) or ('fetch' in restrict):
					pkg_dict[my_cpv] = src_uri
	if destructive:
		if not package_names:
			# list all CPV from vartree
			pkg_list = portage.db[portage.root]["vartree"].dbapi.cpv_all()
		else:
			# list all CPV from portree for CP in vartree
			pkg_list = []
			for package in portage.db[portage.root]["vartree"].dbapi.cp_all():
				pkg_list += portage.portdb.cp_list(package)
		for my_cp in exclDictExpand(exclude_dict):
			# add packages from the exclude file
			pkg_list += portage.portdb.cp_list(my_cp)
		for my_cpv in pkg_list:
			# skip non-existing CPV (avoids ugly aux_get messages)
			if not portage.portdb.cpv_exists(my_cpv): continue
			# get SRC_URI from aux_get
			try: pkg_dict[my_cpv] = \
					portage.portdb.aux_get(my_cpv,["SRC_URI"])[0]
			except KeyError: continue
		del pkg_list

	# create a dictionary of files which should be deleted 
	for file in os.listdir(distdir):
		filepath = os.path.join(distdir, file)
		try: file_stat = os.stat(filepath)
		except: continue
		if not stat.S_ISREG(file_stat[stat.ST_MODE]): continue
		if size_limit and (file_stat[stat.ST_SIZE] >= size_limit):
			continue
		if time_limit and (file_stat[stat.ST_MTIME] >= time_limit):
			continue
		if 'garbage' in exclude_dict and file in exclude_dict['garbage']:
			continue
		# this is a candidate for cleaning
		clean_dict[file]=[filepath]
	# remove files owned by some protected packages
	for my_cpv in pkg_dict:
		for file in file_regexp.findall(pkg_dict[my_cpv]+"\n"):
			if file in clean_dict:
				del clean_dict[file]
		# no need to waste IO time if there is nothing left to clean
		if not len(clean_dict): return clean_dict
	return clean_dict


###############################################################################
# findPackages: find all obsolete binary packages.
# XXX: packages are found only by symlinks. Maybe i should also return .tbz2
#      files from All/ that have no corresponding symlinks.
def findPackages( \
		exclude_dict={}, \
		destructive=False, \
		time_limit=0, \
		package_names=False):
	clean_dict = {}
	# create a full package dictionnary
	for root, dirs, files in os.walk(pkgdir):
		if root[-3:] == 'All': continue
		for file in files:
			if not file[-5:] == ".tbz2":
				# ignore non-tbz2 files
				continue
			path = os.path.join(root, file)
			category = os.path.split(root)[-1]
			cpv = category+"/"+file[:-5]
			mystat = os.lstat(path)
			if time_limit and (mystat[stat.ST_MTIME] >= time_limit):
				# time-limit exclusion
				continue
			# dict is cpv->[files] (2 files in general, because of symlink)
			clean_dict[cpv] = [path]
			#if os.path.islink(path):
			if stat.S_ISLNK(mystat[stat.ST_MODE]):
				clean_dict[cpv].append(os.path.realpath(path))
	# keep only obsolete ones
	if destructive:
		mydbapi = portage.db[portage.root]["vartree"].dbapi
		if package_names: cp_all = dict.fromkeys(mydbapi.cp_all())
		else: cp_all = {}
	else:
		mydbapi = portage.db[portage.root]["porttree"].dbapi
		cp_all = {}
	for mycpv in clean_dict.keys():
		if exclDictMatch(exclude_dict,portage.cpv_getkey(mycpv)):
			# exclusion because of the exclude file
			del clean_dict[mycpv]
			continue
		if mydbapi.cpv_exists(mycpv):
			# exclusion because pkg still exists (in porttree or vartree)
			del clean_dict[mycpv]
			continue
		if portage.cpv_getkey(mycpv) in cp_all:
			# exlusion because of --package-names 
			del clean_dict[mycpv]

	return clean_dict


###############################################################################
# doCleanup: takes a dictionnary {'display name':[list of files]}. Calculate
# size of each entry for display, prompt user if needed, delete files if needed
# and return the total size of files that [have been / would be] deleted.
def doCleanup(clean_dict,action,myoptions):
	# define vocabulary of this action
	if action == 'distfiles': file_type = 'file'
	else: file_type = 'binary package'
	# sorting helps reading
	clean_keys = clean_dict.keys()
	clean_keys.sort()
	clean_size = 0
	# clean all entries one by one
	for mykey in clean_keys:
		key_size = 0
		for file in clean_dict[mykey]:
			# get total size for an entry (may be several files, and
			# symlinks count zero)
			if os.path.islink(file): continue
			try: key_size += os.path.getsize(file)
			except: eerror("Could not read size of "+file, \
			               myoptions['nocolor'])
		if not myoptions['quiet']:
			# pretty print mode
			print prettySize(key_size,True),teal(mykey)
		elif myoptions['pretend'] or myoptions['interactive']:
			# file list mode
			for file in clean_dict[mykey]: print file
		#else: actually delete stuff, but don't print anything
		if myoptions['pretend']: clean_size += key_size
		elif not myoptions['interactive'] \
		     or myoptions['accept_all'] \
		     or yesNoAllPrompt(myoptions, \
		                       "Do you want to delete this " \
				       + file_type+"?"):
			# non-interactive mode or positive answer. 
			# For each file,...
			for file in clean_dict[mykey]:
				# ...get its size...
				filesize = 0
				if not os.path.exists(file): continue
				if not os.path.islink(file):
					try: filesize = os.path.getsize(file)
					except: eerror("Could not read size of "\
					               +file, myoptions['nocolor'])
				# ...and try to delete it.
				try: os.unlink(file)
				except: eerror("Could not delete "+file, \
				               myoptions['nocolor'])
				# only count size if successfully deleted
				else: clean_size += filesize
	# return total size of deleted or to delete files
	return clean_size


###############################################################################
# doAction: execute one action, ie display a few message, call the right find*
# function, and then call doCleanup with its result.
def doAction(action,myoptions,exclude_dict={}):
	# define vocabulary for the output
	if action == 'packages': files_type = "binary packages"
	else: files_type = "distfiles"
	# find files to delete, depending on the action
	if not myoptions['quiet']:
		einfo("Building file list for "+action+" cleaning...", \
		      myoptions['nocolor'])
	if action == 'packages':
		clean_dict = findPackages( \
			exclude_dict=exclude_dict, \
			destructive=myoptions['destructive'], \
			package_names=myoptions['package-names'], \
			time_limit=myoptions['time-limit'])
	else: 
		clean_dict = findDistfiles( \
			exclude_dict=exclude_dict, \
			destructive=myoptions['destructive'], \
			fetch_restricted=myoptions['fetch-restricted'], \
			package_names=myoptions['package-names'], \
			time_limit=myoptions['time-limit'], \
			size_limit=myoptions['size-limit'])
	# actually clean files if something was found
	if len(clean_dict.keys()):
		# verbose pretend message
		if myoptions['pretend'] and not myoptions['quiet']:
			einfo("Here are "+files_type+" that would be deleted:", \
			      myoptions['nocolor'])
		# verbose non-pretend message
		elif not myoptions['quiet']:
			einfo("Cleaning "+files_type+"...",myoptions['nocolor'])
		# do the cleanup, and get size of deleted files
		clean_size = doCleanup(clean_dict,action,myoptions)
		# vocabulary for final message
		if myoptions['pretend']: verb = "would be"
		else: verb = "has been"
		# display freed space
		if not myoptions['quiet']:
			einfo("Total space that "+verb+" freed in " \
			      + action + " directory: " \
			      + red(prettySize(clean_size)), \
			      myoptions['nocolor'])
	# nothing was found, return
	elif not myoptions['quiet']:
		einfo("Your "+action+" directory was already clean.", \
		      myoptions['nocolor'])


###############################################################################
# main: parse command line and execute all actions
def main():
	# set default options
	myoptions = {}
	myoptions['nocolor'] = port_settings["NOCOLOR"] in ('yes','true') \
	                       and sys.stdout.isatty()
	if myoptions['nocolor']: nocolor()
	# parse command line options and actions
	try: myaction = parseArgs(myoptions)
	# filter exception to know what message to display
	except ParseArgsException, e:
		if e.value == 'help':
			printUsage(help='all')
			sys.exit(0)
		elif e.value[:5] == 'help-':
			printUsage(help=e.value[5:])
			sys.exit(0)
		elif e.value == 'version':
			printVersion()
			sys.exit(0)
		else: 
			printUsage(e.value)
			sys.exit(2)
	# parse the exclusion file
	if not 'exclude-file' in myoptions:
		my_exclude_file = "/etc/%s/%s.exclude" % (__productname__ , myaction)
		if os.path.isfile(my_exclude_file):
			myoptions['exclude-file'] = my_exclude_file
	if 'exclude-file' in myoptions:
		try: exclude_dict = parseExcludeFile(myoptions['exclude-file'])
		except ParseExcludeFileException, e:
			eerror(e, myoptions['nocolor'])
			eerror("Invalid exclusion file: %s" % myoptions['exclude-file'], \
					myoptions['nocolor'])
			eerror("See format of this file in `man %s`" % __productname__, \
					myoptions['nocolor'])	
			sys.exit(1)
	else: exclude_dict={}
	# security check for non-pretend mode
	if not myoptions['pretend'] and portage.secpass != 2:
		eerror("Permission denied: you must be root.", \
		       myoptions['nocolor'])
		sys.exit(1)
	# execute action
	doAction(myaction, myoptions, exclude_dict=exclude_dict)


###############################################################################
# actually call main() if launched as a script
if __name__ == "__main__":
	try: main()
	except KeyboardInterrupt:
		print "Aborted."
		sys.exit(130)
	sys.exit(0)