# NOTE: this script is under CVS control. # -*- python -*- # binpkgmain - collection of functions for the maintenence of binary # packages for use with emerge -G # # Baz - 26/05/04 import xpak,getbinpkg,sys,portage,cPickle,time,re,string from output import * from stat import * from types import * # Globals portage.noiselimit = -1 # Dont want class config debug messages settings = portage.config() portdb = portage.portdbapi(settings["PORTDIR"]) portage.noiselimit = 0 # reset it # location of the binary repository in the filesystem BASE_DIR = "/gentoo/dist/1.4/packages" BIN_PKG_DIR = BASE_DIR+"/All" OLD_PKG_DIR = BASE_DIR+"/old-packages" # This is a text file containing a list of packages for the script to ignore # due to portage/ebuild peculiarities. MASK_FILE = BIN_PKG_DIR+"/masked_packages.txt" DEBUG = 0 def check_versions(move_red=0, show_upgrades=1, target_dir=BIN_PKG_DIR, red_pkg_dir=OLD_PKG_DIR): """Looks in the directory specified and checks to see if any of the packages contained therein are outdated in terms of the local current portage tree. Will also advise if there are newer versions of packages available""" # Load the metadata pickle metadata = get_metadata(target_dir) if DEBUG: print(green("Obtaining listing of "+target_dir)) rec_ver = {} mask_ver = {} old_ver = {} pkgs = os.listdir(target_dir) # Loading list of packages to exclude from package mask file. masked_pkgs = [] if os.path.exists(MASK_FILE): print(blue("Package mask file found, loading.")) mask = open(MASK_FILE, "r") for line in mask.readlines(): comment = re.search("^#.*", line) space = re.search("^\s+", line) if ( comment or space ): continue else: masked_pkgs.append(line) else: print(green("No package mask file found.")) print(blue("\nSorting through binary packages.....\n")) for x in pkgs: # first we need to split up the package name to get just the basename with no # version info on the end. if x[-5:]!=".tbz2": continue basename = portage.catpkgsplit(x[:-5]) if not basename[1]: print(green("Can find packagename for "+x+", skipping")) continue # Get the category from the metadata.idx file basename[0] = string.rstrip(metadata[x]["CATEGORY"]) if not basename[0]: print(green("Can find category for package "+x+", skipping")) continue catpkgname = basename[0]+"/"+basename[1] catpkgvername = basename[0]+"/"+x[:-5] flag = 0 for pkg in masked_pkgs: reg = "%s-%s"%(catpkgname,basename[2]) skip = re.search( re.escape(reg), pkg) if skip: print(red("%s identified as a masked package, skipping"%(reg))) flag = 1 break if flag: continue try: versions = portdb.xmatch("match-visible", catpkgname) except: print(red("Undetermined error finding all versions of %s, skipping"(catpkgname))) continue # find SLOT from metadata (take off trailing newline) slot = string.rstrip(metadata[x]["SLOT"]) # compile regexp pkg_regexp = re.compile( "^%s$"%( re.escape(catpkgvername) ) ) flag = 0 for y in versions: if pkg_regexp.search(y): flag = 1 if not rec_ver.has_key(catpkgname): rec_ver[catpkgname] = {} rec_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ] else: if not rec_ver[catpkgname].has_key(slot): rec_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])] else: rec_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3])) if flag == 1: continue flag = 0 try: versions = portdb.xmatch("match-all", catpkgname) except: print(red("Undetermined error finding all versions of %s, skipping"(catpkgname))) continue for y in versions: if pkg_regexp.search(y): flag = 1 if not mask_ver.has_key(catpkgname): mask_ver[catpkgname] = {} mask_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ] else: if not mask_ver[catpkgname].has_key(slot): mask_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])] else: mask_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3])) # Let users know they've got masked packages print(green(catpkgvername+" is MASKED in portage")) if flag == 1: continue # Find all visible packages and recommend upgrade to one with the same SLOT. rec_upgrade = best_in_slot(slot, catpkgname) # If there is no slot match, then we'll just pick out the best one. if ( rec_upgrade == "" ): rec_upgrade = portdb.xmatch("bestmatch-visible", catpkgname) print(red(catpkgvername+" is no longer in portage you should install "+rec_upgrade)) if not old_ver.has_key(catpkgname): old_ver[catpkgname] = {} old_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ] else: if not old_ver[catpkgname]: old_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])] else: old_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3])) # Now we have the dicts of the packages we need to sort through them # First look through current packages and see if we've got 2 or more versions of # a binary that is still current in portage. If so, we'll flag the older # version(s) for removal (emerging with -GU will grab the most recent ones anyway # so the older versions are redundant). red_ver = {} for q in rec_ver.keys(): red_ver[q] = {} if DEBUG: print(green("Determining old versions of %s"%(q))) if ( ( len(rec_ver[q].keys()) > 1 ) & DEBUG ): print(green("Multiple SLOTs of %s detected, treating SLOTs individually"%(q))) for x in rec_ver[q].keys(): # Now use the portage.best function to find the most recent of the versions # in the list. match_list = [] for z in rec_ver[q][x]: match_list.append("%s-%s"%(q,z)) best_pkg = portage.best(match_list) if DEBUG: print(green("\tBest local binary version of %s in slot %s is %s"%(q,x,best_pkg))) if not ( red_ver[q].has_key(x) ): red_ver[q][x] = {} red_ver[q][x]["best"] = best_pkg red_ver[q][x]["redun"] = [] # Now add the rest of the versions to the red_ver list. for z in rec_ver[q][x]: if not re.match( re.compile(re.escape(best_pkg)), "%s-%s"%(q,z) ): red_ver[q][x]["redun"].append( "%s-%s"%(q,z) ) # If the best binary version present is not the most recent in portage, notify # the user. if show_upgrades == 0: continue latest_portage_pkg = best_in_slot(x,q) # If there is no slot match, then we'll just pick out the best one. if ( latest_portage_pkg == "" ): latest_portage_pkg = portdb.xmatch("bestmatch-visible", q) # If we have a package in the binary directory that is masked the above call # will return nothing. If this occurs we assume the binary is a special case # and skip over it. if latest_portage_pkg == "": print(red("Package %s may have wrong SLOT info in XPAK segement or maybe MASKED, please check"%(q))) continue p2 = portage.catpkgsplit(latest_portage_pkg)[1:] #best_pkg = "%s-%s"%(q,best_ver) p1 = portage.catpkgsplit(best_pkg)[1:] ret = portage.pkgcmp(p1,p2) if ( ret < 0 ): print(blue("Upgrade for %s to %s available"%(best_pkg,latest_portage_pkg))) # Now we get rid of the redundant packages if required confirmed_red_ver = [] if move_red: print(blue("\nVerifying and moving redundant packages...")) # Need to add the old packages to the list of redundant packages. for y in old_ver.keys(): for x in old_ver[y].keys(): for z in range( len(old_ver[y][x]) ): old_pkg = "%s-%s"%(y, old_ver[y][x][z] ) # we need to check that old packages do actually have a newer binary # available before we move them or things will break. if ( red_ver.has_key(y) and red_ver[y].has_key(x)): red_ver[y][x]["redun"].append(old_pkg) else: print(red("Can't remove old package %s, no newer binary version present in slot %s"%(old_pkg, x))) # Now we need to check through the dependancies of the packages still in the # binary repository to make sure that none of the packages we intend to remove # are still needed. for t in red_ver.keys(): for q in red_ver[t].keys(): for p in red_ver[t][q]["redun"]: tbz2name = check_red_deps(t,p, metadata, red_ver[t][q]["best"]) if ( tbz2name != None ): confirmed_red_ver.append(tbz2name) # Now all the packages remaining are verified redundant, lets kill em.... for y in confirmed_red_ver: # since catpkgsplit qualifies packages with -r0 suffix if there is only one # version, we need to strip this off so our package names match the names of # binaries. move_pkg(y, target_dir, red_pkg_dir) if ( len(confirmed_red_ver) > 0 ): print(blue("\nContents of binary directory changed, regenerating index")) gen_metadata(target_dir) return [ rec_ver, mask_ver, old_ver ] def gen_metadata(target_dir=BIN_PKG_DIR, idx_file=None): """Looks in the directory specified and generates a metadata.idx file from all the XPAK segments of binaries in the target directory""" if idx_file == None: idx_file = target_dir+"/metadata.idx" if os.path.exists(target_dir): perms = os.stat(target_dir)[ST_MODE] else: print(red("Error "+target_dir+": no such directory")) sys.exit(1) if not perms & S_IRWXU: print(red("Directory "+target_dir+" is not writeable, aborting")) # Create the base metadata structure if DEBUG: print(green("Creating base metadata structure")) metadata = {} metadata["indexname"] = "" metadata["timestamp"] = int(time.time()) metadata["unmodified"] = 0 metadata["data"] = {} if DEBUG: print(green("Obtaining listing of "+target_dir)) pkgs = os.listdir(target_dir) for x in pkgs: # Find the size of of the XPAK segement and read it in from the end of the file tbz_file = xpak.tbz2(target_dir+"/"+x) a = open(target_dir+"/"+x) a.seek(-tbz_file.xpaksize,2) myxpak = a.read(tbz_file.xpaksize-8) a.close myid = xpak.xsplit_mem(myxpak) if not myid: if DEBUG: print(green("No XPAK segment found in "+x+", skipping")) continue # Add to metadata if its a valid segement if myid[0]: metadata["data"][x] = getbinpkg.make_metadata_dict(myid) else: print(red("Error, corrupt XPAK segement in "+x)) # Now dump the metadata to file. I'll stick in some old file rotation here eventually # but for now we'll just see if it works. try: metadatafile = open(idx_file, "w") cPickle.dump(metadata["data"],metadatafile) metadatafile.close() except Exception, e: print("!!! Failed to write binary metadata to disk!") print("!!! "+str(e)) return # Given a slot and package name, find the best version currently in portage. def best_in_slot( slot, pkg ): slot_match = [] match_list = portdb.xmatch("list-visible", pkg) for y in match_list: y_slot = portdb.aux_get( y, ["SLOT"] ) #print(green("\tPackage %s, version %s, slot %s"%(pkg, y, y_slot[0]))) if ( re.match( slot, y_slot[0] ) ): slot_match.append(y) best_match = portage.best(slot_match) return best_match # Check dependencies to see if a package is safe to remove. def check_red_deps( catpkg, pkg, metadata, best_pkg=None ): keep = 0 pot_deps = {} pkg_split = portage.catpkgsplit(pkg) if ( pkg_split[3] == "r0" ): filename = re.compile("%s-%s.tbz2"%(pkg_split[1], pkg_split[2])) tbz2name = "%s-%s.tbz2"%(pkg_split[1], pkg_split[2]) else: filename = re.compile("%s-%s-%s.tbz2"%(pkg_split[1], pkg_split[2], pkg_split[3])) tbz2name = "%s-%s-%s.tbz2"%(pkg_split[1], pkg_split[2], pkg_split[3]) pkg_mo = re.compile("%s/%s"%(pkg_split[0], pkg_split[1])) sys.stdout.write(blue("Checking if %s is needed by another package..."%(pkg))) for z in metadata.keys(): # We don't need to check the target packages own dependencies if ( filename.match(z) ): if DEBUG: print(red("\nFound target package, %s, in metadata, skipping"%(z))) continue for y in ["DEPEND", "RDEPEND", "PDEPEND"]: deps = string.split(metadata[z][y]) # Find potential dependancies. We'll have to look through these # a bit more carefully once we've narrowed them down. for x in deps: if ( pkg_mo.search(x) ): if DEBUG: print(green("Found dependancy potential match, %s, in %s"%(x,z))) if ( ( pot_deps.has_key(z) ) and ( x not in pot_deps[z] ) ): pot_deps[z].append(x) else: pot_deps[z] = [x] # See if the redundant package is an dependency of another package. # Need to set this out here too.... some pkgs have no deps, dont make it into the # following loop and then inherit keep from the previous loop. keep = 0 blockers = [] for x in pot_deps.keys(): if DEBUG: print(blue("Checking deps in %s"%(x))) for y in pot_deps[x]: eq_wc = 0 # If theres a keyword ending in a ? then its a dynamic depend keyword # and we can skip it. If the package that constitues the dynamic # dependancy matched the package regexp it'll be considered anyway. if ( re.search( "\w+\?", y ) ): continue if ( ( re.match( re.escape("!"), y) ) or ( re.match( re.escape("!="), y) ) ): # This is a blocker, not a dependancy. We'll skip it. continue if DEBUG: print(green("\t checking for match with %s"%(y))) # Need to check for extended atom prefixes and treat them accordingly if ( re.match( "=.*\*$", y) ): # Treat =cat/pkg-ver* types specially as need to do some string # munging before we pass to pkgcmp y = y.rstrip("*") eq_wc = 1 # set flag so we can treat this case specially later elif ( re.search( "\d+\*$", y ) ): # If version ends with a star we don't need it (in fact it screws # over catpkgsplit). The function portage.pkgcmp is accurate in # comparing versions with minor numbers removed (provided there is # no leading = prefix) y = y.rstrip("*") if ( re.match( re.escape("~"), y ) ): # Tildes match revisions, since portage.pkgcmp handles these we'll # just take it out (catpkgsplit automatically adds -r0 if there # is no revision specified in the package string). y = y.strip("~") # Take off the prefixes for name comparision as these kill catpkgsplit base_pot_pkg = y.strip(">") base_pot_pkg = base_pot_pkg.strip("<") base_pot_pkg = base_pot_pkg.strip("=") # Break up the potential package name into components pot_pkg_split = portage.catpkgsplit(base_pot_pkg) # If there is no return from catpkgsplit we can assume(!?) that there is # no version info for the dependancy (either its a generic depend, or a # virtual depend). Since we are only considering the removal of packages # for which there is a newer binary in place, its safe to ignore these # particular types of dependancy. if ( pot_pkg_split == None ): if DEBUG: print(green("Can't get version info for %s, skipping\n"%(y))) continue # At this point there is a prefix, therefore there must be version info so # we can just check the 2 element from the catpkgsplit to see if we have a # package match pkg_name_mo = re.compile(pot_pkg_split[0]+"/"+pot_pkg_split[1]) if not ( pkg_name_mo.match(pkg_split[0]+"/"+pkg_split[1]) ): # No package match here, skip. if DEBUG: print(green("\t%s doesn't match %s/%s, skipping"%(pkg_split[0], pkg_split[1],y))) continue # Start checking for the various prefixes and test the package match_flag = 0 if ( ( re.match( re.escape("<"), y) ) and ( keep == 0 ) ): out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) if ( out < 1 ): keep = 1 match_flag = 1 blockers.append(x) elif ( ( re.match( re.escape("<="), y) ) and ( keep == 0 ) ): out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) if ( out <= 1 ): keep = 1 match_flag = 1 blockers.append(x) elif ( ( eq_wc ) and ( keep == 0 ) ): # Since the dep string has a wildcard char at the end we need to # ignore any corresponding digits in the target package we are testing # for removal. eq_wc_pkg_split = pkg_split[:] wc_dig = len(pot_pkg_split[2]) # no. chars in wildcarded string eq_wc_pkg_split[2] = eq_wc_pkg_split[2][:wc_dig] # truncate target ver string eq_wc_pkg_split[3] = "r0" # since we're chopping the version string up # Now check to make sure we aren't left with a hanging decimal point if ( re.match( ".*\.$", eq_wc_pkg_split[2] ) ): eq_wc_pkg_split[2] = eq_wc_pkg_split[2].rstrip(".") out = portage.pkgcmp(eq_wc_pkg_split[1:], pot_pkg_split[1:]) # If this pkg meets a wild card dependancy, we need to see if the best pkg # also meets it. If so, we can still remove this package and the best pkg meets the # dep and is a better version that the one we are considering if ( out == 0 ): best_pkg_split = portage.catpkgsplit(best_pkg) eq_wc_best_pkg_split = best_pkg_split[:] eq_wc_best_pkg_split[2] = eq_wc_best_pkg_split[2][:wc_dig] # truncate target ver string eq_wc_best_pkg_split[3] = "r0" # since we're chopping the version string up # Now check to make sure we aren't left with a hanging decimal point if ( re.match( ".*\.$", eq_wc_best_pkg_split[2] ) ): eq_wc_best_pkg_split[2] = eq_wc_best_pkg_split[2].rstrip(".") if ( portage.pkgcmp(eq_wc_best_pkg_split[1:], pot_pkg_split[1:]) == 0 ): # The best pkg meets the dep, we can safely remove the other. out = 1 else: out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) if ( out == 0 ): keep = 1 blockers.append(x) # At this point we have tested all the deps for the package, if keep is still # zero the package can be added to the list of confirmed redundant versions if ( keep == 0 ): sys.stdout.write(blue("No\n")) return tbz2name if DEBUG: print(red("No deps for %s found in %s"%(pkg,pkg_split))) else: print(red("\nCan't remove %s, as has dependants:"%(pkg))) for t in blockers: print(red("\t%s"%(t))) return None # Load the metadata pickle def get_metadata(target_dir): if not os.path.exists(target_dir): print(red("Error "+target_dir+": no such directory")) idx_file = target_dir+"/metadata.idx" if not os.path.exists(idx_file): print(blue("No metadata file present, will generate one now...")) gen_metadata(target_dir) # Check to see if the metadata file is up to date dir_age = os.stat(target_dir)[ST_MTIME] idx_age = os.stat(idx_file)[ST_MTIME] if ( dir_age > idx_age ): print(blue("Metadata file out of date, generating new one...")) gen_metadata(target_dir) metadata_file = open(idx_file, "r") print(blue("Loading metadata pickle, please wait...")) metadata = cPickle.load(metadata_file) metadata_file.close return metadata # Move redundant packages to the holding area def move_pkg(tbz2, target_dir, red_pkg_dir): if tbz2[-3:] == "-r0": tbz2 = y[:-3] target_file = "%s/%s"%(target_dir,tbz2) dest_file = "%s/%s"%(red_pkg_dir,tbz2) if DEBUG: print("Moving %s to %s"%(target_file,dest_file)) print(blue("Redundant package %s moved"%(tbz2))) os.system("mv %s %s"%(target_file,dest_file)) # Check to see if an individual pkg can be moved, optional move. def check_indiv_pkg(tbz2, move=0, target_dir=BIN_PKG_DIR, red_pkg_dir=OLD_PKG_DIR): pkg_mo = re.compile( tbz2 ) pkg = tbz2[:-5] metadata = get_metadata(target_dir) # Find the pkg in metadata and find its category if ( metadata.has_key(tbz2)): cat = string.rstrip(metadata[tbz2]["CATEGORY"]) else: print(red("Can't find key in metadata that matches %s"%(tbz2))) return -1 basename = portage.catpkgsplit(pkg) basename[0] = cat catpkg = basename[0]+"/"+basename[1] out = check_red_deps( catpkg, pkg, metadata, None ) if ( (out != None) and (move==1) ): move_pkg(tbz2, target_dir, red_pkg_dir)