Updated version of binpkgmain.py

binpkgmain.py (text/plain), 23.55 KB, created by Baz (RETIRED) on 2004-05-25 19:58:26 UTC

# NOTE: this script is under CVS control.

# -*- python -*-

# binpkgmain - collection of functions for the maintenence of binary
# packages for use with emerge -G
#
# Baz - 26/05/04

import xpak,getbinpkg,sys,portage,cPickle,time,re,string
from output import *
from stat import *
from types import *

# Globals 

portage.noiselimit = -1 # Dont want class config debug messages
settings = portage.config()
portdb = portage.portdbapi(settings["PORTDIR"])
portage.noiselimit = 0 # reset it 

# location of the binary repository in the filesystem
BASE_DIR = "/gentoo/dist/1.4/packages"
BIN_PKG_DIR = BASE_DIR+"/All"
OLD_PKG_DIR = BASE_DIR+"/old-packages"
# This is a text file containing a list of packages for the script to ignore
# due to portage/ebuild peculiarities.
MASK_FILE = BIN_PKG_DIR+"/masked_packages.txt"

DEBUG = 0

def check_versions(move_red=0, show_upgrades=1, target_dir=BIN_PKG_DIR, red_pkg_dir=OLD_PKG_DIR):
    """Looks in the directory specified and checks to see if any of the packages 
    contained therein are outdated in terms of the local current portage tree.  Will
    also advise if there are newer versions of packages available"""

    if not os.path.exists(target_dir):
        print(red("Error "+target_dir+": no such directory"))

    idx_file = target_dir+"/metadata.idx"
    if not os.path.exists(idx_file):
        print(blue("No metadata file present, will generate one now..."))
        gen_metadata(target_dir)

    # Check to see if the metadata file is up to date
    dir_age = os.stat(target_dir)[ST_MTIME]
    idx_age = os.stat(idx_file)[ST_MTIME]

    if ( dir_age > idx_age ):
        print(blue("Metadata file out of date, generating new one..."))
        gen_metadata(target_dir)

    metadata_file = open(idx_file, "r")
    print(blue("Loading metadata pickle, please wait..."))
    metadata = cPickle.load(metadata_file)
    metadata_file.close
     

    if DEBUG:
        print(green("Obtaining listing of "+target_dir))

    rec_ver = {}
    mask_ver = {}
    old_ver = {}

    pkgs = os.listdir(target_dir)

    # Loading list of packages to exclude from package mask file.
    masked_pkgs = []
    if os.path.exists(MASK_FILE):
        print(blue("Package mask file found, loading."))
        mask = open(MASK_FILE, "r")
        for line in mask.readlines():
            comment = re.search("^#.*", line)
            space = re.search("^\s+", line)
            if ( comment or space ):
                continue
            else:
                masked_pkgs.append(line)
    else:
        print(green("No package mask file found."))
        

    print(blue("\nSorting through binary packages.....\n"))


    for x in pkgs:

        # first we need to split up the package name to get just the basename with no 
        # version info on the end.

        if x[-5:]!=".tbz2":
            continue

        basename = portage.catpkgsplit(x[:-5])

        if not basename[1]:
            print(green("Can find packagename for "+x+", skipping"))
            continue
          
        # Get the category from the metadata.idx file
        basename[0] = string.rstrip(metadata[x]["CATEGORY"])

        if not basename[0]:
            print(green("Can find category for package "+x+", skipping"))
            continue

        catpkgname = basename[0]+"/"+basename[1]
        catpkgvername = basename[0]+"/"+x[:-5]

        flag = 0
        for pkg in masked_pkgs:
            reg = "%s-%s"%(catpkgname,basename[2])
            skip = re.search( re.escape(reg), pkg)
            if skip:
                print(red("%s identified as a masked package, skipping"%(reg)))
                flag = 1
                break
        
        if flag:
            continue 

        try:
            versions = portdb.xmatch("match-visible", catpkgname)
        except: 
            print(red("Undetermined error finding all versions of %s, skipping"(catpkgname)))
            continue

        # find SLOT from metadata (take off trailing newline)
        slot = string.rstrip(metadata[x]["SLOT"])

        # compile regexp
        pkg_regexp = re.compile( "^%s$"%( re.escape(catpkgvername) ) )

        flag = 0
        for y in versions:
            if pkg_regexp.search(y):
                flag = 1
                if not rec_ver.has_key(catpkgname):
                    rec_ver[catpkgname] = {}
                    rec_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ]
                else:
                    if not rec_ver[catpkgname].has_key(slot):
                        rec_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])]
                    else:
                        rec_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3]))

        if flag == 1:
            continue

        flag = 0
        try:
            versions = portdb.xmatch("match-all", catpkgname)
        except:
            print(red("Undetermined error finding all versions of %s, skipping"(catpkgname)))
            continue

        for y in versions:
            if pkg_regexp.search(y):
                flag = 1
                if not mask_ver.has_key(catpkgname):
                    mask_ver[catpkgname] = {}
                    mask_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ]
                else:
                    if not mask_ver[catpkgname].has_key(slot):
                        mask_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])]
                    else:
                        mask_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3]))
                # Let users know they've got masked packages
                print(green(catpkgvername+" is MASKED in portage"))

        if flag == 1:
            continue

        
        # Find all visible packages and recommend upgrade to one with the same SLOT.

        rec_upgrade = best_in_slot(slot, catpkgname)

        # If there is no slot match, then we'll just pick out the best one.
        if ( rec_upgrade == "" ):
            rec_upgrade = portdb.xmatch("bestmatch-visible", catpkgname)
        
        print(red(catpkgvername+" is no longer in portage you should install "+rec_upgrade))
        if not old_ver.has_key(catpkgname):
            old_ver[catpkgname] = {}
            old_ver[catpkgname][slot] = [ "%s-%s"%(basename[2],basename[3]) ]
        else:
            if not old_ver[catpkgname]:
                old_ver[catpkgname][slot] = ["%s-%s"%(basename[2],basename[3])]
            else:
                old_ver[catpkgname][slot].append("%s-%s"%(basename[2],basename[3]))

    
    # Now we have the dicts of the packages we need to sort through them
    # First look through current packages and see if we've got 2 or more versions of 
    # a binary that is still current in portage.  If so, we'll flag the older
    # version(s) for removal (emerging with -GU will grab the most recent ones anyway
    # so the older versions are redundant).

    red_ver = {}

    for q in rec_ver.keys():

        red_ver[q] = {}
        if DEBUG:
            print(green("Determining old versions of %s"%(q)))

        if ( ( len(rec_ver[q].keys()) > 1 ) & DEBUG ):
            print(green("Multiple SLOTs of %s detected, treating SLOTs individually"%(q)))
        
        for x in rec_ver[q].keys():

            # Now use the portage.best function to find the most recent of the versions 
            # in the list.

            match_list = []
            for z in rec_ver[q][x]:
                match_list.append("%s-%s"%(q,z))
            best_pkg = portage.best(match_list)
            if DEBUG:
                print(green("\tBest local binary version of %s in slot %s is %s"%(q,x,best_pkg)))

            if not ( red_ver[q].has_key(x) ):
                red_ver[q][x] = {}

            red_ver[q][x]["best"] = best_pkg
            red_ver[q][x]["redun"] = []

            # Now add the rest of the versions to the red_ver list.
            for z in rec_ver[q][x]:
                if not re.match( re.compile(re.escape(best_pkg)), "%s-%s"%(q,z) ):
                    red_ver[q][x]["redun"].append( "%s-%s"%(q,z) )
            
            # If the best binary version present is not the most recent in portage, notify 
            # the user.

            if show_upgrades == 0:
                continue

            latest_portage_pkg = best_in_slot(x,q)
            
            # If there is no slot match, then we'll just pick out the best one.
            if ( latest_portage_pkg == "" ):
                latest_portage_pkg = portdb.xmatch("bestmatch-visible", q)

            # If we have a package in the binary directory that is masked the above call
            # will return nothing.  If this occurs we assume the binary is a special case
            # and skip over it.
            if latest_portage_pkg == "":
                print(red("Package %s may have wrong SLOT info in XPAK segement or maybe MASKED, please check"%(q)))
                continue
            p2 = portage.catpkgsplit(latest_portage_pkg)[1:]
            #best_pkg = "%s-%s"%(q,best_ver)
            p1 = portage.catpkgsplit(best_pkg)[1:]
            ret = portage.pkgcmp(p1,p2)
            if ( ret < 0 ):
                print(blue("Upgrade for %s to %s available"%(best_pkg,latest_portage_pkg)))

    # Now we get rid of the redundant packages if required

    confirmed_red_ver = []

    if move_red:

        print(blue("\nVerifying and moving redundant packages..."))

        # Need to add the old packages to the list of redundant packages.
        for y in old_ver.keys():
            for x in old_ver[y].keys():
                for z in range( len(old_ver[y][x]) ):
                    old_pkg = "%s-%s"%(y, old_ver[y][x][z] )

                    # we need to check that old packages do actually have a newer binary
                    # available before we move them or things will break.
                    
                    if ( red_ver.has_key(y) and red_ver[y].has_key(x)):
                        red_ver[y][x]["redun"].append(old_pkg)
                    else:
                        print(red("Can't remove old package %s, no newer binary version present in slot %s"%(old_pkg, x)))
        
        # Now we need to check through the dependancies of the packages still in the 
        # binary repository to make sure that none of the packages we intend to remove 
        # are still needed.
        
        for t in red_ver.keys():
            for q in red_ver[t].keys():
                for p in red_ver[t][q]["redun"]:

                    keep = 0
                    pot_deps = {}
                    pkg_split = portage.catpkgsplit(p)
                    if ( pkg_split[3] == "r0" ):
                        filename = re.compile("%s-%s.tbz2"%(pkg_split[1], pkg_split[2]))
                        tbz2name = "%s-%s.tbz2"%(pkg_split[1], pkg_split[2])
                    else:
                        filename = re.compile("%s-%s-%s.tbz2"%(pkg_split[1], pkg_split[2], pkg_split[3]))
                        tbz2name = "%s-%s-%s.tbz2"%(pkg_split[1], pkg_split[2], pkg_split[3])

                    pkg_mo = re.compile("%s/%s"%(pkg_split[0], pkg_split[1]))
                   
                    sys.stdout.write(blue("Checking if %s is needed by another package..."%(p)))
                    for z in metadata.keys():
                        # We don't need to check the target packages own dependencies
                        if ( filename.match(z) ):
                            if DEBUG:
                                print(red("\nFound target package, %s, in metadata, skipping"%(z)))
                            continue
                        
                        for y in ["DEPEND", "RDEPEND", "PDEPEND"]:
                            deps = string.split(metadata[z][y])
                            # Find potential dependancies.  We'll have to look through these 
                            # a bit more carefully once we've narrowed them down.
                            for x in deps:
                                if ( pkg_mo.search(x) ):
                                    if DEBUG:
                                        print(green("Found dependancy potential match, %s, in %s"%(x,z)))
                                    if ( pot_deps.has_key(z) ):
                                        pot_deps[z].append(x)
                                    else:
                                        pot_deps[z] = [x]	

                    # See if the redundant package is an dependency of another package.
                    [ keep, blockers ] = check_red_deps( pot_deps, pkg_split, red_ver[t][q]["best"])

                    # At this point we have tested all the deps for the package, if keep is still
                    # zero the package can be added to the list of confirmed redundant versions

                    if ( keep == 0 ):
                        sys.stdout.write(blue("No\n"))
                        confirmed_red_ver.append(tbz2name)
                        if DEBUG:
                            print(red("No deps for %s found in %s"%(p,x)))
                    else:
                        print(red("\nCan't remove %s, as has dependants:"%(p)))
                        for t in blockers:
                            print(red("\t%s"%(t)))

        # Now all the packages remaining are verified redundant, lets kill em....
        
        for y in confirmed_red_ver:
            # since catpkgsplit qualifies packages with -r0 suffix if there is only one
            # version, we need to strip this off so our package names match the names of 
            # binaries.
            if y[-3:] == "-r0":
                y = y[:-3]
            target_file = "%s/%s"%(target_dir,y)
            dest_file = "%s/%s"%(red_pkg_dir,y)
            if DEBUG:
                print("Moving %s to %s"%(target_file,dest_file))
            print(blue("Redundant package %s moved"%(y)))
            os.system("mv %s %s"%(target_file,dest_file))

    if ( len(confirmed_red_ver) > 0 ):
        print(blue("\nContents of binary directory changed, regenerating index"))
        gen_metadata(target_dir)
                    

    return [ rec_ver, mask_ver, old_ver ]
        
         
def gen_metadata(target_dir=BIN_PKG_DIR, idx_file=None):
    """Looks in the directory specified and generates a metadata.idx file
    from all the XPAK segments of binaries in the target directory"""

    if idx_file == None:
        idx_file = target_dir+"/metadata.idx"

    if os.path.exists(target_dir):
        perms = os.stat(target_dir)[ST_MODE]
    else:
        print(red("Error "+target_dir+": no such directory"))
        sys.exit(1)
    if not perms & S_IRWXU:
        print(red("Directory "+target_dir+" is not writeable, aborting"))

    # Create the base metadata structure
    if DEBUG:
        print(green("Creating base metadata structure"))
        
    metadata = {}
    metadata["indexname"] = ""
    metadata["timestamp"] = int(time.time())
    metadata["unmodified"] = 0
    metadata["data"] = {}

    if DEBUG:
        print(green("Obtaining listing of "+target_dir))
        
    pkgs = os.listdir(target_dir)

    for x in pkgs:

        # Find the size of of the XPAK segement and read it in from the end of the file
        tbz_file = xpak.tbz2(target_dir+"/"+x)
        a = open(target_dir+"/"+x)
        a.seek(-tbz_file.xpaksize,2)
        myxpak = a.read(tbz_file.xpaksize-8)
        a.close

        myid = xpak.xsplit_mem(myxpak)

        if not myid:
            if DEBUG:
                print(green("No XPAK segment found in "+x+", skipping"))
            continue

        # Add to metadata if its a valid segement
        if myid[0]:
            metadata["data"][x] = getbinpkg.make_metadata_dict(myid)
        else:
            print(red("Error, corrupt XPAK segement in "+x))

    # Now dump the metadata to file.  I'll stick in some old file rotation here eventually
    # but for now we'll just see if it works.

    try:
        metadatafile = open(idx_file, "w")
        cPickle.dump(metadata["data"],metadatafile)
        metadatafile.close()
    except Exception, e:
        print("!!! Failed to write binary metadata to disk!")
        print("!!! "+str(e))
    
    return 
        

# Given a slot and package name, find the best version currently in portage.
def best_in_slot( slot, pkg ):
    slot_match = []
    match_list = portdb.xmatch("list-visible", pkg)
    for y in match_list:
        y_slot = portdb.aux_get( y, ["SLOT"] )
        #print(green("\tPackage %s, version %s, slot %s"%(pkg, y, y_slot[0])))
        if ( re.match( slot, y_slot[0] ) ):
            slot_match.append(y)

    best_match = portage.best(slot_match)
    return best_match


# Check dependencies to see if a package is safe to remove.
def check_red_deps( pot_deps, pkg_split, best_pkg ):

    # Need to set this out here too.... some pkgs have no deps, dont make it into the
    # following loop and then inherit keep from the previous loop.
    keep = 0
    blockers = []

    for x in pot_deps.keys():
        if DEBUG:
            print(blue("Checking deps in %s"%(x)))
        for y in pot_deps[x]:
            
            eq_wc = 0

            # If theres a keyword ending in a ? then its a dynamic depend keyword
            # and we can skip it.  If the package that constitues the dynamic 
            # dependancy matched the package regexp it'll be considered anyway.
            if ( re.search( "\w+\?", y ) ):
                continue
                    
            if ( ( re.match( re.escape("!"), y) ) or ( re.match( re.escape("!="), y) ) ):
                # This is a blocker, not a dependancy.  We'll skip it.
                continue

            if DEBUG:
                print(green("\t checking for match with %s"%(y)))

            # Need to check for extended atom prefixes and treat them accordingly
            if ( re.match( "=.*\*$", y) ):
                # Treat =cat/pkg-ver* types specially as need to do some string
                # munging before we pass to pkgcmp
                y = y.rstrip("*")
                eq_wc = 1 # set flag so we can treat this case specially later

            elif ( re.search( "\d+\*$", y ) ):
                # If version ends with a star we don't need it (in fact it screws
                # over catpkgsplit).  The function portage.pkgcmp is accurate in 
                # comparing versions with minor numbers removed (provided there is 
                # no leading = prefix)
                y = y.rstrip("*")
            
            if ( re.match( re.escape("~"), y ) ):
                # Tildes match revisions, since portage.pkgcmp handles these we'll
                # just take it out (catpkgsplit automatically adds -r0 if there 
                # is no revision specified in the package string).
                y = y.strip("~")
                
            # Take off the prefixes for name comparision as these kill catpkgsplit
            base_pot_pkg = y.strip(">")
            base_pot_pkg = base_pot_pkg.strip("<")
            base_pot_pkg = base_pot_pkg.strip("=")
            # Break up the potential package name into components 
            pot_pkg_split = portage.catpkgsplit(base_pot_pkg)
            
            # If there is no return from catpkgsplit we can assume(!?) that there is 
            # no version info for the dependancy (either its a generic depend, or a
            # virtual depend).  Since we are only considering the removal of packages
            # for which there is a newer binary in place, its safe to ignore these 
            # particular types of dependancy.
            if ( pot_pkg_split == None ):
                continue
            
            # At this point there is a prefix, therefore there must be version info so
            # we can just check the 2 element from the catpkgsplit to see if we have a 
            # package match
            
            pkg_name_mo = re.compile(pot_pkg_split[0]+"/"+pot_pkg_split[1])
            if not ( pkg_name_mo.match(pkg_split[0]+"/"+pkg_split[1]) ):
                # No package match here, skip.
                if DEBUG:
                    print(green("\t%s doesn't match %s/%s, skipping"%(pkg_split[0], pkg_split[1],y)))
                continue
            
            # Start checking for the various prefixes and test the package

            # The first 2 checks are redundant since the existence of a more up to date
            # package is implicit in this binary being in the red_ver list anyway.
            # 
            #if ( re.match( re.escape(">"), y) ):
            #    out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:])
            #    if ( out > 1 ):
            #        # the dep string (pot_pkg) is older (eg lower version) than
            #        # the pkg we are considering removing so its a valid dependency
            #        # and not safe to remove
            #        keep = 1

            #if ( ( re.match( re.escape(">="), y) ) and ( keep == 0 ) ):
            #    out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:])
            #    if ( out >= 1 ):
            #        keep = 1

            if ( ( re.match( re.escape("<"), y) ) and ( keep == 0 ) ):
                out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:])
                if ( out < 1 ):
                    keep = 1
                    blockers.append(x)

            if ( ( re.match( re.escape("<="), y) ) and ( keep == 0 ) ):
                out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:])
                if ( out <= 1 ):
                    keep = 1
                    blockers.append(x)

            if ( ( re.match( re.escape("="), y ) ) and ( keep == 0 ) ):
                if ( eq_wc ):
                    # Since the dep string has a wildcard char at the end we need to 
                    # ignore any corresponding digits in the target package we are testing
                    # for removal.
                    eq_wc_pkg_split = pkg_split[:]
                    wc_dig = len(pot_pkg_split[2]) # no. chars in wildcarded string
                    eq_wc_pkg_split[2] = eq_wc_pkg_split[2][:wc_dig] # truncate target ver string
                    eq_wc_pkg_split[3] = "r0" # since we're chopping the version string up 

                    # Now check to make sure we aren't left with a hanging decimal point
                    if ( re.match( ".*\.$", eq_wc_pkg_split[2] ) ):
                        eq_wc_pkg_split[2] = eq_wc_pkg_split[2].rstrip(".")
                        
                    out = portage.pkgcmp(eq_wc_pkg_split[1:], pot_pkg_split[1:])

                    # If this pkg meets a wild card dependancy, we need to see if the best pkg
                    # also meets it.  If so, we can still remove this package and the best pkg meets the
                    # dep and is a better version that the one we are considering

                    if ( out == 0 ):
                        best_pkg_split = portage.catpkgsplit(best_pkg)
                        
                        eq_wc_best_pkg_split = best_pkg_split[:]
                        eq_wc_best_pkg_split[2] = eq_wc_best_pkg_split[2][:wc_dig] # truncate target ver string
                        eq_wc_best_pkg_split[3] = "r0" # since we're chopping the version string up 

                        # Now check to make sure we aren't left with a hanging decimal point
                        if ( re.match( ".*\.$", eq_wc_best_pkg_split[2] ) ):
                            eq_wc_best_pkg_split[2] = eq_wc_best_pkg_split[2].rstrip(".")

                        if ( portage.pkgcmp(eq_wc_best_pkg_split[1:], pot_pkg_split[1:]) == 0 ):
                            # The best pkg meets the dep, we can safely remove the other.
                            out = 1
                        
                else:
                    out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:])
                    
                if ( out == 0 ):
                    keep = 1
                    blockers.append(x)

    # Return result and any blockers

    return [ keep, blockers ]