Updated versions of previously supplied functions. Bug fix and improvement.

binpkgmain.py (text/plain), 10.43 KB, created by Baz (RETIRED) on 2003-11-26 19:34:34 UTC

# NOTE: this script is under CVS control.

# binpkgmain - collection of functions for the maintenence of binary
# packages for use with emerge -G
#
# Baz - 12/11/03

import xpak,getbinpkg,sys,portage,cPickle,time,re
from output import *
from stat import *

# Globals 

# location of the binary repository in the filesystem
BIN_PKG_DIR = "/diskless/gentoo-1.4-pxe-install/packages/All"
OLD_PKG_DIR = "/diskless/gentoo-1.4-pxe-install/packages/old-packages"


DEBUG = 0

def check_versions(move_red=0, show_upgrades=1, target_dir=BIN_PKG_DIR, red_pkg_dir=OLD_PKG_DIR):
    """Looks in the directory specified and checks to see if any of the packages 
    contained therein are outdated in terms of the local current portage tree.  Will
    also advise if there are newer versions of packages available"""

    if not os.path.exists(target_dir):
        print(red("Error "+target_dir+": no such directory"))

    idx_file = BIN_PKG_DIR+"/metadata.idx"
    if not os.path.exists(idx_file):
        print(green("No metadata file present, will generate one now..."))
        gen_metadata(target_dir)

    # Check to see if the metadata file is up to date
    dir_age = os.stat(target_dir)[ST_MTIME]
    idx_age = os.stat(idx_file)[ST_MTIME]

    if ( dir_age > idx_age ):
        print(green("Metadata file out of date, generating new one..."))
        gen_metadata(target_dir)

    metadata_file = open(idx_file, "r")
    print(blue("Loading metadata pickle, please wait..."))
    metadata = cPickle.load(metadata_file)
    metadata_file.close
     

    if DEBUG:
        print(green("Obtaining listing of "+target_dir))

    rec_ver = {}
    mask_ver = {}
    old_ver = {}

    pkgs = os.listdir(target_dir)

    for x in pkgs:

        # first we need to split up the package name to get just the basename with no 
        # version info on the end.

        if x[-5:]!=".tbz2":
            if DEBUG:
                print(green(x+" doesn't look like a binary package, skipping"))
            continue

        basename = portage.catpkgsplit(x[:-5])

        if not basename[1]:
            print(green("Can find basename for package "+x+", skipping"))
            continue

        versions = portage.portdb.xmatch("match-visible", basename[1])

        # find SLOT from metadata
        slot = metadata[x]["SLOT"]

        # compile regexp
        pkg_regexp = re.compile( "%s$"%( re.escape(x[:-5]) ) )

        flag = 0
        for y in versions:
            if pkg_regexp.search(y):
                flag = 1
                if not rec_ver.has_key(basename[1]):
                    rec_ver[basename[1]] = {}
                    rec_ver[basename[1]][slot] = [ "%s-%s"%(basename[2],basename[3]) ]
                else:
                    if not rec_ver[basename[1]].has_key(slot):
                        rec_ver[basename[1]][slot] = ["%s-%s"%(basename[2],basename[3])]
                    else:
                        rec_ver[basename[1]][slot].append("%s-%s"%(basename[2],basename[3]))
                if DEBUG:
                    print(green(x[:-5]+" is in portage"))

        if flag:
            continue

        flag = 0
        versions = portage.portdb.xmatch("match-all", basename[1])
        for y in versions:
            if pkg_regexp.search(y):
                flag = 1
                if not mask_ver.has_key(basename[1]):
                    mask_ver[basename[1]] = {}
                    mask_ver[basename[1]][slot] = [ "%s-%s"%(basename[2],basename[3]) ]
                else:
                    if not mask_ver[basename[1]].has_key(slot):
                        mask_ver[basename[1]][slot] = ["%s-%s"%(basename[2],basename[3])]
                    else:
                        mask_ver[basename[1]][slot].append("%s-%s"%(basename[2],basename[3]))
                # Let users know they've got masked packages
                print(green(x[:-5]+" is MASKED in portage"))

        if flag == 1:
            continue

        best_match = portage.portdb.xmatch("bestmatch-visible", basename[1])
        print(red(x[:-5]+" is no longer in portage you should upgrade to "+best_match))
        if not old_ver.has_key(basename[1]):
            old_ver[basename[1]] = {}
            old_ver[basename[1]] = [ "%s-%s"%(basename[2],basename[3]) ]
        else:
            if not old_ver[basename[1]]:
                old_ver[basename[1]] = ["%s-%s"%(basename[2],basename[3])]
            else:
                old_ver[basename[1]].append("%s-%s"%(basename[2],basename[3]))

    
    # Now we have the dicts of the packages we need to sort through them
    # First look through current packages and see if we've got 2 or more versions of 
    # a binary that is still current in portage.  If so, we'll flag the older
    # version(s) for removal (emerging with -GU will grab the most recent ones anyway
    # so the older versions are redundant).

    red_ver = []

    for q in rec_ver.keys():

        if ( ( len(rec_ver[q].keys()) > 1 ) & DEBUG ):
            print(green("Multiple SLOTs of %s detected, treating SLOTs individually"%(q)))
        
        for x in rec_ver[q].keys():

            # Now use the portage.pkgcmp function to find the most recent of the versions 
            # in the list.

            i = 0
            best_ver = rec_ver[q][x][i]
            for y in range( len(rec_ver[q][x])-1 ):
                this_ver = rec_ver[q][x][y+1]

                # Really should be using the full portage package names here......this could
                # cause pain later. 
                best_pkg = "%s-%s"%(q,best_ver)
                this_pkg = "%s-%s"%(q,this_ver)
                p1 = portage.catpkgsplit(best_pkg)[1:]
                p2 = portage.catpkgsplit(this_pkg)[1:]
                if DEBUG:
                    print(blue("%s vs %s"%(p1,p2)))
                ret = portage.pkgcmp(p1,p2)

                if ( ret < 0 ):
                    red_ver.append(best_pkg)
                    best_ver = this_ver
                    i = y+1
                elif ( ret > 0 ):
                    red_ver.append(this_pkg)
                else:
                    print(red("Warning, two instances of the version of "+x+" found!"))

            # If the best binary version present is not the most recent in portage, notify 
            # the user.

            if show_upgrades == 0:
                continue

            latest_portage_pkg = portage.portdb.xmatch("bestmatch-visible", q)
            # If we have a packaged in the binary directory that is masked the above call
            # will return nothing.  If this occurs we assume the binary is a special case
            # and skip over it.
            if latest_portage_pkg == None:
                continue
            p2 = portage.catpkgsplit(latest_portage_pkg)[1:]
            best_pkg = "%s-%s"%(q,best_ver)
            p1 = portage.catpkgsplit(best_pkg)[1:]
            ret = portage.pkgcmp(p1,p2)
            if ( ret < 0 ):
                print(blue("Upgrade for %s to %s available"%(best_pkg,latest_portage_pkg)))

        # Now we get rid of the redundant packages if required

    if move_red:

        # Need to add the old packages to the list of redundant packages.
        for y in old_ver.keys():
            for z in range( len(old_ver[y]) ):
                old_pkg = "%s-%s"%(y, old_ver[y][z] )

                # we need to check that old packages do actually have a newer binary
                # available before we move them or things will break.
                
                if ( rec_ver.has_key(y) or mask_ver.has_key(y) ):
                    red_ver.append(old_pkg)
                else:
                    best_match = portage.portdb.xmatch("bestmatch-visible", y)
                    print(red("Can't remove old package %s, no newer binary version present"%(old_pkg)))
        
        for y in red_ver:
            # since catpkgsplit qualifies packages with -r0 suffix if there is only one
            # version, we need to strip this off so our package names match the names of 
            # binaries.
            if y[-3:] == "-r0":
                y = y[:-3]
            target_file = "%s/%s.tbz2"%(target_dir,y)
            dest_file = "%s/%s.tbz2"%(red_pkg_dir,y)
            if DEBUG:
                print("Moving %s to %s"%(target_file,dest_file))
            print(blue("Redundant package %s moved"%(y)))
            os.system("mv %s %s"%(target_file,dest_file))

    if ( len(red_ver) > 0 ):
        print(blue("Contents of binary directory changed, regenerating index"))
        gen_metadata(target_dir)
                    

    return [ rec_ver, mask_ver, old_ver ]
        
         
def gen_metadata(target_dir=BIN_PKG_DIR, idx_file=BIN_PKG_DIR+"/metadata.idx"):
    """Looks in the directory specified and generates a metadata.idx file
    from all the XPAK segments of binaries in the target directory"""

    if os.path.exists(target_dir):
        perms = os.stat(target_dir)[ST_MODE]
    else:
        print(red("Error "+target_dir+": no such directory"))
        sys.exit(1)
    if not perms & S_IRWXU:
        print(red("Directory "+target_dir+" is not writeable, aborting"))

    # Create the base metadata structure
    if DEBUG:
        print(green("Creating base metadata structure"))
        
    metadata = {}
    metadata["indexname"] = ""
    metadata["timestamp"] = int(time.time())
    metadata["unmodified"] = 0
    metadata["data"] = {}

    if DEBUG:
        print(green("Obtaining listing of "+target_dir))
        
    pkgs = os.listdir(target_dir)

    for x in pkgs:

        # Find the size of of the XPAK segement and read it in from the end of the file
        tbz_file = xpak.tbz2(BIN_PKG_DIR+"/"+x)
        a = open(BIN_PKG_DIR+"/"+x)
        a.seek(-tbz_file.xpaksize,2)
        myxpak = a.read(tbz_file.xpaksize-8)
        a.close

        myid = xpak.xsplit_mem(myxpak)

        if not myid:
            print(red("No XPAK segment found in "+x+", skipping"))
            continue

        # Add to metadata if its a valid segement
        if myid[0]:
            if DEBUG:
                print(green("Found valid XPAK segement in "+x))
            metadata["data"][x] = getbinpkg.make_metadata_dict(myid)
        else:
            print(red("Error, corrupt XPAK segement in "+x))

    # Now dump the metadata to file.  I'll stick in some old file rotation here eventually
    # but for now we'll just see if it works.

    try:
        metadatafile = open(idx_file, "w")
        cPickle.dump(metadata["data"],metadatafile)
        metadatafile.close()
    except Exception, e:
        print("!!! Failed to write binary metadata to disk!")
        print("!!! "+str(e))
    
    return