Go to:
Gentoo Home
Documentation
Forums
Lists
Bugs
Planet
Store
Wiki
Get Gentoo!
Gentoo's Bugzilla – Attachment 30046 Details for
Bug 34384
[code] Functions to help maintain GRP
Home
|
New
–
[Ex]
|
Browse
|
Search
|
Privacy Policy
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
Updated version of binpkgmain.py
binpkgmain.py (text/plain), 22.10 KB, created by
Baz (RETIRED)
on 2004-04-25 16:56:32 UTC
(
hide
)
Description:
Updated version of binpkgmain.py
Filename:
MIME Type:
Creator:
Baz (RETIRED)
Created:
2004-04-25 16:56:32 UTC
Size:
22.10 KB
patch
obsolete
># NOTE: this script is under CVS control. > ># binpkgmain - collection of functions for the maintenence of binary ># packages for use with emerge -G ># ># Baz - 12/11/03 > >import xpak,getbinpkg,sys,portage,cPickle,time,re,string >from output import * >from stat import * > ># Globals > >portdb = portage.portdbapi() > ># location of the binary repository in the filesystem >BIN_PKG_DIR = "/gentoo/dist/1.4/packages/All" >OLD_PKG_DIR = "/gentoo/dist/1.4/packages/old-packages" ># This is a text file containing a list of packages for the script to ignore ># due to portage/ebuild peculiarities. >MASK_FILE = BIN_PKG_DIR+"/masked_packages.txt" > >DEBUG = 0 > >def check_versions(move_red=0, show_upgrades=1, target_dir=BIN_PKG_DIR, red_pkg_dir=OLD_PKG_DIR): > """Looks in the directory specified and checks to see if any of the packages > contained therein are outdated in terms of the local current portage tree. Will > also advise if there are newer versions of packages available""" > > if not os.path.exists(target_dir): > print(red("Error "+target_dir+": no such directory")) > > idx_file = target_dir+"/metadata.idx" > if not os.path.exists(idx_file): > print(blue("No metadata file present, will generate one now...")) > gen_metadata(target_dir) > > # Check to see if the metadata file is up to date > dir_age = os.stat(target_dir)[ST_MTIME] > idx_age = os.stat(idx_file)[ST_MTIME] > > if ( dir_age > idx_age ): > print(blue("Metadata file out of date, generating new one...")) > gen_metadata(target_dir) > > metadata_file = open(idx_file, "r") > print(blue("Loading metadata pickle, please wait...")) > metadata = cPickle.load(metadata_file) > metadata_file.close > > > if DEBUG: > print(green("Obtaining listing of "+target_dir)) > > rec_ver = {} > mask_ver = {} > old_ver = {} > > pkgs = os.listdir(target_dir) > > # Loading list of packages to exclude from package mask file. > if os.path.exists(MASK_FILE): > masked_pkgs = [] > print(blue("Package mask file found, loading.")) > mask = open(MASK_FILE, "r") > for line in mask.readlines(): > comment = re.search("^#.*", line) > space = re.search("^\s+", line) > if ( comment or space ): > continue > else: > masked_pkgs.append(line) > else: > print(green("No package mask file found.")) > > > print(blue("\nSorting through binary packages.....\n")) > > > for x in pkgs: > > # first we need to split up the package name to get just the basename with no > # version info on the end. > > if x[-5:]!=".tbz2": > if DEBUG: > print(green(x+" doesn't look like a binary package, skipping")) > continue > > basename = portage.catpkgsplit(x[:-5]) > > if not basename[1]: > print(green("Can find basename for package "+x+", skipping")) > continue > > flag = 0 > for pkg in masked_pkgs: > reg = "%s-%s"%(basename[1],basename[2]) > skip = re.search( re.escape(reg), pkg) > if skip: > print(red("%s identified as a masked package, skipping"%(reg))) > flag = 1 > break > > if flag: > continue > > versions = portdb.xmatch("match-visible", basename[1]) > > # find SLOT from metadata (take off trailing newline) > slot = string.rstrip(metadata[x]["SLOT"]) > > # compile regexp > pkg_regexp = re.compile( "%s$"%( re.escape(x[:-5]) ) ) > > flag = 0 > for y in versions: > if pkg_regexp.search(y): > flag = 1 > if not rec_ver.has_key(basename[1]): > rec_ver[basename[1]] = {} > rec_ver[basename[1]][slot] = [ "%s-%s"%(basename[2],basename[3]) ] > else: > if not rec_ver[basename[1]].has_key(slot): > rec_ver[basename[1]][slot] = ["%s-%s"%(basename[2],basename[3])] > else: > rec_ver[basename[1]][slot].append("%s-%s"%(basename[2],basename[3])) > if DEBUG: > print(green(x[:-5]+" is in portage")) > > if flag == 1: > continue > > flag = 0 > versions = portdb.xmatch("match-all", basename[1]) > for y in versions: > if pkg_regexp.search(y): > flag = 1 > if not mask_ver.has_key(basename[1]): > mask_ver[basename[1]] = {} > mask_ver[basename[1]][slot] = [ "%s-%s"%(basename[2],basename[3]) ] > else: > if not mask_ver[basename[1]].has_key(slot): > mask_ver[basename[1]][slot] = ["%s-%s"%(basename[2],basename[3])] > else: > mask_ver[basename[1]][slot].append("%s-%s"%(basename[2],basename[3])) > # Let users know they've got masked packages > print(green(x[:-5]+" is MASKED in portage")) > > if flag == 1: > continue > > > # Find all visible packages and recommend upgrade to one with the same SLOT. > ># slot_match = [] ># best_match = portdb.xmatch("list-visible", basename[1]) ># for y in best_match: ># y_slot = portdb.aux_get( y, ["SLOT"] ) ># if ( slot == y_slot[0] ): ># slot_match.append(y) ># ># if ( len(slot_match) == 1 ): ># rec_upgrade = slot_match[0] ># else: ># best_pkg = slot_match[0] > > rec_upgrade = best_in_slot(slot, basename[1]) > > > print(red(x[:-5]+" is no longer in portage you should install "+rec_upgrade)) > if not old_ver.has_key(basename[1]): > old_ver[basename[1]] = {} > old_ver[basename[1]] = [ "%s-%s"%(basename[2],basename[3]) ] > else: > if not old_ver[basename[1]]: > old_ver[basename[1]] = ["%s-%s"%(basename[2],basename[3])] > else: > old_ver[basename[1]].append("%s-%s"%(basename[2],basename[3])) > > > # Now we have the dicts of the packages we need to sort through them > # First look through current packages and see if we've got 2 or more versions of > # a binary that is still current in portage. If so, we'll flag the older > # version(s) for removal (emerging with -GU will grab the most recent ones anyway > # so the older versions are redundant). > > red_ver = [] > > for q in rec_ver.keys(): > > if ( ( len(rec_ver[q].keys()) > 1 ) & DEBUG ): > print(green("Multiple SLOTs of %s detected, treating SLOTs individually"%(q))) > > for x in rec_ver[q].keys(): > > # Now use the portage.best function to find the most recent of the versions > # in the list. > > match_list = [] > for z in rec_ver[q][x]: > match_list.append("%s-%s"%(q,z)) > best_pkg = portage.best(match_list) > > # Now add the rest of the versions to the red_ver list. > for z in rec_ver[q][x]: > if not re.match( re.compile(re.escape(best_pkg)), "%s-%s"%(q,z) ): > red_ver.append( "%s-%s"%(q,z) ) > > ># i = 0 ># best_ver = rec_ver[q][x][i] ># for y in range( len(rec_ver[q][x])-1 ): ># this_ver = rec_ver[q][x][y+1] ># ># # Really should be using the full portage package names here......this could ># # cause pain later. ># best_pkg = "%s-%s"%(q,best_ver) ># this_pkg = "%s-%s"%(q,this_ver) ># p1 = portage.catpkgsplit(best_pkg)[1:] ># p2 = portage.catpkgsplit(this_pkg)[1:] ># if DEBUG: ># print(blue("%s vs %s"%(p1,p2))) ># ret = portage.pkgcmp(p1,p2) ># ># if ( ret < 0 ): ># red_ver.append(best_pkg) ># best_ver = this_ver ># i = y+1 ># elif ( ret > 0 ): ># red_ver.append(this_pkg) ># else: ># print(red("Warning, two instances of the version of "+x+" found!")) > > # If the best binary version present is not the most recent in portage, notify > # the user. > > if show_upgrades == 0: > continue > ># latest_portage_pkg = portdb.xmatch("bestmatch-visible", q) ># print(red("Finding best pkg %s in SLOT %s"%(q,x))) > latest_portage_pkg = best_in_slot(x,q) ># print(red("\t%s"%(latest_portage_pkg))) > > # If we have a package in the binary directory that is masked the above call > # will return nothing. If this occurs we assume the binary is a special case > # and skip over it. > if latest_portage_pkg == "": > print(red("Package %s may have wrong SLOT info in XPAK segement or maybe MASKED, please check"%(q))) > continue > p2 = portage.catpkgsplit(latest_portage_pkg)[1:] > #best_pkg = "%s-%s"%(q,best_ver) > p1 = portage.catpkgsplit(best_pkg)[1:] > ret = portage.pkgcmp(p1,p2) > if ( ret < 0 ): > print(blue("Upgrade for %s to %s available"%(best_pkg,latest_portage_pkg))) > > # Now we get rid of the redundant packages if required > > if move_red: > > print(blue("\nVerifying and moving redundant packages...")) > > # Need to add the old packages to the list of redundant packages. > for y in old_ver.keys(): > for z in range( len(old_ver[y]) ): > old_pkg = "%s-%s"%(y, old_ver[y][z] ) > > # we need to check that old packages do actually have a newer binary > # available before we move them or things will break. > > if ( rec_ver.has_key(y) or mask_ver.has_key(y) ): > red_ver.append(old_pkg) > else: > best_match = portdb.xmatch("bestmatch-visible", y) > print(red("Can't remove old package %s, no newer binary version present"%(old_pkg))) > > # Now we need to check through the dependancies of the packages still in the > # binary repository to make sure that none of the packages we intend to remove > # are still needed. > > confirmed_red_ver = [] > > for p in red_ver: > > pot_deps = {} > filename = re.compile("%s.tbz2"%(p)) > pkg_split = portage.catpkgsplit(p) > pkg_mo = re.compile("%s"%(pkg_split[1])) > > sys.stdout.write(blue("Checking if %s is a dependant of another package..."%(p))) > > for z in metadata.keys(): > # We don't need to check the target packages own dependencies > if ( filename.match(z) ): > if DEBUG: > print(red("\nFound target package, %s, in metadata, skipping"%(z))) > continue > > for y in ["DEPEND", "RDEPEND", "PDEPEND"]: > deps = string.split(metadata[z][y]) > # Find potential dependancies. We'll have to look through these > # a bit more carefully once we've narrowed them down. > for x in deps: > if ( pkg_mo.search(x) ): > if DEBUG: > print(green("Found dependancy potential match, %s, in %s"%(x,z))) > if ( pot_deps.has_key(z) ): > pot_deps[z].append(x) > else: > pot_deps[z] = [x] > > # Now check a bit more carefully throught the potential deps > > # Need to set this out here too.... some pkgs have no deps, dont make it into the > # following loop and then inherit keep from the previous loop. > keep = 0 > blockers = [] > > for x in pot_deps.keys(): > if DEBUG: > print(blue("Checking deps in %s"%(x))) > for y in pot_deps[x]: > > eq_wc = 0 > > # If theres a keyword ending in a ? then its a dynamic depend keyword > # and we can skip it. If the package that constitues the dynamic > # dependancy matched the package regexp it'll be considered anyway. > if ( re.search( "\w+\?", y ) ): > continue > > if ( ( re.match( re.escape("!"), y) ) or ( re.match( re.escape("!="), y) ) ): > # This is a blocker, not a dependancy. We'll skip it. > continue > > if DEBUG: > print(green("\t checking for match with %s"%(y))) > > # Need to check for extended atom prefixes and treat them accordingly > if ( re.match( "=.*\*$", y) ): > # Treat =cat/pkg-ver* types specially as need to do some string > # munging before we pass to pkgcmp > y = y.rstrip("*") > eq_wc = 1 # set flag so we can treat this case specially later > > elif ( re.search( "\d+\*$", y ) ): > # If version ends with a star we don't need it (in fact it screws > # over catpkgsplit). The function portage.pkgcmp is accurate in > # comparing versions with minor numbers removed (provided there is > # no leading = prefix) > y = y.rstrip("*") > > if ( re.match( re.escape("~"), y ) ): > # Tildes match revisions, since portage.pkgcmp handles these we'll > # just take it out (catpkgsplit automatically adds -r0 if there > # is no revision specified in the package string). > y = y.strip("~") > > # Take off the prefixes for name comparision as these kill catpkgsplit > base_pot_pkg = y.strip(">") > base_pot_pkg = base_pot_pkg.strip("<") > base_pot_pkg = base_pot_pkg.strip("=") > # Break up the potential package name into components > pot_pkg_split = portage.catpkgsplit(base_pot_pkg) > > # If there is no return from catpkgsplit we can assume(!?) that there is > # no version info for the dependancy (either its a generic depend, or a > # virtual depend). Since we are only considering the removal of packages > # for which there is a newer binary in place, its safe to ignore these > # particular types of dependancy. > if ( pot_pkg_split == None ): > continue > > # At this point there is a prefix, therefore there must be version info so > # we can just check the 2 element from the catpkgsplit to see if we have a > # package match > > pkg_name_mo = re.compile(pot_pkg_split[1]) > if not ( pkg_name_mo.match(pkg_split[1]) ): > # No package match here, skip. > if DEBUG: > print(green("\t%s doesn't match %s, skipping"%(pkg_split[1],y))) > continue > > # Start checking for the various prefixes and test the package > > # The first 2 checks are redundant since the existence of a more up to date > # package is implicit in this binary being in the red_ver list anyway. > # > #if ( re.match( re.escape(">"), y) ): > # out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) > # if ( out > 1 ): > # # the dep string (pot_pkg) is older (eg lower version) than > # # the pkg we are considering removing so its a valid dependency > # # and not safe to remove > # keep = 1 > > #if ( ( re.match( re.escape(">="), y) ) and ( keep == 0 ) ): > # out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) > # if ( out >= 1 ): > # keep = 1 > > if ( ( re.match( re.escape("<"), y) ) and ( keep == 0 ) ): > out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) > if ( out < 1 ): > keep = 1 > blockers.append(x) > > if ( ( re.match( re.escape("<="), y) ) and ( keep == 0 ) ): > out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) > if ( out <= 1 ): > keep = 1 > blockers.append(x) > > if ( ( re.match( re.escape("="), y ) ) and ( keep == 0 ) ): > if ( eq_wc ): > # Since the dep string has a wildcard char at the end we need to > # ignore any corresponding digits in the target package we are testing > # for removal. > eq_wc_pkg_split = pkg_split[:] > wc_dig = len(pot_pkg_split[2]) # no. chars in wildcarded string > eq_wc_pkg_split[2] = eq_wc_pkg_split[2][:wc_dig] # truncate target ver string > eq_wc_pkg_split[3] = "r0" # since we're chopping the version string up > > # Now check to make sure we aren't left with a hanging decimal point > if ( re.match( ".*\.$", eq_wc_pkg_split[2] ) ): > eq_wc_pkg_split[2] = eq_wc_pkg_split[2].rstrip(".") > > out = portage.pkgcmp(eq_wc_pkg_split[1:], pot_pkg_split[1:]) > else: > out = portage.pkgcmp(pkg_split[1:], pot_pkg_split[1:]) > > if ( out == 0 ): > keep = 1 > blockers.append(x) > > > # At this point we have tested all the deps for the package, if keep is still > # zero the package can be added to the list of confirmed redundant versions > > if ( keep == 0 ): > sys.stdout.write(blue("No\n")) > confirmed_red_ver.append(p) > if DEBUG: > print(red("No deps for %s found in %s"%(p,x))) > else: > print(red("\nCan't remove %s, as has dependants:"%(p))) > for t in blockers: > print(red("\t%s"%(t))) > > # Now all the packages remaining are verified redundant, lets kill em.... > > for y in confirmed_red_ver: > # since catpkgsplit qualifies packages with -r0 suffix if there is only one > # version, we need to strip this off so our package names match the names of > # binaries. > if y[-3:] == "-r0": > y = y[:-3] > target_file = "%s/%s.tbz2"%(target_dir,y) > dest_file = "%s/%s.tbz2"%(red_pkg_dir,y) > if DEBUG: > print("Moving %s to %s"%(target_file,dest_file)) > print(blue("Redundant package %s moved"%(y))) > os.system("mv %s %s"%(target_file,dest_file)) > > if ( len(red_ver) > 0 ): > print(blue("\nContents of binary directory changed, regenerating index")) > gen_metadata(target_dir) > > > return [ rec_ver, mask_ver, old_ver ] > > >def gen_metadata(target_dir=BIN_PKG_DIR, idx_file=None): > """Looks in the directory specified and generates a metadata.idx file > from all the XPAK segments of binaries in the target directory""" > > if idx_file == None: > idx_file = target_dir+"/metadata.idx" > > if os.path.exists(target_dir): > perms = os.stat(target_dir)[ST_MODE] > else: > print(red("Error "+target_dir+": no such directory")) > sys.exit(1) > if not perms & S_IRWXU: > print(red("Directory "+target_dir+" is not writeable, aborting")) > > # Create the base metadata structure > if DEBUG: > print(green("Creating base metadata structure")) > > metadata = {} > metadata["indexname"] = "" > metadata["timestamp"] = int(time.time()) > metadata["unmodified"] = 0 > metadata["data"] = {} > > if DEBUG: > print(green("Obtaining listing of "+target_dir)) > > pkgs = os.listdir(target_dir) > > for x in pkgs: > > # Find the size of of the XPAK segement and read it in from the end of the file > tbz_file = xpak.tbz2(target_dir+"/"+x) > a = open(target_dir+"/"+x) > a.seek(-tbz_file.xpaksize,2) > myxpak = a.read(tbz_file.xpaksize-8) > a.close > > myid = xpak.xsplit_mem(myxpak) > > if not myid: > if DEBUG: > print(green("No XPAK segment found in "+x+", skipping")) > continue > > # Add to metadata if its a valid segement > if myid[0]: > metadata["data"][x] = getbinpkg.make_metadata_dict(myid) > else: > print(red("Error, corrupt XPAK segement in "+x)) > > # Now dump the metadata to file. I'll stick in some old file rotation here eventually > # but for now we'll just see if it works. > > try: > metadatafile = open(idx_file, "w") > cPickle.dump(metadata["data"],metadatafile) > metadatafile.close() > except Exception, e: > print("!!! Failed to write binary metadata to disk!") > print("!!! "+str(e)) > > return > > ># Given a slot and package name, find the best version currently in portage. >def best_in_slot( slot, pkg ): > slot_match = [] > match_list = portdb.xmatch("list-visible", pkg) > for y in match_list: > y_slot = portdb.aux_get( y, ["SLOT"] ) > if ( slot == y_slot[0] ): > slot_match.append(y) > > best_match = portage.best(slot_match) > return best_match > >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 34384
:
21300
|
21345
|
24787
|
30046
|
30047
|
32046
|
32115
|
32116
|
32253
|
32424