Index: pym/portage.py =================================================================== --- pym/portage.py (revision 7012) +++ pym/portage.py (revision 7023) @@ -6781,11 +6781,16 @@ self._pkg_paths[mycpv] = mypath # cache for future lookups return os.path.join(self.pkgdir, mypath) - def isremote(self,pkgname): - "Returns true if the package is kept remotely." - mysplit=pkgname.split("/") - remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2") - return remote + def isremote(self, pkgname): + """Returns true if the package is kept remotely and it has not been + downloaded (or it is only partially downloaded).""" + pkg_path = self.getname(pkgname) + if os.path.basename(pkg_path) not in self.remotepkgs: + return False + if os.path.exists(pkg_path) and \ + os.path.basename(pkg_path) not in self.invalids: + return False + return True def get_use(self,pkgname): mysplit=pkgname.split("/") @@ -6795,14 +6800,17 @@ return tbz2.getfile("USE").split() def gettbz2(self,pkgname): - "fetches the package from a remote site, if necessary." + """Fetches the package from a remote site, if necessary. Attempts to + resume if the file appears to be partially downloaded.""" print "Fetching '"+str(pkgname)+"'" - mysplit = pkgname.split("/") - tbz2name = mysplit[1]+".tbz2" - if not self.isremote(pkgname): + tbz2_path = self.getname(pkgname) + tbz2name = os.path.basename(tbz2_path) + resume = False + if os.path.exists(tbz2_path): if (tbz2name not in self.invalids): - return + return 1 else: + resume = True writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n", noiselevel=-1) mydest = self.pkgdir+"/All/" @@ -6810,9 +6818,17 @@ os.makedirs(mydest, 0775) except (OSError, IOError): pass - return getbinpkg.file_get( - self.settings["PORTAGE_BINHOST"] + "/" + tbz2name, - mydest, fcmd=self.settings["RESUMECOMMAND"]) + from urlparse import urljoin, urlparse + url = urljoin(self.settings["PORTAGE_BINHOST"], tbz2name) + protocol = urlparse(url)[0] + fcmd_prefix = "FETCHCOMMAND" + if resume: + fcmd_prefix = "RESUMECOMMAND" + fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper()) + if not fcmd: + fcmd = self.settings.get(fcmd_prefix) + if not getbinpkg.file_get(url, mydest, fcmd=fcmd): + raise portage_exception.FileNotFound(tbz2_path) def getslot(self,mycatpkg): "Get a slot for a catpkg; assume it exists." Index: pym/getbinpkg.py =================================================================== --- pym/getbinpkg.py (revision 7012) +++ pym/getbinpkg.py (revision 7023) @@ -141,6 +141,15 @@ conn.login(username) conn.set_pasv(passive) conn.set_debuglevel(0) + elif protocol == "sftp": + try: + import paramiko + except ImportError: + raise NotImplementedError( + "paramiko must be installed for sftp support") + t = paramiko.Transport(host) + t.connect(username=username, password=password) + conn = paramiko.SFTPClient.from_transport(t) else: raise NotImplementedError, "%s is not a supported protocol." % protocol @@ -307,6 +316,8 @@ del olddir else: listing = conn.nlst(address) + elif protocol == "sftp": + listing = conn.listdir(address) else: raise TypeError, "Unknown protocol. '%s'" % protocol @@ -332,6 +343,13 @@ data,rc,msg = make_http_request(conn, address, params, headers) elif protocol in ["ftp"]: data,rc,msg = make_ftp_request(conn, address, -chunk_size) + elif protocol == "sftp": + f = conn.open(address) + try: + f.seek(-chunk_size, 2) + data = f.read() + finally: + f.close() else: raise TypeError, "Unknown protocol. '%s'" % protocol @@ -366,28 +384,19 @@ if not fcmd: return file_get_lib(baseurl,dest,conn) - fcmd = fcmd.replace("${DISTDIR}",dest) - fcmd = fcmd.replace("${URI}", baseurl) - fcmd = fcmd.replace("${FILE}", os.path.basename(baseurl)) - mysplit = fcmd.split() - mycmd = mysplit[0] - myargs = [os.path.basename(mycmd)]+mysplit[1:] - mypid=os.fork() - if mypid == 0: - try: - os.execv(mycmd,myargs) - except OSError: - pass - sys.stderr.write("!!! Failed to spawn fetcher.\n") - sys.stderr.flush() - os._exit(1) - retval=os.waitpid(mypid,0)[1] - if (retval & 0xff) == 0: - retval = retval >> 8 - else: - sys.stderr.write("Spawned processes caught a signal.\n") - sys.exit(1) - if retval != 0: + variables = { + "DISTDIR": dest, + "URI": baseurl, + "FILE": os.path.basename(baseurl) + } + import shlex, StringIO + from portage_util import varexpand + from portage_exec import spawn + lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True) + lexer.whitespace_split = True + myfetch = [varexpand(x, mydict=variables) for x in lexer] + retval = spawn(myfetch, env=os.environ.copy()) + if retval != os.EX_OK: sys.stderr.write("Fetcher exited with a failure condition.\n") return 0 return 1 @@ -409,6 +418,8 @@ data,rc,msg = make_http_request(conn, address, params, headers, dest=dest) elif protocol in ["ftp"]: data,rc,msg = make_ftp_request(conn, address, dest=dest) + elif protocol == "sftp": + conn.get(address, dest) else: raise TypeError, "Unknown protocol. '%s'" % protocol Index: bin/emerge =================================================================== --- bin/emerge (revision 7012) +++ bin/emerge (revision 7023) @@ -48,6 +48,7 @@ import portage_exception from portage_data import secpass from portage_util import normalize_path as normpath +from portage_util import writemsg if not hasattr(__builtins__, "set"): from sets import Set as set @@ -2954,6 +2955,8 @@ def merge(self, mylist, favorites, mtimedb): failed_fetches = [] + fetchonly = "--fetchonly" in self.myopts or \ + "--fetch-all-uri" in self.myopts mymergelist=[] ldpath_mtimes = mtimedb["ldpath"] xterm_titles = "notitles" not in self.settings.features @@ -3250,9 +3253,29 @@ short_msg = "emerge: (%s of %s) %s Fetch" % \ (mergecount, len(mymergelist), pkg_key) emergelog(xterm_titles, msg, short_msg=short_msg) - if not self.trees[myroot]["bintree"].gettbz2( - pkg_key): - return 1 + try: + self.trees[myroot]["bintree"].gettbz2(pkg_key) + except portage_exception.FileNotFound: + writemsg("!!! Fetching Binary failed " + \ + "for '%s'\n" % pkg_key, noiselevel=-1) + if not fetchonly: + return 1 + failed_fetches.append(pkg_key) + except portage_exception.DigestException, e: + writemsg("\n!!! Digest verification failed:\n", + noiselevel=-1) + writemsg("!!! %s\n" % e.value[0], + noiselevel=-1) + writemsg("!!! Reason: %s\n" % e.value[1], + noiselevel=-1) + writemsg("!!! Got: %s\n" % e.value[2], + noiselevel=-1) + writemsg("!!! Expected: %s\n" % e.value[3], + noiselevel=-1) + os.unlink(mytbz2) + if not fetchonly: + return 1 + failed_fetches.append(pkg_key) finally: if tbz2_lock: portage_locks.unlockfile(tbz2_lock)