[patch] portage-2.1.3_rc5 patch for paramiko + {FETCHCOMMAND,RESUMECOMMAND}_SFTP

sftp.patch (text/plain), 7.32 KB, created by Zac Medico on 2007-06-25 06:32:17 UTC

Index: pym/portage.py
===================================================================
--- pym/portage.py	(revision 7012)
+++ pym/portage.py	(revision 7028)
@@ -6781,11 +6781,16 @@
 		self._pkg_paths[mycpv] = mypath # cache for future lookups
 		return os.path.join(self.pkgdir, mypath)
 
-	def isremote(self,pkgname):
-		"Returns true if the package is kept remotely."
-		mysplit=pkgname.split("/")
-		remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2")
-		return remote
+	def isremote(self, pkgname):
+		"""Returns true if the package is kept remotely and it has not been
+		downloaded (or it is only partially downloaded)."""
+		pkg_path = self.getname(pkgname)
+		if os.path.basename(pkg_path) not in self.remotepkgs:
+			return False
+		if os.path.exists(pkg_path) and \
+			os.path.basename(pkg_path) not in self.invalids:
+			return False
+		return True
 
 	def get_use(self,pkgname):
 		mysplit=pkgname.split("/")
@@ -6795,14 +6800,17 @@
 		return tbz2.getfile("USE").split()
 
 	def gettbz2(self,pkgname):
-		"fetches the package from a remote site, if necessary."
+		"""Fetches the package from a remote site, if necessary.  Attempts to
+		resume if the file appears to be partially downloaded."""
 		print "Fetching '"+str(pkgname)+"'"
-		mysplit  = pkgname.split("/")
-		tbz2name = mysplit[1]+".tbz2"
-		if not self.isremote(pkgname):
+		tbz2_path = self.getname(pkgname)
+		tbz2name = os.path.basename(tbz2_path)
+		resume = False
+		if os.path.exists(tbz2_path):
 			if (tbz2name not in self.invalids):
-				return
+				return 1
 			else:
+				resume = True
 				writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n",
 					noiselevel=-1)
 		mydest = self.pkgdir+"/All/"
@@ -6810,9 +6818,18 @@
 			os.makedirs(mydest, 0775)
 		except (OSError, IOError):
 			pass
-		return getbinpkg.file_get(
-			self.settings["PORTAGE_BINHOST"] + "/" + tbz2name,
-			mydest, fcmd=self.settings["RESUMECOMMAND"])
+		from urlparse import urlparse
+		# urljoin doesn't work correctly with unrecognized protocols like sftp
+		url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
+		protocol = urlparse(url)[0]
+		fcmd_prefix = "FETCHCOMMAND"
+		if resume:
+			fcmd_prefix = "RESUMECOMMAND"
+		fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
+		if not fcmd:
+			fcmd = self.settings.get(fcmd_prefix)
+		if not getbinpkg.file_get(url, mydest, fcmd=fcmd):
+			raise portage_exception.FileNotFound(tbz2_path)
 
 	def getslot(self,mycatpkg):
 		"Get a slot for a catpkg; assume it exists."
Index: pym/getbinpkg.py
===================================================================
--- pym/getbinpkg.py	(revision 7012)
+++ pym/getbinpkg.py	(revision 7028)
@@ -141,6 +141,15 @@
 				conn.login(username)
 			conn.set_pasv(passive)
 			conn.set_debuglevel(0)
+		elif protocol == "sftp":
+			try:
+				import paramiko
+			except ImportError:
+				raise NotImplementedError(
+					"paramiko must be installed for sftp support")
+			t = paramiko.Transport(host)
+			t.connect(username=username, password=password)
+			conn = paramiko.SFTPClient.from_transport(t)
 		else:
 			raise NotImplementedError, "%s is not a supported protocol." % protocol
 
@@ -307,6 +316,8 @@
 			del olddir
 		else:
 			listing = conn.nlst(address)
+	elif protocol == "sftp":
+		listing = conn.listdir(address)
 	else:
 		raise TypeError, "Unknown protocol. '%s'" % protocol
 
@@ -332,6 +343,13 @@
 		data,rc,msg = make_http_request(conn, address, params, headers)
 	elif protocol in ["ftp"]:
 		data,rc,msg = make_ftp_request(conn, address, -chunk_size)
+	elif protocol == "sftp":
+		f = conn.open(address)
+		try:
+			f.seek(-chunk_size, 2)
+			data = f.read()
+		finally:
+			f.close()
 	else:
 		raise TypeError, "Unknown protocol. '%s'" % protocol
 	
@@ -366,28 +384,19 @@
 	if not fcmd:
 		return file_get_lib(baseurl,dest,conn)
 
-	fcmd = fcmd.replace("${DISTDIR}",dest)
-	fcmd = fcmd.replace("${URI}", baseurl)
-	fcmd = fcmd.replace("${FILE}", os.path.basename(baseurl))
-	mysplit = fcmd.split()
-	mycmd   = mysplit[0]
-	myargs  = [os.path.basename(mycmd)]+mysplit[1:]
-	mypid=os.fork()
-	if mypid == 0:
-		try:
-			os.execv(mycmd,myargs)
-		except OSError:
-			pass
-		sys.stderr.write("!!! Failed to spawn fetcher.\n")
-		sys.stderr.flush()
-		os._exit(1)
-	retval=os.waitpid(mypid,0)[1]
-	if (retval & 0xff) == 0:
-		retval = retval >> 8
-	else:
-		sys.stderr.write("Spawned processes caught a signal.\n")
-		sys.exit(1)
-	if retval != 0:
+	variables = {
+		"DISTDIR": dest,
+		"URI":     baseurl,
+		"FILE":    os.path.basename(baseurl)
+	}
+	import shlex, StringIO
+	from portage_util import varexpand
+	from portage_exec import spawn
+	lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True)
+	lexer.whitespace_split = True
+	myfetch = [varexpand(x, mydict=variables) for x in lexer]
+	retval = spawn(myfetch, env=os.environ.copy())
+	if retval != os.EX_OK:
 		sys.stderr.write("Fetcher exited with a failure condition.\n")
 		return 0
 	return 1
@@ -409,6 +418,25 @@
 		data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)
 	elif protocol in ["ftp"]:
 		data,rc,msg = make_ftp_request(conn, address, dest=dest)
+	elif protocol == "sftp":
+		rc = 0
+		try:
+			f = conn.open(address)
+		except SystemExit:
+			raise
+		except Exception:
+			rc = 1
+		else:
+			try:
+				if dest:
+					bufsize = 8192
+					while True:
+						data = f.read(bufsize)
+						if not data:
+							break
+						dest.write(data)
+			finally:
+				f.close()
 	else:
 		raise TypeError, "Unknown protocol. '%s'" % protocol
 	
Index: bin/emerge
===================================================================
--- bin/emerge	(revision 7012)
+++ bin/emerge	(revision 7028)
@@ -48,6 +48,7 @@
 import portage_exception
 from portage_data import secpass
 from portage_util import normalize_path as normpath
+from portage_util import writemsg
 
 if not hasattr(__builtins__, "set"):
 	from sets import Set as set
@@ -2954,6 +2955,8 @@
 
 	def merge(self, mylist, favorites, mtimedb):
 		failed_fetches = []
+		fetchonly = "--fetchonly" in self.myopts or \
+			"--fetch-all-uri" in self.myopts
 		mymergelist=[]
 		ldpath_mtimes = mtimedb["ldpath"]
 		xterm_titles = "notitles" not in self.settings.features
@@ -3250,9 +3253,29 @@
 							short_msg = "emerge: (%s of %s) %s Fetch" % \
 								(mergecount, len(mymergelist), pkg_key)
 							emergelog(xterm_titles, msg, short_msg=short_msg)
-							if not self.trees[myroot]["bintree"].gettbz2(
-								pkg_key):
-								return 1
+							try:
+								self.trees[myroot]["bintree"].gettbz2(pkg_key)
+							except portage_exception.FileNotFound:
+								writemsg("!!! Fetching Binary failed " + \
+									"for '%s'\n" % pkg_key, noiselevel=-1)
+								if not fetchonly:
+									return 1
+								failed_fetches.append(pkg_key)
+							except portage_exception.DigestException, e:
+								writemsg("\n!!! Digest verification failed:\n",
+									noiselevel=-1)
+								writemsg("!!! %s\n" % e.value[0],
+									noiselevel=-1)
+								writemsg("!!! Reason: %s\n" % e.value[1],
+									noiselevel=-1)
+								writemsg("!!! Got: %s\n" % e.value[2],
+									noiselevel=-1)
+								writemsg("!!! Expected: %s\n" % e.value[3],
+									noiselevel=-1)
+								os.unlink(mytbz2)
+								if not fetchonly:
+									return 1
+								failed_fetches.append(pkg_key)
 					finally:
 						if tbz2_lock:
 							portage_locks.unlockfile(tbz2_lock)