Attachment #123003 for bug #175891




		self._pkg_paths[mycpv] = mypath # cache for future lookups
		return os.path.join(self.pkgdir, mypath)

	def isremote(self, pkgname):
		"""Returns true if the package is kept remotely and it has not been
		downloaded (or it is only partially downloaded)."""
		pkg_path = self.getname(pkgname)
		if os.path.basename(pkg_path) not in self.remotepkgs:
			return False
		if os.path.exists(pkg_path) and \
			os.path.basename(pkg_path) not in self.invalids:
			return False
		return True

	def get_use(self,pkgname):
		mysplit=pkgname.split("/")

		return tbz2.getfile("USE").split()

	def gettbz2(self,pkgname):
		"""Fetches the package from a remote site, if necessary.  Attempts to
		resume if the file appears to be partially downloaded."""
		print "Fetching '"+str(pkgname)+"'"
		tbz2_path = self.getname(pkgname)
		tbz2name = os.path.basename(tbz2_path)
		resume = False
		if os.path.exists(tbz2_path):
			if (tbz2name not in self.invalids):
				return 1
			else:
				resume = True
				writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n",
					noiselevel=-1)
		mydest = self.pkgdir+"/All/"

			os.makedirs(mydest, 0775)
		except (OSError, IOError):
			pass
		from urlparse import urljoin, urlparse
		url = urljoin(self.settings["PORTAGE_BINHOST"], tbz2name)
		protocol = urlparse(url)[0]
		fcmd_prefix = "FETCHCOMMAND"
		if resume:
			fcmd_prefix = "RESUMECOMMAND"
		fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
		if not fcmd:
			fcmd = self.settings.get(fcmd_prefix)
		if not getbinpkg.file_get(url, mydest, fcmd=fcmd):
			raise portage_exception.FileNotFound(tbz2_path)

	def getslot(self,mycatpkg):
		"Get a slot for a catpkg; assume it exists."

Lines 141-146 Link Here

(-)pym/getbinpkg.py (-22 / +33 lines)
141	conn.login(username)	141	conn.login(username)
142	conn.set_pasv(passive)	142	conn.set_pasv(passive)
143	conn.set_debuglevel(0)	143	conn.set_debuglevel(0)
		144	elif protocol == "sftp":
		145	try:
		146	import paramiko
		147	except ImportError:
		148	raise NotImplementedError(
		149	"paramiko must be installed for sftp support")
		150	t = paramiko.Transport(host)
		151	t.connect(username=username, password=password)
		152	conn = paramiko.SFTPClient.from_transport(t)
144	else:	153	else:
145	raise NotImplementedError, "%s is not a supported protocol." % protocol	154	raise NotImplementedError, "%s is not a supported protocol." % protocol
146		155
Lines 307-312 Link Here
307	del olddir	316	del olddir
308	else:	317	else:
309	listing = conn.nlst(address)	318	listing = conn.nlst(address)
		319	elif protocol == "sftp":
		320	listing = conn.listdir(address)
310	else:	321	else:
311	raise TypeError, "Unknown protocol. '%s'" % protocol	322	raise TypeError, "Unknown protocol. '%s'" % protocol
312		323
Lines 332-337 Link Here
332	data,rc,msg = make_http_request(conn, address, params, headers)	343	data,rc,msg = make_http_request(conn, address, params, headers)
333	elif protocol in ["ftp"]:	344	elif protocol in ["ftp"]:
334	data,rc,msg = make_ftp_request(conn, address, -chunk_size)	345	data,rc,msg = make_ftp_request(conn, address, -chunk_size)
		346	elif protocol == "sftp":
		347	f = conn.open(address)
		348	try:
		349	f.seek(-chunk_size, 2)
		350	data = f.read()
		351	finally:
		352	f.close()
335	else:	353	else:
336	raise TypeError, "Unknown protocol. '%s'" % protocol	354	raise TypeError, "Unknown protocol. '%s'" % protocol
337		355
Lines 366-393 Link Here
366	if not fcmd:	384	if not fcmd:
367	return file_get_lib(baseurl,dest,conn)	385	return file_get_lib(baseurl,dest,conn)
368		386
369	fcmd = fcmd.replace("${DISTDIR}",dest)	387	variables = {
370	fcmd = fcmd.replace("${URI}", baseurl)	388	"DISTDIR": dest,
371	fcmd = fcmd.replace("${FILE}", os.path.basename(baseurl))	389	"URI": baseurl,
372	mysplit = fcmd.split()	390	"FILE": os.path.basename(baseurl)
373	mycmd = mysplit[0]	391	}
374	myargs = [os.path.basename(mycmd)]+mysplit[1:]	392	import shlex, StringIO
375	mypid=os.fork()	393	from portage_util import varexpand
376	if mypid == 0:	394	from portage_exec import spawn
377	try:	395	lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True)
378	os.execv(mycmd,myargs)	396	lexer.whitespace_split = True
379	except OSError:	397	myfetch = [varexpand(x, mydict=variables) for x in lexer]
380	pass	398	retval = spawn(myfetch, env=os.environ.copy())
381	sys.stderr.write("!!! Failed to spawn fetcher.\n")	399	if retval != os.EX_OK:
382	sys.stderr.flush()
383	os._exit(1)
384	retval=os.waitpid(mypid,0)[1]
385	if (retval & 0xff) == 0:
386	retval = retval >> 8
387	else:
388	sys.stderr.write("Spawned processes caught a signal.\n")
389	sys.exit(1)
390	if retval != 0:
391	sys.stderr.write("Fetcher exited with a failure condition.\n")	400	sys.stderr.write("Fetcher exited with a failure condition.\n")
392	return 0	401	return 0
393	return 1	402	return 1
Lines 409-414 Link Here
409	data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)	418	data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)
410	elif protocol in ["ftp"]:	419	elif protocol in ["ftp"]:
411	data,rc,msg = make_ftp_request(conn, address, dest=dest)	420	data,rc,msg = make_ftp_request(conn, address, dest=dest)
		421	elif protocol == "sftp":
		422	conn.get(address, dest)
412	else:	423	else:
413	raise TypeError, "Unknown protocol. '%s'" % protocol	424	raise TypeError, "Unknown protocol. '%s'" % protocol
414		425

Lines 48-53 Link Here

(-)bin/emerge (-3 / +26 lines)
48	import portage_exception	48	import portage_exception
49	from portage_data import secpass	49	from portage_data import secpass
50	from portage_util import normalize_path as normpath	50	from portage_util import normalize_path as normpath
		51	from portage_util import writemsg
51		52
52	if not hasattr(__builtins__, "set"):	53	if not hasattr(__builtins__, "set"):
53	from sets import Set as set	54	from sets import Set as set
Lines 2954-2959 Link Here
2954		2955
2955	def merge(self, mylist, favorites, mtimedb):	2956	def merge(self, mylist, favorites, mtimedb):
2956	failed_fetches = []	2957	failed_fetches = []
		2958	fetchonly = "--fetchonly" in self.myopts or \
		2959	"--fetch-all-uri" in self.myopts
2957	mymergelist=[]	2960	mymergelist=[]
2958	ldpath_mtimes = mtimedb["ldpath"]	2961	ldpath_mtimes = mtimedb["ldpath"]
2959	xterm_titles = "notitles" not in self.settings.features	2962	xterm_titles = "notitles" not in self.settings.features
Lines 3250-3258 Link Here
3250	short_msg = "emerge: (%s of %s) %s Fetch" % \	3253	short_msg = "emerge: (%s of %s) %s Fetch" % \
3251	(mergecount, len(mymergelist), pkg_key)	3254	(mergecount, len(mymergelist), pkg_key)
3252	emergelog(xterm_titles, msg, short_msg=short_msg)	3255	emergelog(xterm_titles, msg, short_msg=short_msg)
3253	if not self.trees[myroot]["bintree"].gettbz2(	3256	try:
3254	pkg_key):	3257	self.trees[myroot]["bintree"].gettbz2(pkg_key)
3255	return 1	3258	except portage_exception.FileNotFound:
		3259	writemsg("!!! Fetching Binary failed " + \
		3260	"for '%s'\n" % pkg_key, noiselevel=-1)
		3261	if not fetchonly:
		3262	return 1
		3263	failed_fetches.append(pkg_key)
		3264	except portage_exception.DigestException, e:
		3265	writemsg("\n!!! Digest verification failed:\n",
		3266	noiselevel=-1)
		3267	writemsg("!!! %s\n" % e.value[0],
		3268	noiselevel=-1)
		3269	writemsg("!!! Reason: %s\n" % e.value[1],
		3270	noiselevel=-1)
		3271	writemsg("!!! Got: %s\n" % e.value[2],
		3272	noiselevel=-1)
		3273	writemsg("!!! Expected: %s\n" % e.value[3],
		3274	noiselevel=-1)
		3275	os.unlink(mytbz2)
		3276	if not fetchonly:
		3277	return 1
		3278	failed_fetches.append(pkg_key)
3256	finally:	3279	finally:
3257	if tbz2_lock:	3280	if tbz2_lock:
3258	portage_locks.unlockfile(tbz2_lock)	3281	portage_locks.unlockfile(tbz2_lock)

Return to bug 175891

Lines 6781-6791 Link Here

(-)pym/portage.py (-13 / +29 lines)
6781	self._pkg_paths[mycpv] = mypath # cache for future lookups	6781	self._pkg_paths[mycpv] = mypath # cache for future lookups
6782	return os.path.join(self.pkgdir, mypath)	6782	return os.path.join(self.pkgdir, mypath)
6783		6783
6784	def isremote(self,pkgname):	6784	def isremote(self, pkgname):
6785	"Returns true if the package is kept remotely."	6785	"""Returns true if the package is kept remotely and it has not been
6786	mysplit=pkgname.split("/")	6786	downloaded (or it is only partially downloaded)."""
6787	remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2")	6787	pkg_path = self.getname(pkgname)
6788	return remote	6788	if os.path.basename(pkg_path) not in self.remotepkgs:
		6789	return False
		6790	if os.path.exists(pkg_path) and \
		6791	os.path.basename(pkg_path) not in self.invalids:
		6792	return False
		6793	return True
6789		6794
6790	def get_use(self,pkgname):	6795	def get_use(self,pkgname):
6791	mysplit=pkgname.split("/")	6796	mysplit=pkgname.split("/")
Lines 6795-6808 Link Here
6795	return tbz2.getfile("USE").split()	6800	return tbz2.getfile("USE").split()
6796		6801
6797	def gettbz2(self,pkgname):	6802	def gettbz2(self,pkgname):
6798	"fetches the package from a remote site, if necessary."	6803	"""Fetches the package from a remote site, if necessary. Attempts to
		6804	resume if the file appears to be partially downloaded."""
6799	print "Fetching '"+str(pkgname)+"'"	6805	print "Fetching '"+str(pkgname)+"'"
6800	mysplit = pkgname.split("/")	6806	tbz2_path = self.getname(pkgname)
6801	tbz2name = mysplit[1]+".tbz2"	6807	tbz2name = os.path.basename(tbz2_path)
6802	if not self.isremote(pkgname):	6808	resume = False
		6809	if os.path.exists(tbz2_path):
6803	if (tbz2name not in self.invalids):	6810	if (tbz2name not in self.invalids):
6804	return	6811	return 1
6805	else:	6812	else:
		6813	resume = True
6806	writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n",	6814	writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n",
6807	noiselevel=-1)	6815	noiselevel=-1)
6808	mydest = self.pkgdir+"/All/"	6816	mydest = self.pkgdir+"/All/"
Lines 6810-6818 Link Here
6810	os.makedirs(mydest, 0775)	6818	os.makedirs(mydest, 0775)
6811	except (OSError, IOError):	6819	except (OSError, IOError):
6812	pass	6820	pass
6813	return getbinpkg.file_get(	6821	from urlparse import urljoin, urlparse
6814	self.settings["PORTAGE_BINHOST"] + "/" + tbz2name,	6822	url = urljoin(self.settings["PORTAGE_BINHOST"], tbz2name)
6815	mydest, fcmd=self.settings["RESUMECOMMAND"])	6823	protocol = urlparse(url)[0]
		6824	fcmd_prefix = "FETCHCOMMAND"
		6825	if resume:
		6826	fcmd_prefix = "RESUMECOMMAND"
		6827	fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
		6828	if not fcmd:
		6829	fcmd = self.settings.get(fcmd_prefix)
		6830	if not getbinpkg.file_get(url, mydest, fcmd=fcmd):
		6831	raise portage_exception.FileNotFound(tbz2_path)
6816		6832
6817	def getslot(self,mycatpkg):	6833	def getslot(self,mycatpkg):
6818	"Get a slot for a catpkg; assume it exists."	6834	"Get a slot for a catpkg; assume it exists."