Version 1 patch to port getbinpkg to use urllib2 for proxy-awareness

0001-Port-getbinpkg-to-use-urllib2-to-enable-proxies.patch (text/plain), 14.13 KB, created by paul.drews on 2012-10-16 23:48:04 UTC

From 271a15b7191a5622db5809b1f4fdcfe6b4a247a4 Mon Sep 17 00:00:00 2001
From: Paul Drews <[e-mail]>
Date: Tue, 16 Oct 2012 07:39:21 -0700
Subject: [PATCH] Port getbinpkg to use urllib2 to enable proxies

This python file used the httplib packages, which do not
(directly) work with proxies.  This change ports getbinpkg
to use urllib2 for all its retrieval needs.  Urllib2 is
proxy-enabled, so that the build system works from behind
(proxied) firewalls.

Limitations:
  () This does not implement persistent (keepalive)
     connections
  () urllib2 does not (directly) support "sftp" urls, so
     this simply uses the original approach to continue to
     support them but without added code to handle proxies
     for sftp urls.
  () file_get_metadata() does not use any special directives
     to skip ahead to an area near the end of the page
     referenced by the url.  It has to read the entire page
     sequentially.  This not a functional difference, but
     might show a detectable performance difference in some
     obscure circumstances.
---
 pym/portage/getbinpkg.py | 338 +++++++++++++++++++++++++++++++----------------
 1 file changed, 225 insertions(+), 113 deletions(-)

diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 538f8c2..57709fa 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -12,6 +12,7 @@ from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.package.ebuild.fetch import _hide_url_passwd
 from _emerge.Package import _all_metadata_keys
+import urlparse
 
 import sys
 import socket
@@ -344,6 +345,138 @@ def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0):
 			continue            # Doesn't match.
 
 	return myarray
+
+# Begin urllib2 subclassing ========================================
+# Some utility classes to get readily parseable parseable directory
+# listings from ftp urls.
+
+import urllib
+import urllib2
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+class simplelistftpwrapper(urllib.ftpwrapper):
+    """Overrides retrfile() to give simple parsable directory listings"""
+
+    def retrfile(self, file, type):
+        import ftplib
+        self.endtransfer()
+        if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
+        else: cmd = 'TYPE ' + type; isdir = 0
+        try:
+            self.ftp.voidcmd(cmd)
+        except ftplib.all_errors:
+            self.init()
+            self.ftp.voidcmd(cmd)
+        conn = None
+        if file and not isdir:
+            # Try to retrieve as a file
+            try:
+                cmd = 'RETR ' + file
+                conn = self.ftp.ntransfercmd(cmd)
+            except ftplib.error_perm, reason:
+                if str(reason)[:3] != '550':
+                    raise IOError, ('ftp error', reason), sys.exc_info()[2]
+        if not conn:
+            # Set transfer mode to ASCII!
+            self.ftp.voidcmd('TYPE A')
+            # Try a directory listing. Verify that directory exists.
+            if file:
+                pwd = self.ftp.pwd()
+                try:
+                    try:
+                        self.ftp.cwd(file)
+                    except ftplib.error_perm, reason:
+                        raise IOError, ('ftp error', reason), sys.exc_info()[2]
+                finally:
+                    self.ftp.cwd(pwd)
+                cmd = 'NLST ' + file
+            else:
+                cmd = 'NLST'
+            conn = self.ftp.ntransfercmd(cmd)
+        self.busy = 1
+        # Pass back both a suitably decorated object and a retrieval length
+        return (urllib.addclosehook(conn[0].makefile('rb'),
+                             self.endtransfer), conn[1])
+
+from urllib import (
+    splitport,
+    splituser,
+    splitpasswd,
+    unquote,
+    splitattr,
+    splitvalue,
+    addinfourl,
+    )
+
+from urllib2 import (
+    URLError,
+    )
+
+class SimpleListFTPHandler(urllib2.FTPHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        import mimetools
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = unquote(user or '')
+        passwd = unquote(passwd or '')
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = simplelistftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+
+# End urllib2 subclassing ========================================
 			
 
 
@@ -352,46 +485,63 @@ def dir_get_list(baseurl,conn=None):
 	URI should be in the form <proto>://<site>[:port]<path>
 	Connection is used for persistent connection instances."""
 
-	if not conn:
-		keepconnection = 0
-	else:
-		keepconnection = 1
-
-	conn,protocol,address,params,headers = create_conn(baseurl, conn)
+	# Cases:
+	#   sftp with or without proxy:
+	#     Use connection-based proxy-unaware code to attempt to retrieve the
+	#     listing and simply return it.
+	#
+	#     Note that a (future) way to get sftp fully supported through proxies
+	#     would be to write a SFTPHandler (depending on paramiko), add this to
+	#     the url openet, and then let sftp urls be handled like the proxy-ready
+	#     cases below.
+	#
+	# Proxy-ready cases:
+	#   http, https:
+	#     The retrieved page is text/html with links for each file
+	#   ftp with no proxy:
+	#     Use a subclass of urllib2.FTPHandler that overrides retrfile() to use
+	#     "NLST" verb to get a text/plain listing with simple filename per line
+	#   ftp through http proxy:
+	#     The retrieved page is text/html with links for each file
+	#
+	# Bottom Line for proxy-ready cases:
+	#   if retrieved page is text/html:
+	#     use ParseLinks to find anchors and deem them to be files
+	#   elseif retrieved page is text/plain:
+	#     split and strip lines deeming each to be a file
+	#   else
+	#     raise an exception
 
 	listing = None
-	if protocol in ["http","https"]:
-		if not address.endswith("/"):
-			# http servers can return a 400 error here
-			# if the address doesn't end with a slash.
-			address += "/"
-		page,rc,msg = make_http_request(conn,address,params,headers)
-		
-		if page:
+	scheme = urlparse.urlsplit(baseurl)
+	if scheme == "sftp":
+		conn,protocol,address,params,headers = create_conn(baseurl, conn)
+		listing = conn.listdir(address)
+		conn.close()
+		return listing
+
+	opener = urllib2.build_opener(SimpleListFTPHandler)
+	if not baseurl.endswith("/"):
+		# http servers can return a 400 error here
+		# if the address doesn't end with a slash.
+		# ftp servers can return lists in "dir/file" form
+		# if the address doesn't end with a slash.
+		baseurl += "/"
+	try:
+		f = opener.open(baseurl)
+		resptype = f.info().gettype()
+		if resptype == 'text/html':
 			parser = ParseLinks()
-			parser.feed(_unicode_decode(page))
-			del page
+			parser.feed(_unicode_decode(f.read()))
 			listing = parser.get_anchors()
+		elif resptype == 'text/plain':
+			listing = [x.strip() for x in f.readlines()]
 		else:
-			import portage.exception
 			raise portage.exception.PortageException(
-				_("Unable to get listing: %s %s") % (rc,msg))
-	elif protocol in ["ftp"]:
-		if address[-1] == '/':
-			olddir = conn.pwd()
-			conn.cwd(address)
-			listing = conn.nlst()
-			conn.cwd(olddir)
-			del olddir
-		else:
-			listing = conn.nlst(address)
-	elif protocol == "sftp":
-		listing = conn.listdir(address)
-	else:
-		raise TypeError(_("Unknown protocol. '%s'") % protocol)
-
-	if not keepconnection:
-		conn.close()
+					_("Unable to get listing for type %s") % resptype)
+	except Exception as e:
+		raise portage.exception.PortageException(
+					_("Unable to get listing: %s") % str(e))
 
 	return listing
 
@@ -400,28 +550,26 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000):
 	URI should be in the form <proto>://<site>[:port]<path>
 	Connection is used for persistent connection instances."""
 
-	if not conn:
-		keepconnection = 0
-	else:
-		keepconnection = 1
-
-	conn,protocol,address,params,headers = create_conn(baseurl, conn)
+	try:
+		from cStringIO import StringIO
+	except ImportError:
+		from StringIO import StringIO
+
+	f = urllib2.urlopen(baseurl)
+	sf = StringIO.StringIO(f.read())  # seekable
+	f.close()
+	sf.seek(0, 2)                     # to end-of-file
+	fsize = sf.tell()
+	foffset = chunk_size
+	if foffset > fsize:
+		foffset = fsize
+	data = None
+	try:
+		sf.seek(-foffset, 2)            # from end-of-file
+		data = sf.read()
+	finally:
+		sf.close()
 
-	if protocol in ["http","https"]:
-		headers["Range"] = "bytes=-"+str(chunk_size)
-		data,rc,msg = make_http_request(conn, address, params, headers)
-	elif protocol in ["ftp"]:
-		data,rc,msg = make_ftp_request(conn, address, -chunk_size)
-	elif protocol == "sftp":
-		f = conn.open(address)
-		try:
-			f.seek(-chunk_size, 2)
-			data = f.read()
-		finally:
-			f.close()
-	else:
-		raise TypeError(_("Unknown protocol. '%s'") % protocol)
-	
 	if data:
 		xpaksize = portage.xpak.decodeint(data[-8:-4])
 		if (xpaksize+8) > chunk_size:
@@ -440,9 +588,6 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000):
 	else:
 		myid = None,None
 
-	if not keepconnection:
-		conn.close()
-
 	return myid
 
 
@@ -483,53 +628,33 @@ def file_get_lib(baseurl,dest,conn=None):
 	URI should be in the form <proto>://<site>[:port]<path>
 	Connection is used for persistent connection instances."""
 
-	if not conn:
-		keepconnection = 0
+	req = urllib2.Request(baseurl)
+	sel = req.get_selector()
+	basename = str(os.path.basename(sel))
+	sys.stderr.write("Fetching '"+basename+"'\n")
+	rc = 0
+	try:
+		f = urllib2.urlopen(req)
+	except SystemExit:
+		raise
+	except Exception:
+		rc = 1
 	else:
-		keepconnection = 1
-
-	conn,protocol,address,params,headers = create_conn(baseurl, conn)
-
-	sys.stderr.write("Fetching '"+str(os.path.basename(address)+"'\n"))
-	if protocol in ["http","https"]:
-		data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)
-	elif protocol in ["ftp"]:
-		data,rc,msg = make_ftp_request(conn, address, dest=dest)
-	elif protocol == "sftp":
-		rc = 0
 		try:
-			f = conn.open(address)
-		except SystemExit:
-			raise
-		except Exception:
-			rc = 1
-		else:
-			try:
-				if dest:
-					bufsize = 8192
-					while True:
-						data = f.read(bufsize)
-						if not data:
-							break
-						dest.write(data)
-			finally:
-				f.close()
-	else:
-		raise TypeError(_("Unknown protocol. '%s'") % protocol)
-	
-	if not keepconnection:
-		conn.close()
-
+			if dest:
+				bufsize = 8192
+				while True:
+					data = f.read(bufsize)
+					if not data:
+						break
+					dest.write(data)
+		finally:
+			f.close()
 	return rc
 
-
 def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None):
 	"""(baseurl,conn,chunk_size,verbose) -- 
 	"""
-	if not conn:
-		keepconnection = 0
-	else:
-		keepconnection = 1
 
 	cache_path = "/var/cache/edb"
 	metadatafilename = os.path.join(cache_path, 'remote_metadata.pickle')
@@ -537,14 +662,6 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
 	if makepickle is None:
 		makepickle = "/var/cache/edb/metadata.idx.most_recent"
 
-	try:
-		conn, protocol, address, params, headers = create_conn(baseurl, conn)
-	except _all_errors as e:
-		# ftplib.FTP(host) can raise errors like this:
-		#   socket.error: (111, 'Connection refused')
-		sys.stderr.write("!!! %s\n" % (e,))
-		return {}
-
 	out = sys.stdout
 	try:
 		metadatafile = open(_unicode_encode(metadatafilename,
@@ -703,16 +820,14 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
 					# make_http_request().  The docstring for this error in
 					# httplib.py says "Presumably, the server closed the
 					# connection before sending a valid response".
-					conn, protocol, address, params, headers = create_conn(
-						baseurl)
+					continue
 				except http_client_ResponseNotReady:
 					# With some http servers this error is known to be thrown
 					# from conn.getresponse() in make_http_request() when the
 					# remote file does not have appropriate read permissions.
 					# Maybe it's possible to recover from this exception in
 					# cases though, so retry.
-					conn, protocol, address, params, headers = create_conn(
-						baseurl)
+					continue
 
 			if myid and myid[0]:
 				metadata[baseurl]["data"][x] = make_metadata_dict(myid)
@@ -755,9 +870,6 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
 		sys.stderr.write("!!! "+str(e)+"\n")
 		sys.stderr.flush()
 
-	if not keepconnection:
-		conn.close()
-	
 	return metadata[baseurl]["data"]
 
 def _cmp_cpv(d1, d2):
-- 
1.7.12.3