|
Lines 111-121
Link Here
|
| 111 |
), |
111 |
), |
| 112 |
} |
112 |
} |
| 113 |
|
113 |
|
|
|
114 |
port = None |
| 115 |
proxy_host = os.environ.get("http_proxy") |
| 116 |
if proxy_host: |
| 117 |
# Do not send user/password here |
| 118 |
address = protocol + "://" + host + address |
| 119 |
hostport = string.split(proxy_host, ":") |
| 120 |
if len(hostport) > 2: |
| 121 |
raise ValueError, "Unable to interpret proxy:port from http_proxy." |
| 122 |
elif len(hostport) == 2: |
| 123 |
host = hostport[0] |
| 124 |
port = hostport[1] |
| 125 |
elif len(hostport) == 1: |
| 126 |
host = hostport[0] |
| 127 |
del hostport, proxy_host |
| 128 |
|
| 114 |
if not conn: |
129 |
if not conn: |
| 115 |
if protocol == "https": |
130 |
if protocol == "https": |
| 116 |
conn = httplib.HTTPSConnection(host) |
131 |
conn = httplib.HTTPSConnection(host, port) |
| 117 |
elif protocol == "http": |
132 |
elif protocol == "http": |
| 118 |
conn = httplib.HTTPConnection(host) |
133 |
conn = httplib.HTTPConnection(host, port) |
| 119 |
elif protocol == "ftp": |
134 |
elif protocol == "ftp": |
| 120 |
passive = 1 |
135 |
passive = 1 |
| 121 |
if(host[-1] == "*"): |
136 |
if(host[-1] == "*"): |
|
Lines 181-187
Link Here
|
| 181 |
return None,int(str(e)[:4]),str(e) |
196 |
return None,int(str(e)[:4]),str(e) |
| 182 |
|
197 |
|
| 183 |
|
198 |
|
| 184 |
def make_http_request(conn, address, params={}, headers={}, dest=None): |
199 |
def make_http_request(conn, address, params={}, headers={}, dest=None, method="GET"): |
| 185 |
"""(conn,address,params,headers) --- uses the conn object to request |
200 |
"""(conn,address,params,headers) --- uses the conn object to request |
| 186 |
the data from address, performing Location forwarding and using the |
201 |
the data from address, performing Location forwarding and using the |
| 187 |
optional params and headers.""" |
202 |
optional params and headers.""" |
|
Lines 192-198
Link Here
|
| 192 |
try: |
207 |
try: |
| 193 |
if (rc != 0): |
208 |
if (rc != 0): |
| 194 |
conn,ignore,ignore,ignore,ignore = create_conn(address) |
209 |
conn,ignore,ignore,ignore,ignore = create_conn(address) |
| 195 |
conn.request("GET", address, params, headers) |
210 |
conn.request(method, address, params, headers) |
| 196 |
except SystemExit, e: |
211 |
except SystemExit, e: |
| 197 |
raise |
212 |
raise |
| 198 |
except Exception, e: |
213 |
except Exception, e: |
|
Lines 224-229
Link Here
|
| 224 |
dest.write(response.read()) |
239 |
dest.write(response.read()) |
| 225 |
return "",0,"" |
240 |
return "",0,"" |
| 226 |
|
241 |
|
|
|
242 |
if method != "GET": |
| 243 |
return response,0,"" |
| 227 |
return response.read(),0,"" |
244 |
return response.read(),0,"" |
| 228 |
|
245 |
|
| 229 |
|
246 |
|
|
Lines 303-309
Link Here
|
| 303 |
|
320 |
|
| 304 |
return listing |
321 |
return listing |
| 305 |
|
322 |
|
| 306 |
def file_get_metadata(baseurl,conn=None, chunk_size=3000): |
323 |
def file_get_metadata(baseurl,conn=None, chunk_size=3000, file_size=-1): |
| 307 |
"""(baseurl[,connection]) -- Takes a base url to connect to and read from. |
324 |
"""(baseurl[,connection]) -- Takes a base url to connect to and read from. |
| 308 |
URL should be in the for <proto>://<site>[:port]<path> |
325 |
URL should be in the for <proto>://<site>[:port]<path> |
| 309 |
Connection is used for persistent connection instances.""" |
326 |
Connection is used for persistent connection instances.""" |
|
Lines 316-323
Link Here
|
| 316 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
333 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
| 317 |
|
334 |
|
| 318 |
if protocol in ["http","https"]: |
335 |
if protocol in ["http","https"]: |
| 319 |
headers["Range"] = "bytes=-"+str(chunk_size) |
336 |
if file_size==-1: |
| 320 |
data,rc,msg = make_http_request(conn, address, params, headers) |
337 |
data,rc,msg = make_http_request(conn, address, params, headers, method="HEAD") |
|
|
338 |
if data: |
| 339 |
file_size = int(data.getheader("content-length")) |
| 340 |
del data |
| 341 |
# If it's still -1, we were unable to get file_size, fallback to old method |
| 342 |
if file_size==-1: |
| 343 |
headers["Range"] = "bytes=-"+str(chunk_size) |
| 344 |
data,rc,msg = make_http_request(conn, address, params, headers) |
| 345 |
else: |
| 346 |
headers["Range"] = "bytes="+str(file_size-chunk_size)+"-" |
| 347 |
data,rc,msg = make_http_request(conn, address, params, headers) |
| 348 |
|
| 321 |
elif protocol in ["ftp"]: |
349 |
elif protocol in ["ftp"]: |
| 322 |
data,rc,msg = make_ftp_request(conn, address, -chunk_size) |
350 |
data,rc,msg = make_ftp_request(conn, address, -chunk_size) |
| 323 |
else: |
351 |
else: |
|
Lines 326-332
Link Here
|
| 326 |
if data: |
354 |
if data: |
| 327 |
xpaksize = xpak.decodeint(data[-8:-4]) |
355 |
xpaksize = xpak.decodeint(data[-8:-4]) |
| 328 |
if (xpaksize+8) > chunk_size: |
356 |
if (xpaksize+8) > chunk_size: |
| 329 |
myid = file_get_metadata(baseurl, conn, (xpaksize+8)) |
357 |
myid = file_get_metadata(baseurl, conn, (xpaksize+8), file_size) |
| 330 |
if not keepconnection: |
358 |
if not keepconnection: |
| 331 |
conn.close() |
359 |
conn.close() |
| 332 |
return myid |
360 |
return myid |
