Lines 266-276
Link Here
|
266 |
else: |
266 |
else: |
267 |
keepconnection = 1 |
267 |
keepconnection = 1 |
268 |
|
268 |
|
269 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
269 |
conn,protocol,address,params,headers = create_conn(baseurl, conn=conn) |
270 |
|
270 |
|
271 |
listing = None |
271 |
listing = None |
272 |
if protocol in ["http","https"]: |
272 |
if protocol in ["http","https"]: |
273 |
page,rc,msg = make_http_request(conn,address,params,headers) |
273 |
page,rc,msg = make_http_request(conn,address,params=params,headers=headers) |
274 |
|
274 |
|
275 |
if page: |
275 |
if page: |
276 |
parser = ParseLinks() |
276 |
parser = ParseLinks() |
Lines 306-316
Link Here
|
306 |
else: |
306 |
else: |
307 |
keepconnection = 1 |
307 |
keepconnection = 1 |
308 |
|
308 |
|
309 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
309 |
conn,protocol,address,params,headers = create_conn(baseurl, conn=conn) |
310 |
|
310 |
|
311 |
if protocol in ["http","https"]: |
311 |
if protocol in ["http","https"]: |
312 |
headers["Range"] = "bytes=-"+str(chunk_size) |
312 |
headers["Range"] = "bytes=-"+str(chunk_size) |
313 |
data,rc,msg = make_http_request(conn, address, params, headers) |
313 |
data,rc,msg = make_http_request(conn, address, params=params, headers=headers) |
314 |
elif protocol in ["ftp"]: |
314 |
elif protocol in ["ftp"]: |
315 |
data,rc,msg = make_ftp_request(conn, address, -chunk_size) |
315 |
data,rc,msg = make_ftp_request(conn, address, -chunk_size) |
316 |
else: |
316 |
else: |
Lines 319-325
Link Here
|
319 |
if data: |
319 |
if data: |
320 |
xpaksize = xpak.decodeint(data[-8:-4]) |
320 |
xpaksize = xpak.decodeint(data[-8:-4]) |
321 |
if (xpaksize+8) > chunk_size: |
321 |
if (xpaksize+8) > chunk_size: |
322 |
myid = file_get_metadata(baseurl, conn, (xpaksize+8)) |
322 |
myid = file_get_metadata(baseurl, conn=conn, chunk_size=(xpaksize+8)) |
323 |
if not keepconnection: |
323 |
if not keepconnection: |
324 |
conn.close() |
324 |
conn.close() |
325 |
return myid |
325 |
return myid |
Lines 345-351
Link Here
|
345 |
URL should be in the for <proto>://[user[:pass]@]<site>[:port]<path>""" |
345 |
URL should be in the for <proto>://[user[:pass]@]<site>[:port]<path>""" |
346 |
|
346 |
|
347 |
if not fcmd: |
347 |
if not fcmd: |
348 |
return file_get_lib(baseurl,dest,conn) |
348 |
return file_get_lib(baseurl,dest,conn=conn) |
349 |
|
349 |
|
350 |
fcmd = string.replace(fcmd, "${DISTDIR}", dest) |
350 |
fcmd = string.replace(fcmd, "${DISTDIR}", dest) |
351 |
fcmd = string.replace(fcmd, "${URI}", baseurl) |
351 |
fcmd = string.replace(fcmd, "${URI}", baseurl) |
Lines 378-388
Link Here
|
378 |
else: |
378 |
else: |
379 |
keepconnection = 1 |
379 |
keepconnection = 1 |
380 |
|
380 |
|
381 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
381 |
conn,protocol,address,params,headers = create_conn(baseurl, conn=conn) |
382 |
|
382 |
|
383 |
sys.stderr.write("Fetching '"+str(os.path.basename(address)+"'\n")) |
383 |
sys.stderr.write("Fetching '"+str(os.path.basename(address)+"'\n")) |
384 |
if protocol in ["http","https"]: |
384 |
if protocol in ["http","https"]: |
385 |
data,rc,msg = make_http_request(conn, address, params, headers, dest=dest) |
385 |
data,rc,msg = make_http_request(conn, address, params=params, headers=headers, dest=dest) |
386 |
elif protocol in ["ftp"]: |
386 |
elif protocol in ["ftp"]: |
387 |
data,rc,msg = make_ftp_request(conn, address, dest=dest) |
387 |
data,rc,msg = make_ftp_request(conn, address, dest=dest) |
388 |
else: |
388 |
else: |
Lines 405-411
Link Here
|
405 |
if makepickle == None: |
405 |
if makepickle == None: |
406 |
makepickle = "/var/cache/edb/metadata.idx.most_recent" |
406 |
makepickle = "/var/cache/edb/metadata.idx.most_recent" |
407 |
|
407 |
|
408 |
conn,protocol,address,params,headers = create_conn(baseurl, conn) |
408 |
conn,protocol,address,params,headers = create_conn(baseurl, conn=conn) |
409 |
|
409 |
|
410 |
filedict = {} |
410 |
filedict = {} |
411 |
|
411 |
|
Lines 427-433
Link Here
|
427 |
if not metadata[baseurl].has_key("data"): |
427 |
if not metadata[baseurl].has_key("data"): |
428 |
metadata[baseurl]["data"]={} |
428 |
metadata[baseurl]["data"]={} |
429 |
|
429 |
|
430 |
filelist = dir_get_list(baseurl, conn) |
430 |
filelist = dir_get_list(baseurl, conn=conn) |
431 |
tbz2list = match_in_array(filelist, suffix=".tbz2") |
431 |
tbz2list = match_in_array(filelist, suffix=".tbz2") |
432 |
metalist = match_in_array(filelist, prefix="metadata.idx") |
432 |
metalist = match_in_array(filelist, prefix="metadata.idx") |
433 |
del filelist |
433 |
del filelist |
Lines 445-451
Link Here
|
445 |
for trynum in [1,2,3]: |
445 |
for trynum in [1,2,3]: |
446 |
mytempfile = tempfile.TemporaryFile() |
446 |
mytempfile = tempfile.TemporaryFile() |
447 |
try: |
447 |
try: |
448 |
file_get(baseurl+"/"+mfile, mytempfile, conn) |
448 |
file_get(baseurl+"/"+mfile, mytempfile, conn=conn) |
449 |
if mytempfile.tell() > len(data): |
449 |
if mytempfile.tell() > len(data): |
450 |
mytempfile.seek(0) |
450 |
mytempfile.seek(0) |
451 |
data = mytempfile.read() |
451 |
data = mytempfile.read() |
Lines 493-499
Link Here
|
493 |
(x not in metadata[baseurl]["data"].keys())): |
493 |
(x not in metadata[baseurl]["data"].keys())): |
494 |
sys.stderr.write(yellow("x")) |
494 |
sys.stderr.write(yellow("x")) |
495 |
metadata[baseurl]["modified"] = 1 |
495 |
metadata[baseurl]["modified"] = 1 |
496 |
myid = file_get_metadata(baseurl+"/"+x, conn, chunk_size) |
496 |
myid = file_get_metadata(baseurl+"/"+x, conn=conn, chunk_size=chunk_size) |
497 |
|
497 |
|
498 |
if myid[0]: |
498 |
if myid[0]: |
499 |
metadata[baseurl]["data"][x] = make_metadata_dict(myid) |
499 |
metadata[baseurl]["data"][x] = make_metadata_dict(myid) |