|
Lines 7-13
Link Here
|
| 7 |
__revision__ = '1.3' |
7 |
__revision__ = '1.3' |
| 8 |
|
8 |
|
| 9 |
import sys |
9 |
import sys |
| 10 |
import os, time, popen2, re, shutil, signal, shlex, string |
10 |
import os, time, subprocess, re, shutil, signal, shlex, string, urllib2 |
| 11 |
from HTMLParser import HTMLParser |
11 |
from HTMLParser import HTMLParser |
| 12 |
from optparse import IndentedHelpFormatter, OptionParser |
12 |
from optparse import IndentedHelpFormatter, OptionParser |
| 13 |
|
13 |
|
|
Lines 243-262
Link Here
|
| 243 |
Uses the supplied parser to get a list of urls. |
243 |
Uses the supplied parser to get a list of urls. |
| 244 |
Takes a parser object, url, and filering options. |
244 |
Takes a parser object, url, and filering options. |
| 245 |
""" |
245 |
""" |
| 246 |
|
246 |
|
| 247 |
if fetch == 'wget': |
247 |
output.write('getlist(): fetching with urllib2.urlopen(%s)\n' % url, 2) |
| 248 |
fetch_cmd = 'wget --quiet -O - ' |
|
|
| 249 |
if fetch == 'fetch': |
| 250 |
fetch_cmd = 'fetch -q -o - ' |
| 251 |
if fetch == 'curl': |
| 252 |
fetch_cmd = 'curl -s' |
| 253 |
|
| 254 |
output.write('getlist(): fetching with ' + fetch_cmd + url + '\n', 2) |
| 255 |
|
248 |
|
| 256 |
output.print_info('Downloading a list of mirrors...') |
249 |
output.print_info('Downloading a list of mirrors...') |
| 257 |
|
250 |
|
| 258 |
parser.feed(os.popen(fetch_cmd+url).read()) |
251 |
url = urllib2.urlopen(url) |
|
|
252 |
|
| 253 |
parser.feed(url.read()) |
| 259 |
parser.close() |
254 |
parser.close() |
|
|
255 |
url.close() |
| 260 |
|
256 |
|
| 261 |
if len(parser.lines) == 0: |
257 |
if len(parser.lines) == 0: |
| 262 |
output.print_err('Could not get mirror list. Check your internet' |
258 |
output.print_err('Could not get mirror list. Check your internet' |
|
Lines 297-315
Link Here
|
| 297 |
output.print_info('Using netselect to choose the top %d mirrors...' \ |
293 |
output.print_info('Using netselect to choose the top %d mirrors...' \ |
| 298 |
% number) |
294 |
% number) |
| 299 |
|
295 |
|
| 300 |
host_string = ' '.join(hosts) |
|
|
| 301 |
|
| 302 |
output.write('\nnetselect(): running "netselect -s%d %s"' % (int(number), |
296 |
output.write('\nnetselect(): running "netselect -s%d %s"' % (int(number), |
| 303 |
host_string), 2) |
297 |
' '.join(hosts)), 2) |
| 304 |
|
298 |
|
| 305 |
raw_out, raw_in, raw_err = popen2.popen3('netselect -s%d %s' % \ |
299 |
netselect_sp = subprocess.Popen( ['netselect', '-s%d' % int(number)] + hosts, |
| 306 |
(int(number), host_string)) |
300 |
stdout = subprocess.PIPE, stderr = subprocess.PIPE) |
| 307 |
|
|
|
| 308 |
del raw_in |
| 309 |
|
301 |
|
| 310 |
output.write('netselect(): raw_err.read: %s' % raw_err.read(), 2) |
302 |
(out, err) = netselect_sp.communicate() |
|
|
303 |
|
| 304 |
if err: |
| 305 |
output.write('netselect(): raw_err.read: %s' % err, 2) |
| 311 |
|
306 |
|
| 312 |
for line in raw_out.readlines(): |
307 |
for line in out.split('\n'): |
| 313 |
line = line.split() |
308 |
line = line.split() |
| 314 |
if len(line) < 2: |
309 |
if len(line) < 2: |
| 315 |
continue |
310 |
continue |
|
Lines 467-479
Link Here
|
| 467 |
fetch_cmd = 'fetch -o /dev/null -q %s' % (url) |
462 |
fetch_cmd = 'fetch -o /dev/null -q %s' % (url) |
| 468 |
if fetch == 'curl': |
463 |
if fetch == 'curl': |
| 469 |
fetch_cmd = 'curl -o /dev/null -s --retry 1' % (url) |
464 |
fetch_cmd = 'curl -o /dev/null -s --retry 1' % (url) |
| 470 |
|
465 |
|
|
|
466 |
dev_null = open('/dev/null', 'w') |
| 467 |
null_fd = dev_null.fileno() |
| 471 |
stime = time.time() |
468 |
stime = time.time() |
| 472 |
delta = 0 |
469 |
delta = 0 |
| 473 |
fetch = popen2.Popen4(fetch_cmd) |
470 |
fetch = subprocess.Popen(fetch_cmd, stdout = null_fd, stderr = null_fd, shell = True) |
| 474 |
|
471 |
|
| 475 |
while delta < timeout: #while the timeout has not been exceeded... |
472 |
while delta < timeout: #while the timeout has not been exceeded... |
| 476 |
if fetch.poll() == -1: #if the process is still running... |
473 |
if fetch.poll() is None: #if the process is still running... |
| 477 |
time.sleep(0.001) #avoids a busywait... |
474 |
time.sleep(0.001) #avoids a busywait... |
| 478 |
delta = time.time() - stime #update the elapsed time |
475 |
delta = time.time() - stime #update the elapsed time |
| 479 |
continue #and keep going |
476 |
continue #and keep going |
|
Lines 484-490
Link Here
|
| 484 |
|
481 |
|
| 485 |
if fetch.poll() == -1: #if we timed out, we have to kill wget |
482 |
if fetch.poll() == -1: #if we timed out, we have to kill wget |
| 486 |
output.write('deeptime(): download timed out. killing wget.\n', 2) |
483 |
output.write('deeptime(): download timed out. killing wget.\n', 2) |
| 487 |
os.kill(fetch.pid, 9) |
484 |
fetch.kill() |
| 488 |
|
485 |
|
| 489 |
output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2) |
486 |
output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2) |
| 490 |
|
487 |
|
|
Lines 573-582
Link Here
|
| 573 |
|
570 |
|
| 574 |
dialog += ' ' + ' '.join(['"%s" "%s" "OFF"' % host for host in hosts]) |
571 |
dialog += ' ' + ' '.join(['"%s" "%s" "OFF"' % host for host in hosts]) |
| 575 |
|
572 |
|
| 576 |
mirror_fd = os.popen('%s' % dialog) |
573 |
dialog_sp = subprocess.Popen(dialog, stdout = subprocess.PIPE, shell = True) |
| 577 |
mirrors = mirror_fd.read() |
574 |
mirrors = dialog_sp.communicate()[0] |
| 578 |
mirror_fd.close() |
|
|
| 579 |
|
| 580 |
self.urls = mirrors.split('\n') |
575 |
self.urls = mirrors.split('\n') |
| 581 |
|
576 |
|
| 582 |
|
577 |
|
|
Lines 584-603
Link Here
|
| 584 |
""" |
579 |
""" |
| 585 |
Determines whether a particular binary is available on the host system. |
580 |
Determines whether a particular binary is available on the host system. |
| 586 |
""" |
581 |
""" |
| 587 |
return popen2.Popen4('which %s' % name).wait() == 0 |
582 |
dev_null = open('/dev/null', 'w') |
|
|
583 |
null_fd = dev_null.fileno() |
| 584 |
ret = subprocess.Popen( ('which', name), stdout = null_fd, stderr = null_fd ).wait() |
| 585 |
dev_null.close() |
| 586 |
return (ret == 0) |
| 588 |
|
587 |
|
| 589 |
|
588 |
|
| 590 |
def handler(signum, frame): |
589 |
def handler(signum, frame): |
| 591 |
output.print_err('Caught signal %s. Exiting' % signum) |
590 |
output.print_err('Caught signal %s. Exiting' % signum) |
| 592 |
|
591 |
|
| 593 |
|
592 |
|
|
|
593 |
def _wget_supports_ipv6(): |
| 594 |
""" |
| 595 |
Determines whether wget was compiled with IPv6 support. The --inet6-only option is only |
| 596 |
listed in the output of 'wget --help' if wget supports IPv6 downloads. |
| 597 |
""" |
| 598 |
|
| 599 |
wget_sp = subprocess.Popen( ('wget', '--help'), stdout = subprocess.PIPE) |
| 600 |
(out, err) = wget_sp.communicate() |
| 601 |
return ('IPv6' in out) |
| 602 |
|
| 603 |
|
| 594 |
def _select_fetch(): |
604 |
def _select_fetch(): |
| 595 |
""" |
605 |
""" |
| 596 |
Determines the propper fetch command, and the status of ipv6 support. |
606 |
Determines the propper fetch command, and the status of ipv6 support. |
| 597 |
""" |
607 |
""" |
| 598 |
if _have_bin('wget'): |
608 |
if _have_bin('wget'): |
| 599 |
fetch = 'wget' |
609 |
fetch = 'wget' |
| 600 |
ipv6 = False |
610 |
ipv6 = _wget_supports_ipv6() |
| 601 |
elif _have_bin('fetch'): |
611 |
elif _have_bin('fetch'): |
| 602 |
fetch = 'fetch' |
612 |
fetch = 'fetch' |
| 603 |
ipv6 = True |
613 |
ipv6 = True |
