|
Lines 6-11
Link Here
|
| 6 |
import multiprocessing |
6 |
import multiprocessing |
| 7 |
import sys |
7 |
import sys |
| 8 |
|
8 |
|
|
|
9 |
try: |
| 10 |
from concurrent.futures import ProcessPoolExecutor |
| 11 |
except ImportError: |
| 12 |
ProcessPoolExecutor = None |
| 13 |
|
| 9 |
|
14 |
|
| 10 |
class MultiprocessingPoolWrapper(object): |
15 |
class MultiprocessingPoolWrapper(object): |
| 11 |
""" |
16 |
""" |
|
Lines 16-31
class MultiprocessingPoolWrapper(object):
Link Here
|
| 16 |
__slots__ = ['pool'] |
21 |
__slots__ = ['pool'] |
| 17 |
|
22 |
|
| 18 |
def __init__(self, processes): |
23 |
def __init__(self, processes): |
| 19 |
self.pool = multiprocessing.Pool(processes=processes) |
24 |
# ProcessPoolExecutor.map supports chunksize in python3.5+, |
|
|
25 |
# use it as a possible workaround for imap_unordered deadlock |
| 26 |
# reported in https://bugs.gentoo.org/647964. |
| 27 |
if sys.version_info >= (3, 5): |
| 28 |
self.pool = ProcessPoolExecutor(max_workers=processes) |
| 29 |
else: |
| 30 |
self.pool = multiprocessing.Pool(processes=processes) |
| 20 |
|
31 |
|
| 21 |
def __enter__(self): |
32 |
def __enter__(self): |
| 22 |
return self |
33 |
return self |
| 23 |
|
34 |
|
| 24 |
def __exit__(self, exc_type, exc_value, exc_cb): |
35 |
def __exit__(self, exc_type, exc_value, exc_cb): |
| 25 |
if exc_type is None: |
36 |
if hasattr(self.pool, 'shutdown'): |
| 26 |
self.pool.close() |
37 |
self.pool.shutdown() |
| 27 |
self.pool.join() |
38 |
else: |
| 28 |
self.pool.terminate() |
39 |
if exc_type is None: |
|
|
40 |
self.pool.close() |
| 41 |
self.pool.join() |
| 42 |
self.pool.terminate() |
| 29 |
|
43 |
|
| 30 |
def map(self, *args, **kwargs): |
44 |
def map(self, *args, **kwargs): |
| 31 |
return self.pool.map(*args, **kwargs) |
45 |
return self.pool.map(*args, **kwargs) |
|
Lines 35-46
class MultiprocessingPoolWrapper(object):
Link Here
|
| 35 |
Use imap_unordered() if available and safe to use. Fall back |
49 |
Use imap_unordered() if available and safe to use. Fall back |
| 36 |
to regular map() otherwise. |
50 |
to regular map() otherwise. |
| 37 |
""" |
51 |
""" |
| 38 |
if sys.hexversion >= 0x03050400: |
52 |
return self.pool.map(*args, **kwargs) |
| 39 |
return self.pool.imap_unordered(*args, **kwargs) |
|
|
| 40 |
else: |
| 41 |
# in py<3.5.4 imap() swallows exceptions, so fall back |
| 42 |
# to regular map() |
| 43 |
return self.pool.map(*args, **kwargs) |
| 44 |
|
53 |
|
| 45 |
|
54 |
|
| 46 |
def path_starts_with(path, prefix): |
55 |
def path_starts_with(path, prefix): |
| 47 |
- |
|
|
