|
Lines 8-13
import multiprocessing
Link Here
|
| 8 |
import os.path |
8 |
import os.path |
| 9 |
import sys |
9 |
import sys |
| 10 |
|
10 |
|
|
|
11 |
try: |
| 12 |
from concurrent.futures import ProcessPoolExecutor |
| 13 |
except ImportError: |
| 14 |
ProcessPoolExecutor = None |
| 15 |
|
| 11 |
import gemato.compression |
16 |
import gemato.compression |
| 12 |
import gemato.exceptions |
17 |
import gemato.exceptions |
| 13 |
import gemato.manifest |
18 |
import gemato.manifest |
|
Lines 16-21
import gemato.util
Link Here
|
| 16 |
import gemato.verify |
21 |
import gemato.verify |
| 17 |
|
22 |
|
| 18 |
|
23 |
|
|
|
24 |
class Pool(object): |
| 25 |
def __init__(self, processes=None): |
| 26 |
# map supports chunksize in python3.5+ |
| 27 |
self._executor = sys.version_info >= (3, 5) |
| 28 |
if self._executor: |
| 29 |
self._pool = ProcessPoolExecutor(max_workers=processes) |
| 30 |
self.imap_unordered = self._pool.map |
| 31 |
else: |
| 32 |
self._pool = multiprocessing.Pool(processes=processes) |
| 33 |
self.imap_unordered = self._pool.imap_unordered |
| 34 |
|
| 35 |
self.map = self._pool.map |
| 36 |
|
| 37 |
def close(self): |
| 38 |
if self._executor: |
| 39 |
self._pool.shutdown() |
| 40 |
else: |
| 41 |
self._pool.close() |
| 42 |
|
| 43 |
def join(self): |
| 44 |
if self._executor: |
| 45 |
self._pool.shutdown(wait=True) |
| 46 |
else: |
| 47 |
self._pool.join() |
| 48 |
|
| 49 |
def terminate(self): |
| 50 |
if self._executor: |
| 51 |
self._pool.shutdown(wait=True) |
| 52 |
else: |
| 53 |
self._pool.terminate() |
| 54 |
|
| 55 |
|
| 19 |
class ManifestLoader(object): |
56 |
class ManifestLoader(object): |
| 20 |
""" |
57 |
""" |
| 21 |
Helper class to load Manifests in subprocesses. |
58 |
Helper class to load Manifests in subprocesses. |
|
Lines 382-388
class ManifestRecursiveLoader(object):
Link Here
|
| 382 |
unconditionally of whether they match parent checksums. |
419 |
unconditionally of whether they match parent checksums. |
| 383 |
""" |
420 |
""" |
| 384 |
|
421 |
|
| 385 |
pool = multiprocessing.Pool(processes=self.max_jobs) |
422 |
pool = Pool(processes=self.max_jobs) |
| 386 |
|
423 |
|
| 387 |
try: |
424 |
try: |
| 388 |
# TODO: figure out how to avoid confusing uses of 'recursive' |
425 |
# TODO: figure out how to avoid confusing uses of 'recursive' |
|
Lines 657-663
class ManifestRecursiveLoader(object):
Link Here
|
| 657 |
self.manifest_device, |
694 |
self.manifest_device, |
| 658 |
fail_handler, last_mtime) |
695 |
fail_handler, last_mtime) |
| 659 |
|
696 |
|
| 660 |
pool = multiprocessing.Pool(processes=self.max_jobs) |
697 |
pool = Pool(processes=self.max_jobs) |
| 661 |
|
698 |
|
| 662 |
try: |
699 |
try: |
| 663 |
# verify the directories in parallel |
700 |
# verify the directories in parallel |
