Lines 8-13
import multiprocessing
Link Here
|
8 |
import os.path |
8 |
import os.path |
9 |
import sys |
9 |
import sys |
10 |
|
10 |
|
|
|
11 |
try: |
12 |
from concurrent.futures import ProcessPoolExecutor |
13 |
except ImportError: |
14 |
ProcessPoolExecutor = None |
15 |
|
11 |
import gemato.compression |
16 |
import gemato.compression |
12 |
import gemato.exceptions |
17 |
import gemato.exceptions |
13 |
import gemato.manifest |
18 |
import gemato.manifest |
Lines 16-21
import gemato.util
Link Here
|
16 |
import gemato.verify |
21 |
import gemato.verify |
17 |
|
22 |
|
18 |
|
23 |
|
|
|
24 |
class Pool(object): |
25 |
def __init__(self, processes=None): |
26 |
# map supports chunksize in python3.5+ |
27 |
self._executor = sys.version_info >= (3, 5) |
28 |
if self._executor: |
29 |
self._pool = ProcessPoolExecutor(max_workers=processes) |
30 |
self.imap_unordered = self._pool.map |
31 |
else: |
32 |
self._pool = multiprocessing.Pool(processes=processes) |
33 |
self.imap_unordered = self._pool.imap_unordered |
34 |
|
35 |
self.map = self._pool.map |
36 |
|
37 |
def close(self): |
38 |
if self._executor: |
39 |
self._pool.shutdown() |
40 |
else: |
41 |
self._pool.close() |
42 |
|
43 |
def join(self): |
44 |
if self._executor: |
45 |
self._pool.shutdown(wait=True) |
46 |
else: |
47 |
self._pool.join() |
48 |
|
49 |
def terminate(self): |
50 |
if self._executor: |
51 |
self._pool.shutdown(wait=True) |
52 |
else: |
53 |
self._pool.terminate() |
54 |
|
55 |
|
19 |
class ManifestLoader(object): |
56 |
class ManifestLoader(object): |
20 |
""" |
57 |
""" |
21 |
Helper class to load Manifests in subprocesses. |
58 |
Helper class to load Manifests in subprocesses. |
Lines 382-388
class ManifestRecursiveLoader(object):
Link Here
|
382 |
unconditionally of whether they match parent checksums. |
419 |
unconditionally of whether they match parent checksums. |
383 |
""" |
420 |
""" |
384 |
|
421 |
|
385 |
pool = multiprocessing.Pool(processes=self.max_jobs) |
422 |
pool = Pool(processes=self.max_jobs) |
386 |
|
423 |
|
387 |
try: |
424 |
try: |
388 |
# TODO: figure out how to avoid confusing uses of 'recursive' |
425 |
# TODO: figure out how to avoid confusing uses of 'recursive' |
Lines 657-663
class ManifestRecursiveLoader(object):
Link Here
|
657 |
self.manifest_device, |
694 |
self.manifest_device, |
658 |
fail_handler, last_mtime) |
695 |
fail_handler, last_mtime) |
659 |
|
696 |
|
660 |
pool = multiprocessing.Pool(processes=self.max_jobs) |
697 |
pool = Pool(processes=self.max_jobs) |
661 |
|
698 |
|
662 |
try: |
699 |
try: |
663 |
# verify the directories in parallel |
700 |
# verify the directories in parallel |