Как запустить os.walk параллельно в Python?

Вот многопроцессорное решение:

 from multiprocessing.pool import Pool from multiprocessing import JoinableQueue as Queue import os def explore_path(path): directories = [] nondirectories = [] for filename in os.listdir(path): fullname = os.path.join(path, filename) if os.path.isdir(fullname): directories.append(fullname) else: nondirectories.append(filename) outputfile = path.replace(os.sep, '_') + '.txt' with open(outputfile, 'w') as f: for filename in nondirectories: print >> f, filename return directories def parallel_worker(): while True: path = unsearched.get() dirs = explore_path(path) for newdir in dirs: unsearched.put(newdir) unsearched.task_done() # acquire the list of paths with open('paths.txt') as f: paths = f.split() unsearched = Queue() for path in paths: unsearched.put(path) pool = Pool(5) for i in range(5): pool.apply_async(parallel_worker) unsearched.join() print 'Done' 

Это шаблон для потоков в python, который мне был полезен. Я не уверен, что потоки будут увеличивать вашу производительность из-за того, что потоки работают в CPython.

 import threading import Queue import os class PathThread (threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue def printfiles(self, p): for path, dirs, files in os.walk(p): for f in files: print path + "/" + f def run(self): while True: path = self.queue.get() self.printfiles(path) self.queue.task_done() # threadsafe queue pathqueue = Queue.Queue() paths = ["foo", "bar", "baz"] # spawn threads for i in range(0, 5): t = PathThread(pathqueue) t.setDaemon(True) t.start() # add paths to queue for path in paths: pathqueue.put(path) # wait for queue to get empty pathqueue.join()