Загрузка файлов в витой части с использованием очереди

from twisted.internet.defer import inlineCallbacks, DeferredQueue @inlineCallbacks def worker(queue): while 1: url = yield queue.get() # wait for a url from the queue if url is None: # insert None into the queue to kill workers queue.put(None) return # done data = yield download(url) # download the file process(data) # do stuff with it queue = DeferredQueue() # your queue # make workers MAX = 20 workers = [worker(queue) for _ in range(MAX)] 

Вот перевод https://github.com/caolan/async на Python.

 from twisted.internet import defer class Queue: workers = 0 tasks = [] def __init__(self, worker, concurrency): self.worker = worker self.concurrency = concurrency self.saturated = None self.empty = None self.drain = None def push(self, data): deferred = defer.Deferred() self.tasks.append({'data': data, 'callback': deferred}) if self.saturated and len(tasks) == concurrency: self.saturated() self.process() return deferred def task_finished(self, *args): self.workers = self.workers - 1 if self.drain and len(self.tasks) + self.workers == 0: self.drain() self.process() def process(self): if self.workers >= self.concurrency or len(self.tasks) == 0: return task = self.tasks.pop(0) if self.empty and len(self.tasks) == 0: self.empty() self.workers = self.workers + 1 d = self.worker(task['data']) d.addCallback(self.task_finished) d.addCallback(task['callback'].callback) from twisted.web import client from twisted.internet import reactor def dl_worker(data): url = data[0] fname = data[1] print "Download file:", fname d = client.downloadPage(url, fname) return d # very important! q = Queue(dl_worker, 2) q.drain = reactor.stop for i in range(0,3): q.push(["http://download.thinkbroadband.com/5MB.zip", "file"+str(i)]) reactor.run() 

Надеюсь, это пройдет QC Glyph: D Cheers!

Используйте модуль select чтобы сделать это с помощью опроса, или с помощью модуля нитей с потоками.