diff --git a/oyster/client.py b/oyster/client.py index 60f6dde..60c6cae 100644 --- a/oyster/client.py +++ b/oyster/client.py @@ -171,18 +171,21 @@ class Client(object): return self.fs.get_version(url, n) - def get_update_queue(self): + def get_update_queue(self, limit=5000): # results are always sorted by random to avoid piling on single server # first we try to update anything that we've never retrieved new = self.db.tracked.find({'next_update': {'$exists': False}}).sort('_random') - queue = list(new) + queue = list(new.limit(limit)) + + # shorten limit by number returned + limit -= len(queue) # pull the rest from those for which next_update is in the past next = self.db.tracked.find({'next_update': {'$lt': datetime.datetime.utcnow()}}).sort('_random') - queue.extend(next) + queue.extend(next.limit(limit)) return queue