jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restore limit arg

Browse Source
This commit is contained in:
James Turk 2011-09-27 18:09:29 -04:00
parent d1df737474
commit d7de2f762c
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
oyster/client.py
View File

@ -171,18 +171,21 @@ class Client(object):
return self.fs.get_version(url, n) return self.fs.get_version(url, n)
def get_update_queue(self): def get_update_queue(self, limit=5000):
# results are always sorted by random to avoid piling on single server # results are always sorted by random to avoid piling on single server
# first we try to update anything that we've never retrieved # first we try to update anything that we've never retrieved
new = self.db.tracked.find({'next_update': new = self.db.tracked.find({'next_update':
{'$exists': False}}).sort('_random') {'$exists': False}}).sort('_random')
queue = list(new) queue = list(new.limit(limit))
# shorten limit by number returned
limit -= len(queue)
# pull the rest from those for which next_update is in the past # pull the rest from those for which next_update is in the past
next = self.db.tracked.find({'next_update': next = self.db.tracked.find({'next_update':
{'$lt': datetime.datetime.utcnow()}}).sort('_random') {'$lt': datetime.datetime.utcnow()}}).sort('_random')
queue.extend(next) queue.extend(next.limit(limit))
return queue return queue