jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "restore limit arg"

Browse Source 
This reverts commit d7de2f762c.
This commit is contained in:
James Turk 2011-09-27 18:11:46 -04:00
parent d7de2f762c
commit 7b8eda5143
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
oyster/client.py
View File

@ -171,21 +171,18 @@ class Client(object):
return self.fs.get_version(url, n) return self.fs.get_version(url, n)
def get_update_queue(self, limit=5000): def get_update_queue(self):
# results are always sorted by random to avoid piling on single server # results are always sorted by random to avoid piling on single server
# first we try to update anything that we've never retrieved # first we try to update anything that we've never retrieved
new = self.db.tracked.find({'next_update': new = self.db.tracked.find({'next_update':
{'$exists': False}}).sort('_random') {'$exists': False}}).sort('_random')
queue = list(new.limit(limit)) queue = list(new)
# shorten limit by number returned
limit -= len(queue)
# pull the rest from those for which next_update is in the past # pull the rest from those for which next_update is in the past
next = self.db.tracked.find({'next_update': next = self.db.tracked.find({'next_update':
{'$lt': datetime.datetime.utcnow()}}).sort('_random') {'$lt': datetime.datetime.utcnow()}}).sort('_random')
queue.extend(next.limit(limit)) queue.extend(next)
return queue return queue