jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

one time updates

Browse Source
This commit is contained in:
James Turk 2012-02-21 17:31:39 -05:00
parent 2f3825d3e2
commit cedc71bf64
2 changed files with 27 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
oyster/core.py
View File

@ -193,7 +193,10 @@ class Kernel(object):
# last_update/next_update are separate from question of versioning # last_update/next_update are separate from question of versioning
doc['last_update'] = now doc['last_update'] = now
if update_mins:
doc['next_update'] = now + datetime.timedelta(minutes=update_mins) doc['next_update'] = now + datetime.timedelta(minutes=update_mins)
else:
doc['next_update'] = None
self.log('update', url=url, new_doc=new_version, error=error) self.log('update', url=url, new_doc=new_version, error=error)
@ -217,8 +220,10 @@ class Kernel(object):
queue = list(new) queue = list(new)
# pull the rest from those for which next_update is in the past # pull the rest from those for which next_update is in the past
next = self.db.tracked.find({'next_update': next = self.db.tracked.find({'$and': [
{'$lt': datetime.datetime.utcnow()}}).sort('_random') {'next_update': {'$ne': None}},
{'next_update': {'$lt': datetime.datetime.utcnow()}},
]}).sort('_random')
queue.extend(next) queue.extend(next)
return queue return queue
@ -230,8 +235,10 @@ class Kernel(object):
``len(self.get_update_queue())``, but is computed more efficiently. ``len(self.get_update_queue())``, but is computed more efficiently.
""" """
new = self.db.tracked.find({'next_update': {'$exists': False}}).count() new = self.db.tracked.find({'next_update': {'$exists': False}}).count()
next = self.db.tracked.find({'next_update': next = self.db.tracked.find({'$and': [
{'$lt': datetime.datetime.utcnow()}}).count() {'next_update': {'$ne': None}},
{'next_update': {'$lt': datetime.datetime.utcnow()}},
]}).count()
return new+next return new+next

15
oyster/tests/test_kernel.py
View File

@ -23,6 +23,10 @@ class KernelTests(TestCase):
{'update_mins': 1/60., 'storage_engine': 'dummy', {'update_mins': 1/60., 'storage_engine': 'dummy',
'onchanged': [] 'onchanged': []
}, },
'one-time':
{'update_mins': None, 'storage_engine': 'dummy',
'onchanged': [],
},
'change-hook': 'change-hook':
{'update_mins': 30, 'storage_engine': 'dummy', {'update_mins': 30, 'storage_engine': 'dummy',
'onchanged': [hook_fired] 'onchanged': [hook_fired]
@ -102,6 +106,17 @@ class KernelTests(TestCase):
assert self.kernel.db.logs.find_one({'error': 'tracking conflict'}) assert self.kernel.db.logs.find_one({'error': 'tracking conflict'})
def test_no_update(self):
# update
self.kernel.track_url('http://example.com', 'one-time')
obj = self.kernel.db.tracked.find_one()
self.kernel.update(obj)
newobj = self.kernel.db.tracked.find_one()
assert newobj['next_update'] == None
assert self.kernel.get_update_queue() == []
assert self.kernel.get_update_queue_size() == 0
def test_md5_versioning(self): def test_md5_versioning(self):
assert not self.kernel.md5_versioning('hello!', 'hello!') assert not self.kernel.md5_versioning('hello!', 'hello!')