From cedc71bf6453c4b50fb4d89539eb947d4a7970eb Mon Sep 17 00:00:00 2001 From: James Turk Date: Tue, 21 Feb 2012 17:31:39 -0500 Subject: [PATCH] one time updates --- oyster/core.py | 17 ++++++++++++----- oyster/tests/test_kernel.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/oyster/core.py b/oyster/core.py index 2643a4f..c52d466 100644 --- a/oyster/core.py +++ b/oyster/core.py @@ -193,7 +193,10 @@ class Kernel(object): # last_update/next_update are separate from question of versioning doc['last_update'] = now - doc['next_update'] = now + datetime.timedelta(minutes=update_mins) + if update_mins: + doc['next_update'] = now + datetime.timedelta(minutes=update_mins) + else: + doc['next_update'] = None self.log('update', url=url, new_doc=new_version, error=error) @@ -217,8 +220,10 @@ class Kernel(object): queue = list(new) # pull the rest from those for which next_update is in the past - next = self.db.tracked.find({'next_update': - {'$lt': datetime.datetime.utcnow()}}).sort('_random') + next = self.db.tracked.find({'$and': [ + {'next_update': {'$ne': None}}, + {'next_update': {'$lt': datetime.datetime.utcnow()}}, + ]}).sort('_random') queue.extend(next) return queue @@ -230,8 +235,10 @@ class Kernel(object): ``len(self.get_update_queue())``, but is computed more efficiently. """ new = self.db.tracked.find({'next_update': {'$exists': False}}).count() - next = self.db.tracked.find({'next_update': - {'$lt': datetime.datetime.utcnow()}}).count() + next = self.db.tracked.find({'$and': [ + {'next_update': {'$ne': None}}, + {'next_update': {'$lt': datetime.datetime.utcnow()}}, + ]}).count() return new+next diff --git a/oyster/tests/test_kernel.py b/oyster/tests/test_kernel.py index 8d9c28b..91be5a8 100644 --- a/oyster/tests/test_kernel.py +++ b/oyster/tests/test_kernel.py @@ -23,6 +23,10 @@ class KernelTests(TestCase): {'update_mins': 1/60., 'storage_engine': 'dummy', 'onchanged': [] }, + 'one-time': + {'update_mins': None, 'storage_engine': 'dummy', + 'onchanged': [], + }, 'change-hook': {'update_mins': 30, 'storage_engine': 'dummy', 'onchanged': [hook_fired] @@ -102,6 +106,17 @@ class KernelTests(TestCase): assert self.kernel.db.logs.find_one({'error': 'tracking conflict'}) + def test_no_update(self): + # update + self.kernel.track_url('http://example.com', 'one-time') + obj = self.kernel.db.tracked.find_one() + self.kernel.update(obj) + + newobj = self.kernel.db.tracked.find_one() + assert newobj['next_update'] == None + + assert self.kernel.get_update_queue() == [] + assert self.kernel.get_update_queue_size() == 0 def test_md5_versioning(self): assert not self.kernel.md5_versioning('hello!', 'hello!')