diff --git a/oyster/core.py b/oyster/core.py index 522b167..194c99a 100644 --- a/oyster/core.py +++ b/oyster/core.py @@ -53,7 +53,8 @@ class Kernel(object): self.storage[name] = StorageCls(self) # set document classes - _doc_class_fields = ('update_mins', 'storage_engine') + _doc_class_fields = ('update_mins', 'storage_engine', + 'onchanged') self.doc_classes = doc_classes or {} for dc_name, dc_props in self.doc_classes.iteritems(): for key in _doc_class_fields: @@ -165,6 +166,9 @@ class Kernel(object): 'storage_key': storage_id, 'storage_type': storage.storage_type, }) + # fire off onchanged functions + for onchanged in doc_class['onchanged']: + onchanged(doc) if error: # if there's been an error, increment the consecutive_errors count diff --git a/oyster/tasks.py b/oyster/tasks.py index f659aad..c14b665 100644 --- a/oyster/tasks.py +++ b/oyster/tasks.py @@ -14,7 +14,7 @@ class UpdateTask(Task): def run(self, doc_id): doc = kernel.db.tracked.find_one({'_id': doc_id}) kernel.update(doc) - for hook in doc.get('post_update_hooks', []): + for task in doc.get('post_update_tasks', []): send_task(hook, (doc_id,)) kernel.db.status.update({}, {'$inc': {'update_queue': -1}}) diff --git a/oyster/tests/test_kernel.py b/oyster/tests/test_kernel.py index 5b27430..7be0642 100644 --- a/oyster/tests/test_kernel.py +++ b/oyster/tests/test_kernel.py @@ -7,14 +7,26 @@ import pymongo from oyster.core import Kernel +def hook_fired(doc): + doc['hook_fired'] = doc.get('hook_fired', 0) + 1 + +RANDOM_URL = 'http://www.random.org/integers/?num=1&min=-1000000000&max=1000000000&col=1&base=10&format=plain&rnd=new' class KernelTests(TestCase): def setUp(self): doc_classes = {'default': - {'update_mins': 30, 'storage_engine': 'dummy'}, + {'update_mins': 30, 'storage_engine': 'dummy', + 'onchanged': [] + }, 'fast-update': - {'update_mins': 1/60., 'storage_engine': 'dummy'}, + {'update_mins': 1/60., 'storage_engine': 'dummy', + 'onchanged': [] + }, + 'change-hook': + {'update_mins': 30, 'storage_engine': 'dummy', + 'onchanged': [hook_fired] + } } self.kernel = Kernel(mongo_db='oyster_test', retry_wait_minutes=1/60., doc_classes=doc_classes) @@ -136,6 +148,33 @@ class KernelTests(TestCase): assert obj['consecutive_errors'] == 2 + def test_update_onchanged_fire_only_on_change(self): + self.kernel.track_url('http://example.com', 'change-hook') + obj = self.kernel.db.tracked.find_one() + self.kernel.update(obj) + + doc = self.kernel.db.tracked.find_one() + assert doc['hook_fired'] == 1 + + # again, we rely on example.com not updating + self.kernel.update(obj) + doc = self.kernel.db.tracked.find_one() + assert doc['hook_fired'] == 1 + + def test_update_onchanged_fire_again_on_change(self): + self.kernel.track_url(RANDOM_URL, 'change-hook') + obj = self.kernel.db.tracked.find_one() + self.kernel.update(obj) + + doc = self.kernel.db.tracked.find_one() + assert doc['hook_fired'] == 1 + + # we rely on this URL updating + self.kernel.update(obj) + doc = self.kernel.db.tracked.find_one() + assert doc['hook_fired'] == 2 + + def test_get_update_queue(self): self.kernel.track_url('never-updates', 'fast-update') self.kernel.track_url('bad-uri', 'fast-update')