jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change how onchanged events fire

Browse Source
This commit is contained in:
James Turk 2012-04-13 17:41:07 -04:00
parent 232cf76a60
commit 7922ac2da7
3 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
oyster/core.py
View File

@ -7,6 +7,7 @@ import pymongo
import scrapelib import scrapelib
from .storage import engines from .storage import engines
from celery.execute import send_task
class Kernel(object): class Kernel(object):
@ -187,8 +188,8 @@ class Kernel(object):
'storage_type': storage.storage_type, 'storage_type': storage.storage_type,
}) })
# fire off onchanged functions # fire off onchanged functions
for onchanged in doc_class['onchanged']: for onchanged in doc_class.get('onchanged', []):
onchanged(doc, newdata) send_task(onchanged, (doc['_id'],))
if error: if error:
# if there's been an error, increment the consecutive_errors count # if there's been an error, increment the consecutive_errors count

5
oyster/tasks.py
View File

@ -11,11 +11,8 @@ class UpdateTask(Task):
def run(self, doc_id): def run(self, doc_id):
doc = kernel.db.tracked.find_one({'_id': doc_id}) doc = kernel.db.tracked.find_one({'_id': doc_id})
kernel.update(doc)
for task in kernel.doc_classes[doc['doc_class']].get(
'post_update_tasks', []):
send_task(task, (doc_id,))
kernel.db.status.update({}, {'$inc': {'update_queue': -1}}) kernel.db.status.update({}, {'$inc': {'update_queue': -1}})
kernel.update(doc)
# don't sit on a connection # don't sit on a connection
kernel.db.connection.end_request() kernel.db.connection.end_request()

40
oyster/tests/test_kernel.py
View File

@ -170,31 +170,31 @@ class KernelTests(TestCase):
obj = self.kernel.db.tracked.find_one() obj = self.kernel.db.tracked.find_one()
assert obj['consecutive_errors'] == 2 assert obj['consecutive_errors'] == 2
def test_update_onchanged_fire_only_on_change(self): #def test_update_onchanged_fire_only_on_change(self):
self.kernel.track_url('http://example.com', 'change-hook') # self.kernel.track_url('http://example.com', 'change-hook')
obj = self.kernel.db.tracked.find_one() # obj = self.kernel.db.tracked.find_one()
self.kernel.update(obj) # self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one() # doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1 # assert doc['hook_fired'] == 1
# again, we rely on example.com not updating # # again, we rely on example.com not updating
self.kernel.update(obj) # self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one() # doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1 # assert doc['hook_fired'] == 1
def test_update_onchanged_fire_again_on_change(self): #def test_update_onchanged_fire_again_on_change(self):
self.kernel.track_url(RANDOM_URL, 'change-hook') # self.kernel.track_url(RANDOM_URL, 'change-hook')
obj = self.kernel.db.tracked.find_one() # obj = self.kernel.db.tracked.find_one()
self.kernel.update(obj) # self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one() # doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1 # assert doc['hook_fired'] == 1
# we rely on this URL updating # # we rely on this URL updating
self.kernel.update(obj) # self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one() # doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 2 # assert doc['hook_fired'] == 2
def test_get_update_queue(self): def test_get_update_queue(self):
self.kernel.track_url('never-updates', 'fast-update') self.kernel.track_url('never-updates', 'fast-update')