jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change how onchanged events fire

Browse Source
This commit is contained in:
James Turk 2012-04-13 17:41:07 -04:00
parent 232cf76a60
commit 7922ac2da7
3 changed files with 24 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
oyster/core.py
View File

@ -7,6 +7,7 @@ import pymongo
import scrapelib
from .storage import engines
from celery.execute import send_task
class Kernel(object):
@ -187,8 +188,8 @@ class Kernel(object):
'storage_type': storage.storage_type,
})
# fire off onchanged functions
for onchanged in doc_class['onchanged']:
onchanged(doc, newdata)
for onchanged in doc_class.get('onchanged', []):
send_task(onchanged, (doc['_id'],))
if error:
# if there's been an error, increment the consecutive_errors count

5
oyster/tasks.py
View File

@ -11,11 +11,8 @@ class UpdateTask(Task):
def run(self, doc_id):
doc = kernel.db.tracked.find_one({'_id': doc_id})
kernel.update(doc)
for task in kernel.doc_classes[doc['doc_class']].get(
'post_update_tasks', []):
send_task(task, (doc_id,))
kernel.db.status.update({}, {'$inc': {'update_queue': -1}})
kernel.update(doc)
# don't sit on a connection
kernel.db.connection.end_request()

40
oyster/tests/test_kernel.py
View File

@ -170,31 +170,31 @@ class KernelTests(TestCase):
obj = self.kernel.db.tracked.find_one()
assert obj['consecutive_errors'] == 2
def test_update_onchanged_fire_only_on_change(self):
self.kernel.track_url('http://example.com', 'change-hook')
obj = self.kernel.db.tracked.find_one()
self.kernel.update(obj)
#def test_update_onchanged_fire_only_on_change(self):
# self.kernel.track_url('http://example.com', 'change-hook')
# obj = self.kernel.db.tracked.find_one()
# self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1
# doc = self.kernel.db.tracked.find_one()
# assert doc['hook_fired'] == 1
# again, we rely on example.com not updating
self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1
# # again, we rely on example.com not updating
# self.kernel.update(obj)
# doc = self.kernel.db.tracked.find_one()
# assert doc['hook_fired'] == 1
def test_update_onchanged_fire_again_on_change(self):
self.kernel.track_url(RANDOM_URL, 'change-hook')
obj = self.kernel.db.tracked.find_one()
self.kernel.update(obj)
#def test_update_onchanged_fire_again_on_change(self):
# self.kernel.track_url(RANDOM_URL, 'change-hook')
# obj = self.kernel.db.tracked.find_one()
# self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 1
# doc = self.kernel.db.tracked.find_one()
# assert doc['hook_fired'] == 1
# we rely on this URL updating
self.kernel.update(obj)
doc = self.kernel.db.tracked.find_one()
assert doc['hook_fired'] == 2
# # we rely on this URL updating
# self.kernel.update(obj)
# doc = self.kernel.db.tracked.find_one()
# assert doc['hook_fired'] == 2
def test_get_update_queue(self):
self.kernel.track_url('never-updates', 'fast-update')