jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

have celery settings rely on oyster settings

Browse Source
This commit is contained in:
James Turk 2011-09-20 11:43:44 -04:00
parent 67efb9b681
commit dd622ab250
4 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.rst
View File

@ -26,4 +26,6 @@ Requirements
Usage Usage
===== =====
* Run celeryd with beat ``celeryd -B`` * Run celeryd with beat ``celeryd -B --config=oyster.celeryconfig``
* Run oyster HTTP portal ``python oyster/web.py``

7
oyster/celeryconfig.py
View File

@ -1,8 +1,11 @@
from oyster.conf import settings
CELERY_IMPORTS = ("oyster.tasks",) CELERY_IMPORTS = ("oyster.tasks",)
BROKER_TRANSPORT = 'mongodb' BROKER_TRANSPORT = 'mongodb'
CELERY_RESULT_BACKEND = 'mongodb' CELERY_RESULT_BACKEND = 'mongodb'
CELERY_MONGODB_BACKEND_SETTINGS = { CELERY_MONGODB_BACKEND_SETTINGS = {
'host': 'localhost', 'host': settings.MONGO_HOST,
'port': 27017, 'port': settings.MONGO_PORT,
} }

10
oyster/client.py
View File

@ -10,6 +10,7 @@ import scrapelib
def get_configured_client(): def get_configured_client():
""" helper factory, gets a client configured with oyster.conf.settings """
from oyster.conf import settings from oyster.conf import settings
return Client(mongo_host=settings.MONGO_HOST, return Client(mongo_host=settings.MONGO_HOST,
mongo_port=settings.MONGO_PORT, mongo_port=settings.MONGO_PORT,
@ -22,17 +23,15 @@ def get_configured_client():
retry_wait_seconds=settings.RETRY_WAIT_SECONDS) retry_wait_seconds=settings.RETRY_WAIT_SECONDS)
class Client(object): class Client(object):
""" oyster's workhorse, handles tracking """
def __init__(self, mongo_host='localhost', mongo_port=27017, def __init__(self, mongo_host='localhost', mongo_port=27017,
mongo_db='oyster', mongo_log_maxsize=100000000, mongo_db='oyster', mongo_log_maxsize=100000000,
user_agent='oyster', rpm=600, timeout=None, user_agent='oyster', rpm=600, timeout=None,
retry_attempts=0, retry_wait_seconds=5): retry_attempts=0, retry_wait_seconds=5):
# set up a capped log if it doesn't exist
self.db = pymongo.Connection(mongo_host, mongo_port)[mongo_db] self.db = pymongo.Connection(mongo_host, mongo_port)[mongo_db]
try: try:
self.db.create_collection('logs', capped=True, self.db.create_collection('logs', capped=True,
@ -66,6 +65,7 @@ class Client(object):
def log(self, action, url, error=False, **kwargs): def log(self, action, url, error=False, **kwargs):
""" add an entry to the oyster log """
kwargs['action'] = action kwargs['action'] = action
kwargs['url'] = url kwargs['url'] = url
kwargs['error'] = error kwargs['error'] = error

3
oyster/tasks.py
View File

@ -5,6 +5,7 @@ from oyster.client import get_configured_client
class UpdateTask(Task): class UpdateTask(Task):
""" task that updates documents """
# results go straight to database # results go straight to database
ignore_result = True ignore_result = True
@ -14,7 +15,6 @@ class UpdateTask(Task):
def run(self, doc_id): def run(self, doc_id):
# maybe fetch doc instead?
doc = self.client.db.tracked.find_one({'_id': doc_id}) doc = self.client.db.tracked.find_one({'_id': doc_id})
self.client.update(doc) self.client.update(doc)
for hook in doc.get('post_update_hooks', []): for hook in doc.get('post_update_hooks', []):
@ -23,6 +23,7 @@ class UpdateTask(Task):
class UpdateTaskScheduler(PeriodicTask): class UpdateTaskScheduler(PeriodicTask):
""" recurring task that populates the update queue """
# 60s tick # 60s tick
run_every = 60 run_every = 60