have celery settings rely on oyster settings

README.rst

@@ -26,4 +26,6 @@ Requirements
 Usage
 =====
 
-* Run celeryd with beat ``celeryd -B``
+* Run celeryd with beat ``celeryd -B --config=oyster.celeryconfig``
+* Run oyster HTTP portal ``python oyster/web.py``
+

oyster/celeryconfig.py

@@ -1,8 +1,11 @@
+from oyster.conf import settings
+
 CELERY_IMPORTS = ("oyster.tasks",)
 
 BROKER_TRANSPORT = 'mongodb'
 CELERY_RESULT_BACKEND = 'mongodb'
+
 CELERY_MONGODB_BACKEND_SETTINGS = {
-    'host': 'localhost',
-    'port': 27017,
+    'host': settings.MONGO_HOST,
+    'port': settings.MONGO_PORT,
 }

oyster/client.py

@@ -10,6 +10,7 @@ import scrapelib
 
 
 def get_configured_client():
+    """ helper factory, gets a client configured with oyster.conf.settings """
     from oyster.conf import settings
     return Client(mongo_host=settings.MONGO_HOST,
                   mongo_port=settings.MONGO_PORT,
@@ -22,17 +23,15 @@ def get_configured_client():
                   retry_wait_seconds=settings.RETRY_WAIT_SECONDS)
 
 
-
-
-
-
 class Client(object):
-
+    """ oyster's workhorse, handles tracking """
 
     def __init__(self, mongo_host='localhost', mongo_port=27017,
                  mongo_db='oyster', mongo_log_maxsize=100000000,
                  user_agent='oyster', rpm=600, timeout=None,
                  retry_attempts=0, retry_wait_seconds=5):
+
+        # set up a capped log if it doesn't exist
         self.db = pymongo.Connection(mongo_host, mongo_port)[mongo_db]
         try:
             self.db.create_collection('logs', capped=True,
@@ -66,6 +65,7 @@ class Client(object):
 
 
     def log(self, action, url, error=False, **kwargs):
+        """ add an entry to the oyster log """
         kwargs['action'] = action
         kwargs['url'] = url
         kwargs['error'] = error

oyster/tasks.py

@@ -5,6 +5,7 @@ from oyster.client import get_configured_client
 
 
 class UpdateTask(Task):
+    """ task that updates documents """
     # results go straight to database
     ignore_result = True
 
@@ -14,7 +15,6 @@ class UpdateTask(Task):
 
 
     def run(self, doc_id):
-        # maybe fetch doc instead?
         doc = self.client.db.tracked.find_one({'_id': doc_id})
         self.client.update(doc)
         for hook in doc.get('post_update_hooks', []):
@@ -23,6 +23,7 @@ class UpdateTask(Task):
 
 
 class UpdateTaskScheduler(PeriodicTask):
+    """ recurring task that populates the update queue """
 
     # 60s tick
     run_every = 60