From dd622ab25053350823765c34c4aa997061a3d3b7 Mon Sep 17 00:00:00 2001
From: James Turk <james.p.turk@gmail.com>
Date: Tue, 20 Sep 2011 11:43:44 -0400
Subject: [PATCH] have celery settings rely on oyster settings

---
 README.rst             |  4 +++-
 oyster/celeryconfig.py |  7 +++++--
 oyster/client.py       | 10 +++++-----
 oyster/tasks.py        |  3 ++-
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index 11d92b9..ac646fe 100644
--- a/README.rst
+++ b/README.rst
@@ -26,4 +26,6 @@ Requirements
 Usage
 =====
 
-* Run celeryd with beat ``celeryd -B``
+* Run celeryd with beat ``celeryd -B --config=oyster.celeryconfig``
+* Run oyster HTTP portal ``python oyster/web.py``
+
diff --git a/oyster/celeryconfig.py b/oyster/celeryconfig.py
index f64f8c9..a1c6af6 100644
--- a/oyster/celeryconfig.py
+++ b/oyster/celeryconfig.py
@@ -1,8 +1,11 @@
+from oyster.conf import settings
+
 CELERY_IMPORTS = ("oyster.tasks",)
 
 BROKER_TRANSPORT = 'mongodb'
 CELERY_RESULT_BACKEND = 'mongodb'
+
 CELERY_MONGODB_BACKEND_SETTINGS = {
-    'host': 'localhost',
-    'port': 27017,
+    'host': settings.MONGO_HOST,
+    'port': settings.MONGO_PORT,
 }
diff --git a/oyster/client.py b/oyster/client.py
index df39ad5..e92aa34 100644
--- a/oyster/client.py
+++ b/oyster/client.py
@@ -10,6 +10,7 @@ import scrapelib
 
 
 def get_configured_client():
+    """ helper factory, gets a client configured with oyster.conf.settings """
     from oyster.conf import settings
     return Client(mongo_host=settings.MONGO_HOST,
                   mongo_port=settings.MONGO_PORT,
@@ -22,17 +23,15 @@ def get_configured_client():
                   retry_wait_seconds=settings.RETRY_WAIT_SECONDS)
 
 
-
-
-
-
 class Client(object):
-
+    """ oyster's workhorse, handles tracking """
 
     def __init__(self, mongo_host='localhost', mongo_port=27017,
                  mongo_db='oyster', mongo_log_maxsize=100000000,
                  user_agent='oyster', rpm=600, timeout=None,
                  retry_attempts=0, retry_wait_seconds=5):
+
+        # set up a capped log if it doesn't exist
         self.db = pymongo.Connection(mongo_host, mongo_port)[mongo_db]
         try:
             self.db.create_collection('logs', capped=True,
@@ -66,6 +65,7 @@ class Client(object):
 
 
     def log(self, action, url, error=False, **kwargs):
+        """ add an entry to the oyster log """
         kwargs['action'] = action
         kwargs['url'] = url
         kwargs['error'] = error
diff --git a/oyster/tasks.py b/oyster/tasks.py
index 4758e12..1a1ea22 100644
--- a/oyster/tasks.py
+++ b/oyster/tasks.py
@@ -5,6 +5,7 @@ from oyster.client import get_configured_client
 
 
 class UpdateTask(Task):
+    """ task that updates documents """
     # results go straight to database
     ignore_result = True
 
@@ -14,7 +15,6 @@ class UpdateTask(Task):
 
 
     def run(self, doc_id):
-        # maybe fetch doc instead?
         doc = self.client.db.tracked.find_one({'_id': doc_id})
         self.client.update(doc)
         for hook in doc.get('post_update_hooks', []):
@@ -23,6 +23,7 @@ class UpdateTask(Task):
 
 
 class UpdateTaskScheduler(PeriodicTask):
+    """ recurring task that populates the update queue """
 
     # 60s tick
     run_every = 60