use configured client

oyster/client.py

@@ -9,6 +9,23 @@ import gridfs
 import scrapelib
 
 
+def get_configured_client():
+    from oyster.conf import settings
+    return Client(mongo_host=settings.MONGO_HOST,
+                  mongo_port=settings.MONGO_PORT,
+                  mongo_db=settings.MONGO_DATABASE,
+                  mongo_log_maxsize=settings.MONGO_LOG_MAXSIZE,
+                  user_agent=settings.USER_AGENT,
+                  rpm=settings.REQUESTS_PER_MINUTE,
+                  timeout=settings.REQUEST_TIMEOUT,
+                  retry_attempts=settings.RETRY_ATTEMPTS,
+                  retry_wait_seconds=settings.RETRY_WAIT_SECONDS)
+
+
+
+
+
+
 class Client(object):
 
 

oyster/conf/__init__.py

@@ -0,0 +1,21 @@
+from oyster.conf import default_settings
+
+class Settings(object):
+    def __init__(self):
+        pass
+
+    def update(self, module):
+        for setting in dir(module):
+            if setting.isupper():
+                val = getattr(module, setting)
+                if val is not None:
+                    setattr(self, setting, val)
+
+settings = Settings()
+settings.update(default_settings)
+
+try:
+    import oyster_settings
+    settings.update(oyster_settings)
+except ImportError:
+    pass

oyster/conf/default_settings.py

@@ -0,0 +1,14 @@
+# mongodb
+MONGO_HOST = 'localhost'
+MONGO_PORT = 27017
+MONGO_DATABASE = 'oyster'
+MONGO_LOG_MAXSIZE = 100000000
+
+# scrapelib
+USER_AGENT = 'oyster'
+REQUESTS_PER_MINUTE = 300
+REQUEST_TIMEOUT = 0
+RETRY_ATTEMPTS = 0
+RETRY_WAIT_SECONDS = 5
+
+

oyster/tasks.py

@@ -1,21 +1,25 @@
-from celery.task import task
-from celery.task.base import PeriodicTask
-from celery.task.schedules import schedule
+from celery.task.base import Task, PeriodicTask
 from celery.execute import send_task
 
-from oyster.client import Client
-
-client = Client()
+from oyster.client import get_configured_client
 
 
-@task(ignore_result=True)
-def update_task(doc_id):
-    # maybe fetch doc instead?
-    doc = client.db.tracked.find_one({'_id': doc_id})
-    client.update(doc)
-    for hook in doc.get('post_update_hooks', []):
-        send_task(hook, (doc_id,))
-    client.db.status.update({}, {'$inc': {'update_queue': -1}})
+class UpdateTask(Task):
+    # results go straight to database
+    ignore_result = True
+
+    def __init__(self):
+        # one client per process
+        client = get_configured_client()
+
+
+    def run(self, doc_id):
+        # maybe fetch doc instead?
+        doc = client.db.tracked.find_one({'_id': doc_id})
+        client.update(doc)
+        for hook in doc.get('post_update_hooks', []):
+            send_task(hook, (doc_id,))
+        client.db.status.update({}, {'$inc': {'update_queue': -1}})
 
 
 class UpdateTaskScheduler(PeriodicTask):

oyster/web.py

@@ -5,7 +5,7 @@ import functools
 import flask
 import pymongo.objectid
 
-from oyster.client import Client
+from oyster.client import get_configured_client
 
 
 class JSONEncoder(json.JSONEncoder):
@@ -33,14 +33,13 @@ def api_wrapper(template=None):
 
 
 app = flask.Flask('oyster')
-client = Client()
+client = get_configured_client()
 
 
 @app.route('/')
 @api_wrapper('index.html')
 def index():
     status = {
-        'queue_size': app.work_queue.qsize(),
         'tracking': client.db.tracked.count(),
         'need_update': client.get_update_queue_size(),
         'logs': client.db.logs.find().sort('$natural', -1).limit(20)
@@ -52,7 +51,6 @@ def index():
 @api_wrapper()
 def doc_list():
     status = {
-        'queue_size': app.work_queue.qsize(),
         'tracking': client.db.tracked.count(),
         'need_update': client.get_update_queue_size(),
     }