use configured client
This commit is contained in:
parent
67cc05078e
commit
47d094e919
@ -9,6 +9,23 @@ import gridfs
|
|||||||
import scrapelib
|
import scrapelib
|
||||||
|
|
||||||
|
|
||||||
|
def get_configured_client():
|
||||||
|
from oyster.conf import settings
|
||||||
|
return Client(mongo_host=settings.MONGO_HOST,
|
||||||
|
mongo_port=settings.MONGO_PORT,
|
||||||
|
mongo_db=settings.MONGO_DATABASE,
|
||||||
|
mongo_log_maxsize=settings.MONGO_LOG_MAXSIZE,
|
||||||
|
user_agent=settings.USER_AGENT,
|
||||||
|
rpm=settings.REQUESTS_PER_MINUTE,
|
||||||
|
timeout=settings.REQUEST_TIMEOUT,
|
||||||
|
retry_attempts=settings.RETRY_ATTEMPTS,
|
||||||
|
retry_wait_seconds=settings.RETRY_WAIT_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class Client(object):
|
class Client(object):
|
||||||
|
|
||||||
|
|
||||||
|
21
oyster/conf/__init__.py
Normal file
21
oyster/conf/__init__.py
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
from oyster.conf import default_settings
|
||||||
|
|
||||||
|
class Settings(object):
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def update(self, module):
|
||||||
|
for setting in dir(module):
|
||||||
|
if setting.isupper():
|
||||||
|
val = getattr(module, setting)
|
||||||
|
if val is not None:
|
||||||
|
setattr(self, setting, val)
|
||||||
|
|
||||||
|
settings = Settings()
|
||||||
|
settings.update(default_settings)
|
||||||
|
|
||||||
|
try:
|
||||||
|
import oyster_settings
|
||||||
|
settings.update(oyster_settings)
|
||||||
|
except ImportError:
|
||||||
|
pass
|
14
oyster/conf/default_settings.py
Normal file
14
oyster/conf/default_settings.py
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
# mongodb
|
||||||
|
MONGO_HOST = 'localhost'
|
||||||
|
MONGO_PORT = 27017
|
||||||
|
MONGO_DATABASE = 'oyster'
|
||||||
|
MONGO_LOG_MAXSIZE = 100000000
|
||||||
|
|
||||||
|
# scrapelib
|
||||||
|
USER_AGENT = 'oyster'
|
||||||
|
REQUESTS_PER_MINUTE = 300
|
||||||
|
REQUEST_TIMEOUT = 0
|
||||||
|
RETRY_ATTEMPTS = 0
|
||||||
|
RETRY_WAIT_SECONDS = 5
|
||||||
|
|
||||||
|
|
@ -1,15 +1,19 @@
|
|||||||
from celery.task import task
|
from celery.task.base import Task, PeriodicTask
|
||||||
from celery.task.base import PeriodicTask
|
|
||||||
from celery.task.schedules import schedule
|
|
||||||
from celery.execute import send_task
|
from celery.execute import send_task
|
||||||
|
|
||||||
from oyster.client import Client
|
from oyster.client import get_configured_client
|
||||||
|
|
||||||
client = Client()
|
|
||||||
|
|
||||||
|
|
||||||
@task(ignore_result=True)
|
class UpdateTask(Task):
|
||||||
def update_task(doc_id):
|
# results go straight to database
|
||||||
|
ignore_result = True
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# one client per process
|
||||||
|
client = get_configured_client()
|
||||||
|
|
||||||
|
|
||||||
|
def run(self, doc_id):
|
||||||
# maybe fetch doc instead?
|
# maybe fetch doc instead?
|
||||||
doc = client.db.tracked.find_one({'_id': doc_id})
|
doc = client.db.tracked.find_one({'_id': doc_id})
|
||||||
client.update(doc)
|
client.update(doc)
|
||||||
|
@ -5,7 +5,7 @@ import functools
|
|||||||
import flask
|
import flask
|
||||||
import pymongo.objectid
|
import pymongo.objectid
|
||||||
|
|
||||||
from oyster.client import Client
|
from oyster.client import get_configured_client
|
||||||
|
|
||||||
|
|
||||||
class JSONEncoder(json.JSONEncoder):
|
class JSONEncoder(json.JSONEncoder):
|
||||||
@ -33,14 +33,13 @@ def api_wrapper(template=None):
|
|||||||
|
|
||||||
|
|
||||||
app = flask.Flask('oyster')
|
app = flask.Flask('oyster')
|
||||||
client = Client()
|
client = get_configured_client()
|
||||||
|
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
@api_wrapper('index.html')
|
@api_wrapper('index.html')
|
||||||
def index():
|
def index():
|
||||||
status = {
|
status = {
|
||||||
'queue_size': app.work_queue.qsize(),
|
|
||||||
'tracking': client.db.tracked.count(),
|
'tracking': client.db.tracked.count(),
|
||||||
'need_update': client.get_update_queue_size(),
|
'need_update': client.get_update_queue_size(),
|
||||||
'logs': client.db.logs.find().sort('$natural', -1).limit(20)
|
'logs': client.db.logs.find().sort('$natural', -1).limit(20)
|
||||||
@ -52,7 +51,6 @@ def index():
|
|||||||
@api_wrapper()
|
@api_wrapper()
|
||||||
def doc_list():
|
def doc_list():
|
||||||
status = {
|
status = {
|
||||||
'queue_size': app.work_queue.qsize(),
|
|
||||||
'tracking': client.db.tracked.count(),
|
'tracking': client.db.tracked.count(),
|
||||||
'need_update': client.get_update_queue_size(),
|
'need_update': client.get_update_queue_size(),
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user