jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all created clients in oysterd should obey settings

Browse Source
This commit is contained in:
James Turk 2011-08-02 16:18:26 -04:00
parent f5e463e366
commit c14363ba38
1 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
oyster/bin/oysterd.py
View File

@ -12,17 +12,17 @@ from oyster.web import app
class UpdateProcess(multiprocessing.Process):
def __init__(self, task_q):
def __init__(self, task_q, client):
super(UpdateProcess, self).__init__()
self.task_q = task_q
self.client = Client()
self.client = client
def run(self):
while True:
task = self.task_q.get()
# break on 'None' poison pill
# break on 'None' poison pill -- doesn't actually happen currently
if task is None:
self.task_q.task_done()
break
@ -34,6 +34,16 @@ class UpdateProcess(multiprocessing.Process):
self.task_q.task_done()
def _client_from_args(args):
return Client(mongo_host=args.mongo_host,
mongo_port=args.mongo_port,
mongo_db=args.mongo_db,
mongo_log_maxsize=args.logsize,
user_agent=args.useragent,
rpm=args.rpm,
timeout=args.timeout,
retry_attempts=args.retry_attempts,
retry_wait_seconds=args.retry_wait)
def main():
@ -67,11 +77,13 @@ def main():
args = parser.parse_args()
work_queue = multiprocessing.JoinableQueue()
client = _client_from_args(args)
# workers defaults to cpu_count
if not args.workers:
args.workers = multiprocessing.cpu_count()
workers = [UpdateProcess(work_queue) for i in xrange(args.workers)]
workers = [UpdateProcess(work_queue, _client_from_args(args))
for i in xrange(args.workers)]
# separate process for Flask app
def flask_process():
@ -86,16 +98,6 @@ def main():
worker.start()
server.start()
client = Client(mongo_host=args.mongo_host,
mongo_port=args.mongo_port,
mongo_db=args.mongo_db,
mongo_log_maxsize=args.logsize,
user_agent=args.useragent,
rpm=args.rpm,
timeout=args.timeout,
retry_attempts=args.retry_attempts,
retry_wait_seconds=args.retry_wait_seconds)
while True:
# get everything overdue and put it into the queue
next_set = client.get_update_queue()