From 1047ca89273b55c21236e77efa356d5b37c11eef Mon Sep 17 00:00:00 2001 From: James Turk Date: Wed, 21 Sep 2011 17:00:06 -0400 Subject: [PATCH] default timeout and return tracked _id --- oyster/client.py | 12 ++++++------ oyster/conf/default_settings.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/oyster/client.py b/oyster/client.py index 6aa5cf2..1439677 100644 --- a/oyster/client.py +++ b/oyster/client.py @@ -28,8 +28,8 @@ class Client(object): def __init__(self, mongo_host='localhost', mongo_port=27017, mongo_db='oyster', mongo_log_maxsize=100000000, - user_agent='oyster', rpm=600, timeout=None, - retry_attempts=100, retry_wait_minutes=60): + user_agent='oyster', rpm=60, timeout=300, + retry_attempts=3, retry_wait_minutes=60): # set up a capped log if it doesn't exist self.db = pymongo.Connection(mongo_host, mongo_port)[mongo_db] @@ -86,10 +86,10 @@ class Client(object): raise ValueError('%s is already tracked' % url) self.log('track', url=url) - self.db.tracked.insert(dict(url=url, versioning=versioning, - update_mins=update_mins, - _random=random.randint(0, sys.maxint), - metadata=kwargs)) + return self.db.tracked.insert(dict(url=url, versioning=versioning, + update_mins=update_mins, + _random=random.randint(0, sys.maxint), + metadata=kwargs)) def md5_versioning(self, doc, data): diff --git a/oyster/conf/default_settings.py b/oyster/conf/default_settings.py index d745b20..557d086 100644 --- a/oyster/conf/default_settings.py +++ b/oyster/conf/default_settings.py @@ -6,7 +6,7 @@ MONGO_LOG_MAXSIZE = 100000000 # scrapelib USER_AGENT = 'oyster' -REQUESTS_PER_MINUTE = 300 +REQUESTS_PER_MINUTE = 60 REQUEST_TIMEOUT = 300 # other