From 376d27a8ffa741bafba306f57b7443d800fd5d8b Mon Sep 17 00:00:00 2001 From: James Turk Date: Tue, 20 Sep 2011 16:07:24 -0400 Subject: [PATCH] 3 retry_attempts and a default 5 minute timeout --- oyster/client.py | 13 +++++++------ oyster/conf/default_settings.py | 6 ++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/oyster/client.py b/oyster/client.py index 0f2d137..6aa5cf2 100644 --- a/oyster/client.py +++ b/oyster/client.py @@ -49,11 +49,9 @@ class Client(object): requests_per_minute=rpm, follow_robots=False, raise_errors=True, - timeout=timeout, - # disable scrapelib's retries - retry_attempts=0, - retry_wait_seconds=0, - ) + timeout=timeout) + + self.retry_attempts = retry_attempts self.retry_wait_minutes = retry_wait_minutes @@ -136,7 +134,10 @@ class Client(object): if error: c_errors = doc.get('consecutive_errors', 0) doc['consecutive_errors'] = c_errors + 1 - update_mins = self.retry_wait_minutes * (2**c_errors) + if c_errors <= self.retry_attempts: + update_mins = self.retry_wait_minutes * (2**c_errors) + else: + update_mins = doc['update_mins'] else: doc['consecutive_errors'] = 0 update_mins = doc['update_mins'] diff --git a/oyster/conf/default_settings.py b/oyster/conf/default_settings.py index 6c520e8..d745b20 100644 --- a/oyster/conf/default_settings.py +++ b/oyster/conf/default_settings.py @@ -7,6 +7,8 @@ MONGO_LOG_MAXSIZE = 100000000 # scrapelib USER_AGENT = 'oyster' REQUESTS_PER_MINUTE = 300 -REQUEST_TIMEOUT = 0 -RETRY_ATTEMPTS = 0 +REQUEST_TIMEOUT = 300 + +# other +RETRY_ATTEMPTS = 3 RETRY_WAIT_MINUTES = 60