jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

3 retry_attempts and a default 5 minute timeout

Browse Source
This commit is contained in:
James Turk 2011-09-20 16:07:24 -04:00
parent c31faa3fb7
commit 376d27a8ff
2 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
oyster/client.py
View File

@ -49,11 +49,9 @@ class Client(object):
requests_per_minute=rpm, requests_per_minute=rpm,
follow_robots=False, follow_robots=False,
raise_errors=True, raise_errors=True,
timeout=timeout, timeout=timeout)
# disable scrapelib's retries
retry_attempts=0, self.retry_attempts = retry_attempts
retry_wait_seconds=0,
)
self.retry_wait_minutes = retry_wait_minutes self.retry_wait_minutes = retry_wait_minutes
@ -136,7 +134,10 @@ class Client(object):
if error: if error:
c_errors = doc.get('consecutive_errors', 0) c_errors = doc.get('consecutive_errors', 0)
doc['consecutive_errors'] = c_errors + 1 doc['consecutive_errors'] = c_errors + 1
update_mins = self.retry_wait_minutes * (2**c_errors) if c_errors <= self.retry_attempts:
update_mins = self.retry_wait_minutes * (2**c_errors)
else:
update_mins = doc['update_mins']
else: else:
doc['consecutive_errors'] = 0 doc['consecutive_errors'] = 0
update_mins = doc['update_mins'] update_mins = doc['update_mins']

6
oyster/conf/default_settings.py
View File

@ -7,6 +7,8 @@ MONGO_LOG_MAXSIZE = 100000000
# scrapelib # scrapelib
USER_AGENT = 'oyster' USER_AGENT = 'oyster'
REQUESTS_PER_MINUTE = 300 REQUESTS_PER_MINUTE = 300
REQUEST_TIMEOUT = 0 REQUEST_TIMEOUT = 300
RETRY_ATTEMPTS = 0
# other
RETRY_ATTEMPTS = 3
RETRY_WAIT_MINUTES = 60 RETRY_WAIT_MINUTES = 60