From b1d9151b08be4c5619dfdcb5817737086162ebd9 Mon Sep 17 00:00:00 2001 From: James Turk Date: Mon, 1 Aug 2011 15:02:47 -0400 Subject: [PATCH] fix to double-stacking queue problem and string-ize exceptions --- oyster/bin/oysterd.py | 6 +++--- oyster/client.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/oyster/bin/oysterd.py b/oyster/bin/oysterd.py index 30a4e13..2289837 100644 --- a/oyster/bin/oysterd.py +++ b/oyster/bin/oysterd.py @@ -64,9 +64,9 @@ def main(): # do all queued work work_queue.join() - else: - # allow for a quiet period if queue is exhausted - time.sleep(60) + # sleep for a minute between attempts to check the queue + time.sleep(60) + if __name__ == '__main__': main() diff --git a/oyster/client.py b/oyster/client.py index 542fc11..a09bc79 100644 --- a/oyster/client.py +++ b/oyster/client.py @@ -87,10 +87,9 @@ class Client(object): url = doc['url'].replace(' ', '%20') data = self.scraper.urlopen(url) content_type = data.response.headers['content-type'] - except scrapelib.HTTPError as e: - # TODO: log error + except Exception as e: do_put = False - error = e + error = str(e) # versioning is a concept for future use, but here's how it can work: # versioning functions take doc & data, and return True if data is @@ -117,7 +116,6 @@ class Client(object): else: doc['_consecutive_errors'] = 0 - self.log('update', url=url, new_doc=do_put, error=error) self.db.tracked.save(doc, safe=True)