jpt-archive/oyster
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix to double-stacking queue problem and string-ize exceptions

Browse Source
This commit is contained in:
James Turk 2011-08-01 15:02:47 -04:00
parent fc1a311615
commit b1d9151b08
2 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
oyster/bin/oysterd.py
View File

@ -64,9 +64,9 @@ def main():
# do all queued work # do all queued work
work_queue.join() work_queue.join()
else: # sleep for a minute between attempts to check the queue
# allow for a quiet period if queue is exhausted time.sleep(60)
time.sleep(60)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

6
oyster/client.py
View File

@ -87,10 +87,9 @@ class Client(object):
url = doc['url'].replace(' ', '%20') url = doc['url'].replace(' ', '%20')
data = self.scraper.urlopen(url) data = self.scraper.urlopen(url)
content_type = data.response.headers['content-type'] content_type = data.response.headers['content-type']
except scrapelib.HTTPError as e: except Exception as e:
# TODO: log error
do_put = False do_put = False
error = e error = str(e)
# versioning is a concept for future use, but here's how it can work: # versioning is a concept for future use, but here's how it can work:
# versioning functions take doc & data, and return True if data is # versioning functions take doc & data, and return True if data is
@ -117,7 +116,6 @@ class Client(object):
else: else:
doc['_consecutive_errors'] = 0 doc['_consecutive_errors'] = 0
self.log('update', url=url, new_doc=do_put, error=error) self.log('update', url=url, new_doc=do_put, error=error)
self.db.tracked.save(doc, safe=True) self.db.tracked.save(doc, safe=True)