diff --git a/oyster/core.py b/oyster/core.py index 3ad4ba1..dbd20d0 100644 --- a/oyster/core.py +++ b/oyster/core.py @@ -72,10 +72,10 @@ class Kernel(object): self.db.drop_collection('logs') self.db.drop_collection('status') - def log(self, action, url, error=False, **kwargs): + def log(self, action, doc_id, error=False, **kwargs): """ add an entry to the oyster log """ kwargs['action'] = action - kwargs['url'] = url + kwargs['doc_id'] = doc_id kwargs['error'] = error kwargs['timestamp'] = datetime.datetime.utcnow() self.db.logs.insert(kwargs) @@ -96,7 +96,7 @@ class Kernel(object): """ if doc_class not in self.doc_classes: error = 'unregistered doc_class %s' % doc_class - self.log('track', url=url, error=error) + self.log('track', id, url=url, error=error) raise ValueError(error) # try and find an existing version of this document @@ -123,10 +123,10 @@ class Kernel(object): tracked['url'], tracked['doc_class'], url, doc_class)) - self.log('track', url=url, error=error) + self.log('track', id, url=url, error=error) raise ValueError(error) - self.log('track', url=url) + self.log('track', id, url=url) newdoc = dict(url=url, doc_class=doc_class, _random=random.randint(0, sys.maxint), @@ -209,7 +209,8 @@ class Kernel(object): else: doc['next_update'] = None - self.log('update', url=url, new_doc=new_version, error=error) + self.log('update', doc['_id'], url=url, new_doc=new_version, + error=error) self.db.tracked.save(doc, safe=True) diff --git a/oyster/ext/elasticsearch.py b/oyster/ext/elasticsearch.py index 7f783f0..289503d 100644 --- a/oyster/ext/elasticsearch.py +++ b/oyster/ext/elasticsearch.py @@ -10,14 +10,19 @@ es = ES(settings.ELASTICSEARCH_HOST) class ElasticSearchPush(Task): # results go straight to elasticsearch ignore_result = True + action = 'elasticsearch' def run(self, doc_id): doc = kernel.db.tracked.find_one({'_id': doc_id}) - text = kernel.extract_text(doc) - self.get_logger().debug('adding {0} to ElasticSearch'.format(doc_id)) + try: + text = kernel.extract_text(doc) - es.index(dict(doc['metadata'], text=text), - settings.ELASTICSEARCH_INDEX, - settings.ELASTICSEARCH_DOC_TYPE, - id=doc_id) + kernel.log(self.action, doc_id, error=False) + + es.index(dict(doc['metadata'], text=text), + settings.ELASTICSEARCH_INDEX, + settings.ELASTICSEARCH_DOC_TYPE, + id=doc_id) + except Exception as e: + kernel.log(self.action, doc_id, error=True, exception=str(e)) diff --git a/oyster/templates/log_row.html b/oyster/templates/log_row.html index f97d4f1..1f7fab4 100644 --- a/oyster/templates/log_row.html +++ b/oyster/templates/log_row.html @@ -1,6 +1,6 @@