From 66acbde1d86c5989fe89d918542c68f716c18f8f Mon Sep 17 00:00:00 2001 From: James Turk Date: Tue, 15 May 2012 16:38:14 -0400 Subject: [PATCH] add --sample to signal.py --- oyster/ext/elasticsearch.py | 1 + oyster/scripts/signal.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/oyster/ext/elasticsearch.py b/oyster/ext/elasticsearch.py index 289503d..44a7b87 100644 --- a/oyster/ext/elasticsearch.py +++ b/oyster/ext/elasticsearch.py @@ -26,3 +26,4 @@ class ElasticSearchPush(Task): id=doc_id) except Exception as e: kernel.log(self.action, doc_id, error=True, exception=str(e)) + raise diff --git a/oyster/scripts/signal.py b/oyster/scripts/signal.py index c6a6608..784070c 100644 --- a/oyster/scripts/signal.py +++ b/oyster/scripts/signal.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import argparse from celery.execute import send_task +from celery import current_app from oyster.core import kernel @@ -12,6 +13,7 @@ def main(): parser.add_argument('task', type=str, help='task name to apply') parser.add_argument('doc_class', type=str, help='doc_class to apply function to') + parser.add_argument('--sample', action='store_true') args = parser.parse_args() @@ -20,8 +22,16 @@ def main(): }, timeout=False) print '%s docs in %s' % (docs.count(), args.doc_class) - for doc in docs: - send_task(args.task, (doc['_id'], )) + if args.sample: + print 'sampling 100 documents' + docs = docs.limit(100) + task = current_app.tasks[name] + for doc in docs: + task.apply((doc['_id'],)) + + else: + for doc in docs: + send_task(args.task, (doc['_id'], )) if __name__ == '__main__': main()