diff --git a/oyster/core.py b/oyster/core.py index 4a7dfb3..9ff7072 100644 --- a/oyster/core.py +++ b/oyster/core.py @@ -37,7 +37,8 @@ class Kernel(object): self.db.status.insert({'update_queue': 0}) # ensure an index on _random - self.db.tracked.ensure_index([('_random', pymongo.ASCENDING)]) + self.db.tracked.ensure_index('_random') + self.db.tracked.ensure_index('url') self.scraper = scrapelib.Scraper(user_agent=user_agent, requests_per_minute=rpm, diff --git a/oyster/ext/cloudsearch.py b/oyster/ext/cloudsearch.py index 5d3f3d7..a8ace5d 100644 --- a/oyster/ext/cloudsearch.py +++ b/oyster/ext/cloudsearch.py @@ -1,58 +1,8 @@ -import json -import requests from celery.task.base import Task from ..core import kernel from ..conf import settings - -class CloudSearch(object): - - # slightly under 5MB - MAX_BYTES = 5240000 - - def __init__(self, name, domainid, flush_every=None): - self.doc_url = 'http://doc-{0}-{1}.us-east-1.cloudsearch.amazonaws.com/2011-02-01/documents/batch'.format(name, domainid) - self.search_url = 'http://search-{0}-{1}.us-east-1.cloudsearch.amazonaws.com/2011-02-01/search'.format(name, domainid) - - self._current_batch = [] - self._current_size = 0 - self._flush_every = flush_every - - - def flush(self): - print 'flushing!' - if self._current_batch: - payload = '[{0}]'.format(','.join(self._current_batch)) - resp = requests.post(self.doc_url, payload, - headers={'content-type': 'application/json'}) - self._current_batch = [] - self._current_size = 0 - if resp.status_code >= 400: - # http://docs.amazonwebservices.com/cloudsearch/latest/developerguide/DocumentsBatch.html - raise Exception('{0}: {1}'.format(resp.status_code, resp.text)) - - def add_document(self, id, **kwargs): - newdoc = {'type': 'add', 'version': 1, 'lang': 'en', - 'id': id, 'fields': kwargs} - newjson = json.dumps(newdoc) - newsize = len(newjson) - - if (self._current_size + newsize > self.MAX_BYTES or - (self._flush_every and - len(self._current_batch) > self._flush_every)): - self.flush() - - self._current_batch.append(json.dumps(newdoc)) - self._current_size += newsize - - def search_by_expr(self, q, bq=None, size=10, start=0): - " http://docs.amazonwebservices.com/cloudsearch/latest/developerguide/SearchAPI.html " - params = {'q': q, 'size': size, 'start': start} - if bq: - params['bq'] = bq - return requests.get(self.search_url, params=params).text - cs = CloudSearch(settings.CLOUDSEARCH_DOMAIN, settings.CLOUDSEARCH_ID, 20) class CloudSearchPush(Task):