jpt-archive/saucebrush
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

separate Unicode and String filters and fix bug in utils.Files

Browse Source
This commit is contained in:
Jeremy Carbaugh 2009-08-06 14:29:22 -04:00
parent 20a7beebab
commit dc1d3eab14
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
saucebrush/filters.py
View File

@ -361,9 +361,23 @@ class UnicodeFilter(Filter):
if isinstance(value, str): if isinstance(value, str):
record[key] = unicode(value, self._encoding, self._errors) record[key] = unicode(value, self._encoding, self._errors)
elif isinstance(value, unicode): elif isinstance(value, unicode):
record[key] = value.decode(self._encoding, self._errors)
return record
class StringFilter(Filter):
def __init__(self, encoding='utf-8', errors='ignore'):
super(UnicodeFilter, self).__init__()
self._encoding = encoding
self._errors = errors
def process_record(self, record):
for key, value in record.iteritems():
if isinstance(value, unicode):
record[key] = value.encode(self._encoding, self._errors) record[key] = value.encode(self._encoding, self._errors)
return record return record
########################### ###########################
## Commonly Used Filters ## ## Commonly Used Filters ##
########################### ###########################

6
saucebrush/utils.py
View File

@ -167,11 +167,11 @@ def dotted_key_set(dict_or_list, dotted_key, value, separator='.'):
class Files(object): class Files(object):
def __init__(self, file_open_callback=None, *args): def __init__(self, *args):
self.paths = [] self.paths = []
for arg in args: for arg in args:
self.add(arg) self.add(arg)
self.file_open_callback = file_open_callback self.file_open_callback = None
def add(self, path): def add(self, path):
self.paths.append(path) self.paths.append(path)
@ -180,8 +180,10 @@ class Files(object):
return self.linereader() return self.linereader()
def linereader(self): def linereader(self):
import os
for path in iter(self.paths): for path in iter(self.paths):
if os.path.exists(path): if os.path.exists(path):
if self.file_open_callback:
self.file_open_callback(path) self.file_open_callback(path)
f = open(path) f = open(path)
for line in f: for line in f: