From 6503320b3945e70ac3556b249f7d5eeb2d7dedf4 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 21:48:42 -0700 Subject: [PATCH 01/19] add requirements file for unicodecsv --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a3bc39e --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +unicodecsv==0.9.0 \ No newline at end of file From c25550f8da27002cc62cb03a0b828b99c0e07d47 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 21:51:58 -0700 Subject: [PATCH 02/19] remove print statement --- saucebrush/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/saucebrush/utils.py b/saucebrush/utils.py index aa8c241..6f98257 100644 --- a/saucebrush/utils.py +++ b/saucebrush/utils.py @@ -60,9 +60,8 @@ def flatten(item, prefix='', separator='_', keys=None): #elif isinstance(item, (tuple, list)): # return {prefix: [flatten(i, prefix, separator, keys) for i in item]} else: - print item, prefix return {prefix: item} - + def str_or_list(obj): if isinstance(obj, str): return [obj] From 12847d342e17aa6878c3fcf27b32ab237fd447be Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:05:46 -0700 Subject: [PATCH 03/19] change iteritems and iterkeys to items and keys views --- examples/earmarkwatch.py | 2 +- examples/fec_electronic.py | 2 +- saucebrush/filters.py | 22 +++++++++++----------- saucebrush/sources.py | 4 ++-- saucebrush/utils.py | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/earmarkwatch.py b/examples/earmarkwatch.py index 88f7454..54a0088 100644 --- a/examples/earmarkwatch.py +++ b/examples/earmarkwatch.py @@ -3,7 +3,7 @@ from saucebrush.outputs import CSVOutput, DebugOutput def merge_columns(datasource, mapping, merge_func): for rowdata in datasource: - for to_col,from_cols in mapping.iteritems(): + for to_col,from_cols in mapping.items(): values = [rowdata.pop(col, None) for col in from_cols] rowdata[to_col] = reduce(merge_func, values) yield rowdata diff --git a/examples/fec_electronic.py b/examples/fec_electronic.py index 6bf1b78..d482eef 100644 --- a/examples/fec_electronic.py +++ b/examples/fec_electronic.py @@ -84,7 +84,7 @@ class FECSource(object): @staticmethod def get_form_type(rectype): - for type_re, type in FECSource.FORM_MAPPING.iteritems(): + for type_re, type in FECSource.FORM_MAPPING.items(): if type_re.match(rectype): return type diff --git a/saucebrush/filters.py b/saucebrush/filters.py index 380628f..2b9584b 100644 --- a/saucebrush/filters.py +++ b/saucebrush/filters.py @@ -217,17 +217,17 @@ class FieldModifier(FieldFilter): class FieldKeeper(Filter): """ Filter that removes all but the given set of fields. - + FieldKeeper(('spam', 'eggs')) removes all bu tthe spam and eggs fields from every record filtered. """ - + def __init__(self, keys): super(FieldKeeper, self).__init__() self._target_keys = utils.str_or_list(keys) - + def process_record(self, record): - for key in record.keys(): + for key in list(record.keys()): if key not in self._target_keys: del record[key] return record @@ -269,7 +269,7 @@ class FieldMerger(Filter): self._keep_fields = keep_fields def process_record(self, record): - for to_col, from_cols in self._field_mapping.iteritems(): + for to_col, from_cols in self._field_mapping.items(): if self._keep_fields: values = [record.get(col, None) for col in from_cols] else: @@ -328,7 +328,7 @@ class FieldCopier(Filter): def process_record(self, record): # mapping is dest:source - for dest, source in self._copy_mapping.iteritems(): + for dest, source in self._copy_mapping.items(): record[dest] = record[source] return record @@ -343,7 +343,7 @@ class FieldRenamer(Filter): def process_record(self, record): # mapping is dest:source - for dest, source in self._rename_mapping.iteritems(): + for dest, source in self._rename_mapping.items(): record[dest] = record.pop(source) return record @@ -363,7 +363,7 @@ class Splitter(Filter): self._split_mapping = split_mapping def process_record(self, record): - for key, filters in self._split_mapping.iteritems(): + for key, filters in self._split_mapping.items(): # if the key doesn't exist -- move on to next key try: @@ -479,7 +479,7 @@ class UnicodeFilter(Filter): self._errors = errors def process_record(self, record): - for key, value in record.iteritems(): + for key, value in record.items(): if isinstance(value, str): record[key] = unicode(value, self._encoding, self._errors) elif isinstance(value, unicode): @@ -494,7 +494,7 @@ class StringFilter(Filter): self._errors = errors def process_record(self, record): - for key, value in record.iteritems(): + for key, value in record.items(): if isinstance(value, unicode): record[key] = value.encode(self._encoding, self._errors) return record @@ -584,7 +584,7 @@ class NameCleaner(Filter): # if there is a match, remove original name and add pieces if match: record.pop(key) - for k,v in match.groupdict().iteritems(): + for k,v in match.groupdict().items(): record[self._name_prefix + k] = v break diff --git a/saucebrush/sources.py b/saucebrush/sources.py index 7427471..06cf60b 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -62,7 +62,7 @@ class FixedWidthFileSource(object): def next(self): line = self._fwfile.next() record = {} - for name, range_ in self._fields_dict.iteritems(): + for name, range_ in self._fields_dict.items(): record[name] = line[range_[0]:range_[1]].rstrip(self._fillchars) return record @@ -182,7 +182,7 @@ class SqliteSource(object): self._conn = sqlite3.connect(self._dbpath) self._conn.row_factory = dict_factory if self._conn_params: - for param, value in self._conn_params.iteritems(): + for param, value in self._conn_params.items(): setattr(self._conn, param, value) def _process_query(self): diff --git a/saucebrush/utils.py b/saucebrush/utils.py index 6f98257..2139137 100644 --- a/saucebrush/utils.py +++ b/saucebrush/utils.py @@ -38,20 +38,20 @@ def string_dig(element, separator=''): def flatten(item, prefix='', separator='_', keys=None): """ Flatten nested dictionary into one with its keys concatenated together. - + >>> flatten({'a':1, 'b':{'c':2}, 'd':[{'e':{'r':7}}, {'e':5}], 'f':{'g':{'h':6}}}) {'a': 1, 'b_c': 2, 'd': [{'e_r': 7}, {'e': 5}], 'f_g_h': 6} """ - + # update dictionaries recursively - + if isinstance(item, dict): # don't prepend a leading _ if prefix != '': prefix += separator retval = {} - for key, value in item.iteritems(): + for key, value in item.items(): if (not keys) or (key in keys): retval.update(flatten(value, prefix + key, separator, keys)) else: From 1cfc4404cf437da1212acdbf7514339313389608 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:06:14 -0700 Subject: [PATCH 04/19] change import to package relative import --- saucebrush/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saucebrush/__init__.py b/saucebrush/__init__.py index 4a9babd..d900a21 100644 --- a/saucebrush/__init__.py +++ b/saucebrush/__init__.py @@ -2,7 +2,7 @@ Saucebrush is a data loading & manipulation framework written in python. """ -import filters, emitters, sources, utils +from . import filters, emitters, sources, utils class SaucebrushError(Exception): From e5f1012e17f0bd2a73222a84282bf1d95667f9ef Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:14:24 -0700 Subject: [PATCH 05/19] use new as keyword for exceptions --- saucebrush/emitters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saucebrush/emitters.py b/saucebrush/emitters.py index 8460984..a2a1f5e 100644 --- a/saucebrush/emitters.py +++ b/saucebrush/emitters.py @@ -146,7 +146,7 @@ class SqliteEmitter(Emitter): qmarks) try: self._cursor.execute(insert, record.values()) - except sqlite3.IntegrityError, ie: + except sqlite3.IntegrityError as ie: if not self._quiet: raise ie self.reject_record(record, ie.message) From 8d512d7e07bcea5cf9d2d4b39cfd3889ef691b7b Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:17:20 -0700 Subject: [PATCH 06/19] handle switch from generator next() to __next__() --- saucebrush/filters.py | 6 +++++- saucebrush/sources.py | 9 +++++++-- saucebrush/tests/filters.py | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/saucebrush/filters.py b/saucebrush/filters.py index 2b9584b..bc1c7be 100644 --- a/saucebrush/filters.py +++ b/saucebrush/filters.py @@ -301,7 +301,11 @@ class FieldAdder(Filter): self._field_name = field_name self._field_value = field_value if hasattr(self._field_value, '__iter__'): - self._field_value = iter(self._field_value).next + value_iter = iter(self._field_value) + if hasattr(value_iter, "next"): + self._field_value = value_iter.next + else: + self._field_value = value_iter.__next__ self._replace = replace def process_record(self, record): diff --git a/saucebrush/sources.py b/saucebrush/sources.py index 06cf60b..c5c6554 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -59,13 +59,18 @@ class FixedWidthFileSource(object): def __iter__(self): return self - def next(self): - line = self._fwfile.next() + def __next__(self): + line = next(self._fwfile) record = {} for name, range_ in self._fields_dict.items(): record[name] = line[range_[0]:range_[1]].rstrip(self._fillchars) return record + def next(self): + """ Keep Python 2 next() method that defers to __next__(). + """ + return self.__next__() + class HtmlTableSource(object): """ Saucebrush source for reading data from an HTML table. diff --git a/saucebrush/tests/filters.py b/saucebrush/tests/filters.py index d0f3b85..c7e4dc7 100644 --- a/saucebrush/tests/filters.py +++ b/saucebrush/tests/filters.py @@ -63,7 +63,8 @@ class FilterTestCase(unittest.TestCase): recipe = DummyRecipe() f = Doubler() result = f.attach([1,2,3], recipe=recipe) - result.next() # next has to be called for attach to take effect + # next has to be called for attach to take effect + next(result) f.reject_record('bad', 'this one was bad') # ensure that the rejection propagated to the recipe @@ -202,7 +203,7 @@ class FilterTestCase(unittest.TestCase): def test_field_keeper(self): fk = FieldKeeper(['c']) - + # check against expected results expected_data = [{'c':3}, {'c':5}, {'c':100}] self.assert_filter_result(fk, expected_data) From fad10e5a9840ba7c30f99f8b5cbf02d0cda9ab74 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:18:12 -0700 Subject: [PATCH 07/19] use either urllib2 or urllib.request as necessary --- saucebrush/utils.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/saucebrush/utils.py b/saucebrush/utils.py index 2139137..afa67dc 100644 --- a/saucebrush/utils.py +++ b/saucebrush/utils.py @@ -1,5 +1,10 @@ import os -import urllib2 + +try: + from urllib.request import urlopen # attemp py3 first +except ImportError: + from urllib2 import urlopen # fallback to py2 + """ General utilities used within saucebrush that may be useful elsewhere. """ @@ -75,7 +80,7 @@ def str_or_list(obj): class Files(object): """ Iterate over multiple files as a single file. Pass the paths of the files as arguments to the class constructor: - + for line in Files('/path/to/file/a', '/path/to/file/b'): pass """ @@ -104,15 +109,15 @@ class Files(object): class RemoteFile(object): """ Stream data from a remote file. - + :param url: URL to remote file """ - + def __init__(self, url): self._url = url - + def __iter__(self): - resp = urllib2.urlopen(self._url) + resp = urlopen(self._url) for line in resp: yield line.rstrip() resp.close() \ No newline at end of file From de72f7755c4b086f1fa6a73c82e90379346a08c0 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:19:15 -0700 Subject: [PATCH 08/19] make int casting explicit --- saucebrush/stats.py | 92 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/saucebrush/stats.py b/saucebrush/stats.py index 0eed9fd..3193474 100644 --- a/saucebrush/stats.py +++ b/saucebrush/stats.py @@ -5,7 +5,7 @@ import math def _average(values): """ Calculate the average of a list of values. - + :param values: an iterable of ints or floats to average """ value_count = len(values) @@ -14,64 +14,64 @@ def _average(values): def _median(values): """ Calculate the median of a list of values. - + :param values: an iterable of ints or floats to calculate """ - + count = len(values) - + # bail early before sorting if 0 or 1 values in list if count == 0: return None elif count == 1: return values[0] - + values = sorted(values) - + if count % 2 == 1: # odd number of items, return middle value - return float(values[count / 2]) + return float(values[int(count / 2)]) else: # even number of items, return average of middle two items - mid = count / 2 + mid = int(count / 2) return sum(values[mid - 1:mid + 1]) / 2.0 def _stddev(values, population=False): """ Calculate the standard deviation and variance of a list of values. - + :param values: an iterable of ints or floats to calculate :param population: True if values represents entire population, False if it is a sample of the population """ - + avg = _average(values) count = len(values) if population else len(values) - 1 - + # square the difference between each value and the average diffsq = ((i - avg) ** 2 for i in values) - + # the average of the squared differences variance = sum(diffsq) / float(count) - + return (math.sqrt(variance), variance) # stddev is sqrt of variance class StatsFilter(Filter): """ Base for all stats filters. """ - + def __init__(self, field, test=None): self._field = field self._test = test - + def process_record(self, record): if self._test is None or self._test(record): self.process_field(record[self._field]) return record - + def process_field(self, record): raise NotImplementedError('process_field not defined in ' + self.__class__.__name__) - + def value(self): raise NotImplementedError('value not defined in ' + self.__class__.__name__) @@ -80,14 +80,14 @@ class Sum(StatsFilter): """ Calculate the sum of the values in a field. Field must contain either int or float values. """ - + def __init__(self, field, initial=0, **kwargs): super(Sum, self).__init__(field, **kwargs) self._value = initial - + def process_field(self, item): self._value += item or 0 - + def value(self): return self._value @@ -95,35 +95,35 @@ class Average(StatsFilter): """ Calculate the average (mean) of the values in a field. Field must contain either int or float values. """ - + def __init__(self, field, initial=0, **kwargs): super(Average, self).__init__(field, **kwargs) self._value = initial self._count = 0 - + def process_field(self, item): if item is not None: self._value += item self._count += 1 - + def value(self): return self._value / float(self._count) class Median(StatsFilter): """ Calculate the median of the values in a field. Field must contain either int or float values. - + **This filter keeps a list of field values in memory.** """ - + def __init__(self, field, **kwargs): super(Median, self).__init__(field, **kwargs) self._values = [] - + def process_field(self, item): if item is not None: self._values.append(item) - + def value(self): return _median(self._values) @@ -131,19 +131,19 @@ class MinMax(StatsFilter): """ Find the minimum and maximum values in a field. Field must contain either int or float values. """ - + def __init__(self, field, **kwargs): super(MinMax, self).__init__(field, **kwargs) self._max = None self._min = None - + def process_field(self, item): if item is not None: if self._max is None or item > self._max: self._max = item if self._min is None or item < self._min: self._min = item - + def value(self): return (self._min, self._max) @@ -156,24 +156,24 @@ class StandardDeviation(StatsFilter): **This filter keeps a list of field values in memory.** """ - + def __init__(self, field, **kwargs): super(StandardDeviation, self).__init__(field, **kwargs) self._values = [] - + def process_field(self, item): if item is not None: self._values.append(item) - + def average(self): return _average(self._values) - + def median(self): return _median(self._values) - + def value(self, population=False): """ Return a tuple of (standard_deviation, variance). - + :param population: True if values represents entire population, False if values is a sample. Default: False """ @@ -185,34 +185,34 @@ class Histogram(StatsFilter): generates a basic and limited histogram useful for printing to the command line. The label_length attribute determines the padding and cut-off of the basic histogram labels. - + **This filters maintains a dict of unique field values in memory.** """ - + label_length = 6 - + def __init__(self, field, **kwargs): super(Histogram, self).__init__(field, **kwargs) self._counter = collections.Counter() - + def process_field(self, item): self._counter[self.prep_field(item)] += 1 - + def prep_field(self, item): return item - + def value(self): return self._counter.copy() - + def in_order(self): ordered = [] for key in sorted(self._counter.keys()): ordered.append((key, self._counter[key])) return ordered - + def most_common(self, n=None): return self._counter.most_common(n) - + @classmethod def as_string(self, occurences, label_length): output = "\n" @@ -220,6 +220,6 @@ class Histogram(StatsFilter): key_str = str(key).ljust(label_length)[:label_length] output += "%s %s\n" % (key_str, "*" * count) return output - + def __str__(self): return Histogram.as_string(self.in_order(), label_length=self.label_length) From 432c4f22eee6e59aa2ad2b9d4315629e80a9af13 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:26:29 -0700 Subject: [PATCH 09/19] handle removal of xrange --- saucebrush/sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saucebrush/sources.py b/saucebrush/sources.py index c5c6554..c0d925b 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -25,8 +25,8 @@ class CSVSource(object): def __init__(self, csvfile, fieldnames=None, skiprows=0, **kwargs): import csv self._dictreader = csv.DictReader(csvfile, fieldnames, **kwargs) - for _ in xrange(skiprows): - self._dictreader.next() + for _ in range(skiprows): + next(self._dictreader) def __iter__(self): return self._dictreader From 46c8ab3ae3e89b6e1759bbb7919fc271d70f8756 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 22:54:22 -0700 Subject: [PATCH 10/19] remove duplicate test method --- saucebrush/tests/sources.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/saucebrush/tests/sources.py b/saucebrush/tests/sources.py index 49c6721..f2f670d 100644 --- a/saucebrush/tests/sources.py +++ b/saucebrush/tests/sources.py @@ -42,17 +42,5 @@ class SourceTestCase(unittest.TestCase): 'year':'1999'}] self.assertEquals(list(source), expected_data) - def test_fixed_width_source(self): - data = cStringIO.StringIO('JamesNovember.3.1986\nTim..September151999') - fields = (('name',5), ('month',9), ('day',2), ('year',4)) - source = FixedWidthFileSource(data, fields, fillchars='.') - expected_data = [{'name':'James', 'month':'November', 'day':'3', - 'year':'1986'}, - {'name':'Tim', 'month':'September', 'day':'15', - 'year':'1999'}] - self.assertEquals(list(source), expected_data) - - - if __name__ == '__main__': unittest.main() From d5b56b931b9f67d82fd866ec6b5b9242af213d56 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 23:14:39 -0700 Subject: [PATCH 11/19] whole bunch of changes to support new unicode-based strings in Py3 --- saucebrush/emitters.py | 28 +++++++------- saucebrush/tests/emitters.py | 72 +++++++++++++++++++----------------- saucebrush/tests/filters.py | 28 +++++++------- saucebrush/tests/sources.py | 16 ++++---- 4 files changed, 77 insertions(+), 67 deletions(-) diff --git a/saucebrush/emitters.py b/saucebrush/emitters.py index a2a1f5e..31a9464 100644 --- a/saucebrush/emitters.py +++ b/saucebrush/emitters.py @@ -2,7 +2,7 @@ Saucebrush Emitters are filters that instead of modifying the record, output it in some manner. """ - +from __future__ import unicode_literals from saucebrush.filters import Filter class Emitter(Filter): @@ -50,12 +50,12 @@ class DebugEmitter(Emitter): self._outfile = outfile def emit_record(self, record): - self._outfile.write(str(record) + '\n') + self._outfile.write("{0}\n".format(record)) class CountEmitter(Emitter): """ Emitter that writes the record count to a file-like object. - + CountEmitter() by default writes to stdout. CountEmitter(outfile=open('text', 'w')) would print to a file name test. CountEmitter(every=1000000) would write the count every 1,000,000 records. @@ -63,36 +63,36 @@ class CountEmitter(Emitter): """ def __init__(self, every=1000, of=None, outfile=None, format=None): - + super(CountEmitter, self).__init__() - + if not outfile: import sys self._outfile = sys.stdout else: self._outfile = outfile - + if format is None: if of is not None: format = "%(count)s of %(of)s\n" else: format = "%(count)s\n" - + self._format = format self._every = every self._of = of self.count = 0 - - def __str__(self): + + def format(self): return self._format % {'count': self.count, 'of': self._of} def emit_record(self, record): self.count += 1 if self.count % self._every == 0: - self._outfile.write(str(self)) - + self._outfile.write(self.format()) + def done(self): - self._outfile.write(str(self)) + self._outfile.write(self.format()) class CSVEmitter(Emitter): @@ -107,7 +107,9 @@ class CSVEmitter(Emitter): import csv self._dictwriter = csv.DictWriter(csvfile, fieldnames) # write header row - self._dictwriter.writerow(dict(zip(fieldnames, fieldnames))) + header_row = dict(zip(fieldnames, fieldnames)) + print(header_row) + self._dictwriter.writerow(header_row) def emit_record(self, record): self._dictwriter.writerow(record) diff --git a/saucebrush/tests/emitters.py b/saucebrush/tests/emitters.py index 3660f9f..ea4e282 100644 --- a/saucebrush/tests/emitters.py +++ b/saucebrush/tests/emitters.py @@ -1,47 +1,53 @@ +from __future__ import unicode_literals +from contextlib import closing +from io import BytesIO, StringIO import unittest -from cStringIO import StringIO + from saucebrush.emitters import DebugEmitter, CSVEmitter, CountEmitter class EmitterTestCase(unittest.TestCase): - def setUp(self): - self.output = StringIO() - def test_debug_emitter(self): - de = DebugEmitter(self.output) - data = de.attach([1,2,3]) - for _ in data: - pass - self.assertEquals(self.output.getvalue(), '1\n2\n3\n') + with closing(StringIO()) as output: + de = DebugEmitter(output) + list(de.attach([1,2,3])) + self.assertEqual(output.getvalue(), '1\n2\n3\n') def test_csv_emitter(self): - ce = CSVEmitter(self.output, ('x','y','z')) - data = ce.attach([{'x':1,'y':2,'z':3}, {'x':5, 'y':5, 'z':5}]) - for _ in data: - pass - self.assertEquals(self.output.getvalue(), 'x,y,z\r\n1,2,3\r\n5,5,5\r\n') - + + try: + import cStringIO # if Python 2.x then use BytesIO + io = BytesIO() + except: + io = StringIO() # if Python 3.x then use StringIO + + with closing(io) as output: + ce = CSVEmitter(output, ('x','y','z')) + list(ce.attach([{'x':1, 'y':2, 'z':3}, {'x':5, 'y':5, 'z':5}])) + self.assertEqual(output.getvalue(), 'x,y,z\r\n1,2,3\r\n5,5,5\r\n') + def test_count_emitter(self): - + # values for test values = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22] - - # test without of parameter - ce = CountEmitter(every=10, outfile=self.output, format="%(count)s records\n") - list(ce.attach(values)) - self.assertEquals(self.output.getvalue(), '10 records\n20 records\n') - ce.done() - self.assertEquals(self.output.getvalue(), '10 records\n20 records\n22 records\n') - - # reset output - self.output.truncate(0) - - # test with of parameter - ce = CountEmitter(every=10, outfile=self.output, of=len(values)) - list(ce.attach(values)) - self.assertEquals(self.output.getvalue(), '10 of 22\n20 of 22\n') - ce.done() - self.assertEquals(self.output.getvalue(), '10 of 22\n20 of 22\n22 of 22\n') + + with closing(StringIO()) as output: + + # test without of parameter + ce = CountEmitter(every=10, outfile=output, format="%(count)s records\n") + list(ce.attach(values)) + self.assertEqual(output.getvalue(), '10 records\n20 records\n') + ce.done() + self.assertEqual(output.getvalue(), '10 records\n20 records\n22 records\n') + + with closing(StringIO()) as output: + + # test with of parameter + ce = CountEmitter(every=10, outfile=output, of=len(values)) + list(ce.attach(values)) + self.assertEqual(output.getvalue(), '10 of 22\n20 of 22\n') + ce.done() + self.assertEqual(output.getvalue(), '10 of 22\n20 of 22\n22 of 22\n') if __name__ == '__main__': unittest.main() diff --git a/saucebrush/tests/filters.py b/saucebrush/tests/filters.py index c7e4dc7..04ce0c4 100644 --- a/saucebrush/tests/filters.py +++ b/saucebrush/tests/filters.py @@ -57,7 +57,7 @@ class FilterTestCase(unittest.TestCase): def assert_filter_result(self, filter_obj, expected_data): result = filter_obj.attach(self._simple_data()) - self.assertEquals(list(result), expected_data) + self.assertEqual(list(result), expected_data) def test_reject_record(self): recipe = DummyRecipe() @@ -68,31 +68,31 @@ class FilterTestCase(unittest.TestCase): f.reject_record('bad', 'this one was bad') # ensure that the rejection propagated to the recipe - self.assertEquals('bad', recipe.rejected_record) - self.assertEquals('this one was bad', recipe.rejected_msg) + self.assertEqual('bad', recipe.rejected_record) + self.assertEqual('this one was bad', recipe.rejected_msg) def test_simple_filter(self): df = Doubler() result = df.attach([1,2,3]) # ensure we got a generator that yields 2,4,6 - self.assertEquals(type(result), types.GeneratorType) - self.assertEquals(list(result), [2,4,6]) + self.assertEqual(type(result), types.GeneratorType) + self.assertEqual(list(result), [2,4,6]) def test_simple_filter_return_none(self): cf = OddRemover() result = cf.attach(range(10)) # ensure only even numbers remain - self.assertEquals(list(result), [0,2,4,6,8]) + self.assertEqual(list(result), [0,2,4,6,8]) def test_simple_yield_filter(self): lf = ListFlattener() result = lf.attach([[1],[2,3],[4,5,6]]) # ensure we got a generator that yields 1,2,3,4,5,6 - self.assertEquals(type(result), types.GeneratorType) - self.assertEquals(list(result), [1,2,3,4,5,6]) + self.assertEqual(type(result), types.GeneratorType) + self.assertEqual(list(result), [1,2,3,4,5,6]) def test_simple_field_filter(self): ff = FieldDoubler(['a', 'c']) @@ -108,7 +108,7 @@ class FilterTestCase(unittest.TestCase): result = cf.attach(range(10)) # ensure only even numbers remain - self.assertEquals(list(result), [0,2,4,6,8]) + self.assertEqual(list(result), [0,2,4,6,8]) ### Tests for Subrecord @@ -124,7 +124,7 @@ class FilterTestCase(unittest.TestCase): sf = SubrecordFilter('a', NonModifyingFieldDoubler('b')) result = sf.attach(data) - self.assertEquals(list(result), expected) + self.assertEqual(list(result), expected) def test_subrecord_filter_deep(self): data = [{'a': {'d':[{'b': 2}, {'b': 4}]}}, @@ -138,7 +138,7 @@ class FilterTestCase(unittest.TestCase): sf = SubrecordFilter('a.d', NonModifyingFieldDoubler('b')) result = sf.attach(data) - self.assertEquals(list(result), expected) + self.assertEqual(list(result), expected) def test_subrecord_filter_nonlist(self): data = [ @@ -156,7 +156,7 @@ class FilterTestCase(unittest.TestCase): sf = SubrecordFilter('a.b', NonModifyingFieldDoubler('c')) result = sf.attach(data) - self.assertEquals(list(result), expected) + self.assertEqual(list(result), expected) def test_subrecord_filter_list_in_path(self): data = [ @@ -174,7 +174,7 @@ class FilterTestCase(unittest.TestCase): sf = SubrecordFilter('a.b', NonModifyingFieldDoubler('c')) result = sf.attach(data) - self.assertEquals(list(result), expected) + self.assertEqual(list(result), expected) def test_conditional_path(self): @@ -296,7 +296,7 @@ class FilterTestCase(unittest.TestCase): expected_data = [{'a': 77}, {'a':33}] result = u.attach(in_data) - self.assertEquals(list(result), expected_data) + self.assertEqual(list(result), expected_data) # TODO: unicode & string filter tests diff --git a/saucebrush/tests/sources.py b/saucebrush/tests/sources.py index f2f670d..82c7086 100644 --- a/saucebrush/tests/sources.py +++ b/saucebrush/tests/sources.py @@ -1,5 +1,7 @@ +from __future__ import unicode_literals +from io import BytesIO, StringIO import unittest -import cStringIO + from saucebrush.sources import CSVSource, FixedWidthFileSource class SourceTestCase(unittest.TestCase): @@ -9,14 +11,14 @@ class SourceTestCase(unittest.TestCase): 1,2,3 5,5,5 1,10,100''' - return cStringIO.StringIO(data) + return StringIO(data) def test_csv_source_basic(self): source = CSVSource(self._get_csv()) expected_data = [{'a':'1', 'b':'2', 'c':'3'}, {'a':'5', 'b':'5', 'c':'5'}, {'a':'1', 'b':'10', 'c':'100'}] - self.assertEquals(list(source), expected_data) + self.assertEqual(list(source), expected_data) def test_csv_source_fieldnames(self): source = CSVSource(self._get_csv(), ['x','y','z']) @@ -24,23 +26,23 @@ class SourceTestCase(unittest.TestCase): {'x':'1', 'y':'2', 'z':'3'}, {'x':'5', 'y':'5', 'z':'5'}, {'x':'1', 'y':'10', 'z':'100'}] - self.assertEquals(list(source), expected_data) + self.assertEqual(list(source), expected_data) def test_csv_source_skiprows(self): source = CSVSource(self._get_csv(), skiprows=1) expected_data = [{'a':'5', 'b':'5', 'c':'5'}, {'a':'1', 'b':'10', 'c':'100'}] - self.assertEquals(list(source), expected_data) + self.assertEqual(list(source), expected_data) def test_fixed_width_source(self): - data = cStringIO.StringIO('JamesNovember 3 1986\nTim September151999') + data = StringIO('JamesNovember 3 1986\nTim September151999') fields = (('name',5), ('month',9), ('day',2), ('year',4)) source = FixedWidthFileSource(data, fields) expected_data = [{'name':'James', 'month':'November', 'day':'3', 'year':'1986'}, {'name':'Tim', 'month':'September', 'day':'15', 'year':'1999'}] - self.assertEquals(list(source), expected_data) + self.assertEqual(list(source), expected_data) if __name__ == '__main__': unittest.main() From 79976e3c78081ff73c235fde667691af8ee97826 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 23:16:33 -0700 Subject: [PATCH 12/19] remove print statement --- saucebrush/emitters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/saucebrush/emitters.py b/saucebrush/emitters.py index 31a9464..acd5665 100644 --- a/saucebrush/emitters.py +++ b/saucebrush/emitters.py @@ -108,7 +108,6 @@ class CSVEmitter(Emitter): self._dictwriter = csv.DictWriter(csvfile, fieldnames) # write header row header_row = dict(zip(fieldnames, fieldnames)) - print(header_row) self._dictwriter.writerow(header_row) def emit_record(self, record): From 7f69f6a769aff932475b44bda75a2881382d26a8 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 23:17:02 -0700 Subject: [PATCH 13/19] use cStringIO on Python 2.x, io.StringIO on 3.x --- saucebrush/tests/emitters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/saucebrush/tests/emitters.py b/saucebrush/tests/emitters.py index ea4e282..612cd8e 100644 --- a/saucebrush/tests/emitters.py +++ b/saucebrush/tests/emitters.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals from contextlib import closing -from io import BytesIO, StringIO +from io import StringIO import unittest from saucebrush.emitters import DebugEmitter, CSVEmitter, CountEmitter @@ -16,8 +16,8 @@ class EmitterTestCase(unittest.TestCase): def test_csv_emitter(self): try: - import cStringIO # if Python 2.x then use BytesIO - io = BytesIO() + import cStringIO # if Python 2.x then use old cStringIO + io = cStringIO.StringIO() except: io = StringIO() # if Python 3.x then use StringIO From 0e2caaf77c868a1c6889ac22b8df844b4c6b6590 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Sun, 11 Mar 2012 23:18:23 -0700 Subject: [PATCH 14/19] unicodecsv is no longer required --- requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a3bc39e..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -unicodecsv==0.9.0 \ No newline at end of file From 4225b96ae124e5e71e7a092caa4de8382ac16923 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Mon, 12 Mar 2012 00:03:20 -0700 Subject: [PATCH 15/19] fix sqlite emitter and add test --- saucebrush/emitters.py | 2 +- saucebrush/tests/emitters.py | 49 +++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/saucebrush/emitters.py b/saucebrush/emitters.py index acd5665..1138581 100644 --- a/saucebrush/emitters.py +++ b/saucebrush/emitters.py @@ -146,7 +146,7 @@ class SqliteEmitter(Emitter): ','.join(record.keys()), qmarks) try: - self._cursor.execute(insert, record.values()) + self._cursor.execute(insert, list(record.values())) except sqlite3.IntegrityError as ie: if not self._quiet: raise ie diff --git a/saucebrush/tests/emitters.py b/saucebrush/tests/emitters.py index 612cd8e..14ec8d8 100644 --- a/saucebrush/tests/emitters.py +++ b/saucebrush/tests/emitters.py @@ -1,9 +1,10 @@ from __future__ import unicode_literals from contextlib import closing from io import StringIO +import os import unittest -from saucebrush.emitters import DebugEmitter, CSVEmitter, CountEmitter +from saucebrush.emitters import DebugEmitter, CSVEmitter, CountEmitter, SqliteEmitter class EmitterTestCase(unittest.TestCase): @@ -13,19 +14,6 @@ class EmitterTestCase(unittest.TestCase): list(de.attach([1,2,3])) self.assertEqual(output.getvalue(), '1\n2\n3\n') - def test_csv_emitter(self): - - try: - import cStringIO # if Python 2.x then use old cStringIO - io = cStringIO.StringIO() - except: - io = StringIO() # if Python 3.x then use StringIO - - with closing(io) as output: - ce = CSVEmitter(output, ('x','y','z')) - list(ce.attach([{'x':1, 'y':2, 'z':3}, {'x':5, 'y':5, 'z':5}])) - self.assertEqual(output.getvalue(), 'x,y,z\r\n1,2,3\r\n5,5,5\r\n') - def test_count_emitter(self): # values for test @@ -49,5 +37,38 @@ class EmitterTestCase(unittest.TestCase): ce.done() self.assertEqual(output.getvalue(), '10 of 22\n20 of 22\n22 of 22\n') + def test_csv_emitter(self): + + try: + import cStringIO # if Python 2.x then use old cStringIO + io = cStringIO.StringIO() + except: + io = StringIO() # if Python 3.x then use StringIO + + with closing(io) as output: + ce = CSVEmitter(output, ('x','y','z')) + list(ce.attach([{'x':1, 'y':2, 'z':3}, {'x':5, 'y':5, 'z':5}])) + self.assertEqual(output.getvalue(), 'x,y,z\r\n1,2,3\r\n5,5,5\r\n') + + def test_sqlite_emitter(self): + + import sqlite3, tempfile + + with closing(tempfile.NamedTemporaryFile(suffix='.db')) as f: + db_path = f.name + + sle = SqliteEmitter(db_path, 'testtable', fieldnames=('a','b','c')) + list(sle.attach([{'a': '1', 'b': '2', 'c': '3'}])) + sle.done() + + with closing(sqlite3.connect(db_path)) as conn: + cur = conn.cursor() + cur.execute("""SELECT a, b, c FROM testtable""") + results = cur.fetchall() + + os.unlink(db_path) + + self.assertEqual(results, [('1', '2', '3')]) + if __name__ == '__main__': unittest.main() From c7544204d83b4101aeb43adff88dc76d46ad367a Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Mon, 12 Mar 2012 15:57:57 -0700 Subject: [PATCH 16/19] add test for HtmlTableSource --- saucebrush/sources.py | 9 +++++++-- saucebrush/tests/sources.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/saucebrush/sources.py b/saucebrush/sources.py index c0d925b..a3d272b 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -4,8 +4,9 @@ All sources must implement the iterable interface and return python dictionaries. """ - +from __future__ import unicode_literals import string + from saucebrush import utils class CSVSource(object): @@ -95,7 +96,7 @@ class HtmlTableSource(object): soup = BeautifulSoup(htmlfile.read()) if isinstance(id_or_num, int): table = soup.findAll('table')[id_or_num] - elif isinstance(id_or_num, str): + else: table = soup.find('table', id=id_or_num) # skip the necessary number of rows @@ -105,9 +106,13 @@ class HtmlTableSource(object): if not fieldnames: self._fieldnames = [td.string for td in self._rows[0].findAll(('td','th'))] + skiprows += 1 else: self._fieldnames = fieldnames + # skip the necessary number of rows + self._rows = table.findAll('tr')[skiprows:] + def process_tr(self): for row in self._rows: strings = [utils.string_dig(td) for td in row.findAll('td')] diff --git a/saucebrush/tests/sources.py b/saucebrush/tests/sources.py index 82c7086..f3a07dc 100644 --- a/saucebrush/tests/sources.py +++ b/saucebrush/tests/sources.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from io import BytesIO, StringIO import unittest -from saucebrush.sources import CSVSource, FixedWidthFileSource +from saucebrush.sources import CSVSource, FixedWidthFileSource, HtmlTableSource class SourceTestCase(unittest.TestCase): @@ -44,5 +44,34 @@ class SourceTestCase(unittest.TestCase): 'year':'1999'}] self.assertEqual(list(source), expected_data) + def test_html_table_source(self): + + content = StringIO(""" + + + + + + + + + + + + +
abc
123
+ + """) + + try: + + from BeautifulSoup import BeautifulSoup + + hts = HtmlTableSource(content, 'thetable') + self.assertEqual(list(hts), [{'a': '1', 'b': '2', 'c': '3'}]) + + except ImportError: + self.skipTest("BeautifulSoup is not installed") + if __name__ == '__main__': unittest.main() From e0e4dd43e226c959469a81c32a906cb1c39dda7f Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Mon, 12 Mar 2012 16:09:15 -0700 Subject: [PATCH 17/19] switch from BeautifulSoup to lxml --- saucebrush/sources.py | 20 +++++++++++--------- saucebrush/tests/sources.py | 4 ++-- saucebrush/utils.py | 15 --------------- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/saucebrush/sources.py b/saucebrush/sources.py index a3d272b..9ebe532 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -92,30 +92,32 @@ class HtmlTableSource(object): def __init__(self, htmlfile, id_or_num, fieldnames=None, skiprows=0): # extract the table - from BeautifulSoup import BeautifulSoup - soup = BeautifulSoup(htmlfile.read()) + from lxml.html import parse + doc = parse(htmlfile).getroot() if isinstance(id_or_num, int): - table = soup.findAll('table')[id_or_num] + table = doc.cssselect('table')[id_or_num] else: - table = soup.find('table', id=id_or_num) + table = doc.cssselect('table#%s' % id_or_num) + + table = table[0] # get the first table # skip the necessary number of rows - self._rows = table.findAll('tr')[skiprows:] + self._rows = table.cssselect('tr')[skiprows:] # determine the fieldnames if not fieldnames: - self._fieldnames = [td.string - for td in self._rows[0].findAll(('td','th'))] + self._fieldnames = [td.text_content() + for td in self._rows[0].cssselect('td, th')] skiprows += 1 else: self._fieldnames = fieldnames # skip the necessary number of rows - self._rows = table.findAll('tr')[skiprows:] + self._rows = table.cssselect('tr')[skiprows:] def process_tr(self): for row in self._rows: - strings = [utils.string_dig(td) for td in row.findAll('td')] + strings = [td.text_content() for td in row.cssselect('td')] yield dict(zip(self._fieldnames, strings)) def __iter__(self): diff --git a/saucebrush/tests/sources.py b/saucebrush/tests/sources.py index f3a07dc..1db434c 100644 --- a/saucebrush/tests/sources.py +++ b/saucebrush/tests/sources.py @@ -65,13 +65,13 @@ class SourceTestCase(unittest.TestCase): try: - from BeautifulSoup import BeautifulSoup + import lxml hts = HtmlTableSource(content, 'thetable') self.assertEqual(list(hts), [{'a': '1', 'b': '2', 'c': '3'}]) except ImportError: - self.skipTest("BeautifulSoup is not installed") + self.skipTest("lxml is not installed") if __name__ == '__main__': unittest.main() diff --git a/saucebrush/utils.py b/saucebrush/utils.py index afa67dc..e30d4ef 100644 --- a/saucebrush/utils.py +++ b/saucebrush/utils.py @@ -25,21 +25,6 @@ def get_django_model(dj_settings, app_label, model_name): from django.db.models import get_model return get_model(app_label, model_name) - -def string_dig(element, separator=''): - """ - Dig into BeautifulSoup HTML elements looking for inner strings. - - If element resembled:

testtest

- then string_dig(element, '~') would return test~test - """ - if element.string: - return element.string - else: - return separator.join([string_dig(child) - for child in element.findAll(True)]) - - def flatten(item, prefix='', separator='_', keys=None): """ Flatten nested dictionary into one with its keys concatenated together. From 5ac3948ffe2bd55223f74058ef6496d3f2750369 Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Mon, 12 Mar 2012 17:09:55 -0700 Subject: [PATCH 18/19] add JSONSource test --- saucebrush/sources.py | 15 ++++++++------- saucebrush/tests/sources.py | 10 +++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/saucebrush/sources.py b/saucebrush/sources.py index 9ebe532..92844a7 100644 --- a/saucebrush/sources.py +++ b/saucebrush/sources.py @@ -226,20 +226,20 @@ class FileSource(object): def __iter__(self): # This method would be a lot cleaner with the proposed # 'yield from' expression (PEP 380) - if hasattr(self._input, '__read__'): - for record in self._process_file(input): + if hasattr(self._input, '__read__') or hasattr(self._input, 'read'): + for record in self._process_file(self._input): yield record - elif isinstance(self._input, basestring): + elif isinstance(self._input, str): with open(self._input) as f: for record in self._process_file(f): yield record elif hasattr(self._input, '__iter__'): for el in self._input: - if isinstance(el, basestring): + if isinstance(el, str): with open(el) as f: for record in self._process_file(f): yield record - elif hasattr(el, '__read__'): + elif hasattr(el, '__read__') or hasattr(el, 'read'): for record in self._process_file(f): yield record @@ -256,10 +256,11 @@ class JSONSource(FileSource): object. """ - def _process_file(self, file): + def _process_file(self, f): + import json - obj = json.load(file) + obj = json.load(f) # If the top-level JSON object in the file is a list # then yield each element separately; otherwise, yield diff --git a/saucebrush/tests/sources.py b/saucebrush/tests/sources.py index 1db434c..83c645f 100644 --- a/saucebrush/tests/sources.py +++ b/saucebrush/tests/sources.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals from io import BytesIO, StringIO import unittest -from saucebrush.sources import CSVSource, FixedWidthFileSource, HtmlTableSource +from saucebrush.sources import ( + CSVSource, FixedWidthFileSource, HtmlTableSource, JSONSource) class SourceTestCase(unittest.TestCase): @@ -44,6 +45,13 @@ class SourceTestCase(unittest.TestCase): 'year':'1999'}] self.assertEqual(list(source), expected_data) + def test_json_source(self): + + content = StringIO("""[{"a": 1, "b": "2", "c": 3}]""") + + js = JSONSource(content) + self.assertEqual(list(js), [{'a': 1, 'b': '2', 'c': 3}]) + def test_html_table_source(self): content = StringIO(""" From 05f8d4c24c77f09b173d4ef90284c35f56438edf Mon Sep 17 00:00:00 2001 From: Jeremy Carbaugh Date: Mon, 12 Mar 2012 17:39:26 -0700 Subject: [PATCH 19/19] fix SqlDumpEmitter and write basic test --- saucebrush/emitters.py | 16 ++++++++++------ saucebrush/tests/emitters.py | 14 +++++++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/saucebrush/emitters.py b/saucebrush/emitters.py index 1138581..b165407 100644 --- a/saucebrush/emitters.py +++ b/saucebrush/emitters.py @@ -180,21 +180,25 @@ class SqlDumpEmitter(Emitter): table_name, '`,`'.join(fieldnames)) def quote(self, item): + if item is None: return "null" - elif isinstance(item, (unicode, str)): + + try: + types = (basestring,) + except NameError: + types = (str,) + + if isinstance(item, types): item = item.replace("\\","\\\\").replace("'","\\'").replace(chr(0),'0') return "'%s'" % item - else: - return "%s" % item + + return "%s" % item def emit_record(self, record): quoted_data = [self.quote(record[field]) for field in self._fieldnames] self._outfile.write(self._insert_str % ','.join(quoted_data)) - def done(self): - self._outfile.close() - class DjangoModelEmitter(Emitter): """ Emitter that populates a table corresponding to a django model. diff --git a/saucebrush/tests/emitters.py b/saucebrush/tests/emitters.py index 14ec8d8..606178e 100644 --- a/saucebrush/tests/emitters.py +++ b/saucebrush/tests/emitters.py @@ -4,7 +4,8 @@ from io import StringIO import os import unittest -from saucebrush.emitters import DebugEmitter, CSVEmitter, CountEmitter, SqliteEmitter +from saucebrush.emitters import ( + DebugEmitter, CSVEmitter, CountEmitter, SqliteEmitter, SqlDumpEmitter) class EmitterTestCase(unittest.TestCase): @@ -70,5 +71,16 @@ class EmitterTestCase(unittest.TestCase): self.assertEqual(results, [('1', '2', '3')]) + def test_sql_dump_emitter(self): + + with closing(StringIO()) as bffr: + + sde = SqlDumpEmitter(bffr, 'testtable', ('a', 'b')) + list(sde.attach([{'a': 1, 'b': '2'}])) + sde.done() + + self.assertEqual(bffr.getvalue(), "INSERT INTO `testtable` (`a`,`b`) VALUES (1,'2');\n") + + if __name__ == '__main__': unittest.main()