2022-11-11 03:31:18 +00:00
|
|
|
from io import StringIO
|
2010-02-21 18:44:39 +00:00
|
|
|
import unittest
|
2012-03-12 06:14:39 +00:00
|
|
|
|
2012-03-13 00:09:55 +00:00
|
|
|
from saucebrush.sources import (
|
2022-11-11 03:26:09 +00:00
|
|
|
CSVSource,
|
|
|
|
|
FixedWidthFileSource,
|
|
|
|
|
HtmlTableSource,
|
|
|
|
|
JSONSource,
|
|
|
|
|
)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
|
|
|
|
|
2022-11-11 03:26:09 +00:00
|
|
|
class SourceTestCase(unittest.TestCase):
|
2010-02-21 18:44:39 +00:00
|
|
|
def _get_csv(self):
|
2022-11-11 03:26:09 +00:00
|
|
|
data = """a,b,c
|
2010-02-21 18:44:39 +00:00
|
|
|
1,2,3
|
|
|
|
|
5,5,5
|
2022-11-11 03:26:09 +00:00
|
|
|
1,10,100"""
|
2012-03-12 06:14:39 +00:00
|
|
|
return StringIO(data)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
|
|
|
|
def test_csv_source_basic(self):
|
|
|
|
|
source = CSVSource(self._get_csv())
|
2022-11-11 03:26:09 +00:00
|
|
|
expected_data = [
|
|
|
|
|
{"a": "1", "b": "2", "c": "3"},
|
|
|
|
|
{"a": "5", "b": "5", "c": "5"},
|
|
|
|
|
{"a": "1", "b": "10", "c": "100"},
|
|
|
|
|
]
|
2012-03-12 06:14:39 +00:00
|
|
|
self.assertEqual(list(source), expected_data)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
|
|
|
|
def test_csv_source_fieldnames(self):
|
2022-11-11 03:26:09 +00:00
|
|
|
source = CSVSource(self._get_csv(), ["x", "y", "z"])
|
|
|
|
|
expected_data = [
|
|
|
|
|
{"x": "a", "y": "b", "z": "c"},
|
|
|
|
|
{"x": "1", "y": "2", "z": "3"},
|
|
|
|
|
{"x": "5", "y": "5", "z": "5"},
|
|
|
|
|
{"x": "1", "y": "10", "z": "100"},
|
|
|
|
|
]
|
2012-03-12 06:14:39 +00:00
|
|
|
self.assertEqual(list(source), expected_data)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
|
|
|
|
def test_csv_source_skiprows(self):
|
|
|
|
|
source = CSVSource(self._get_csv(), skiprows=1)
|
2022-11-11 03:26:09 +00:00
|
|
|
expected_data = [
|
|
|
|
|
{"a": "5", "b": "5", "c": "5"},
|
|
|
|
|
{"a": "1", "b": "10", "c": "100"},
|
|
|
|
|
]
|
2012-03-12 06:14:39 +00:00
|
|
|
self.assertEqual(list(source), expected_data)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
|
|
|
|
def test_fixed_width_source(self):
|
2022-11-11 03:26:09 +00:00
|
|
|
data = StringIO("JamesNovember 3 1986\nTim September151999")
|
|
|
|
|
fields = (("name", 5), ("month", 9), ("day", 2), ("year", 4))
|
2010-02-21 18:44:39 +00:00
|
|
|
source = FixedWidthFileSource(data, fields)
|
2022-11-11 03:26:09 +00:00
|
|
|
expected_data = [
|
|
|
|
|
{"name": "James", "month": "November", "day": "3", "year": "1986"},
|
|
|
|
|
{"name": "Tim", "month": "September", "day": "15", "year": "1999"},
|
|
|
|
|
]
|
2012-03-12 06:14:39 +00:00
|
|
|
self.assertEqual(list(source), expected_data)
|
2010-02-21 18:44:39 +00:00
|
|
|
|
2012-03-13 00:09:55 +00:00
|
|
|
def test_json_source(self):
|
|
|
|
|
|
|
|
|
|
content = StringIO("""[{"a": 1, "b": "2", "c": 3}]""")
|
|
|
|
|
|
|
|
|
|
js = JSONSource(content)
|
2022-11-11 03:26:09 +00:00
|
|
|
self.assertEqual(list(js), [{"a": 1, "b": "2", "c": 3}])
|
2012-03-13 00:09:55 +00:00
|
|
|
|
2012-03-12 22:57:57 +00:00
|
|
|
def test_html_table_source(self):
|
|
|
|
|
|
2022-11-11 03:26:09 +00:00
|
|
|
content = StringIO(
|
|
|
|
|
"""
|
2012-03-12 22:57:57 +00:00
|
|
|
<html>
|
|
|
|
|
<table id="thetable">
|
|
|
|
|
<tr>
|
|
|
|
|
<th>a</th>
|
|
|
|
|
<th>b</th>
|
|
|
|
|
<th>c</th>
|
|
|
|
|
</tr>
|
|
|
|
|
<tr>
|
|
|
|
|
<td>1</td>
|
|
|
|
|
<td>2</td>
|
|
|
|
|
<td>3</td>
|
|
|
|
|
</tr>
|
|
|
|
|
</table>
|
|
|
|
|
</html>
|
2022-11-11 03:26:09 +00:00
|
|
|
"""
|
|
|
|
|
)
|
2012-03-12 22:57:57 +00:00
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
2022-11-11 03:26:09 +00:00
|
|
|
hts = HtmlTableSource(content, "thetable")
|
|
|
|
|
self.assertEqual(list(hts), [{"a": "1", "b": "2", "c": "3"}])
|
2012-03-12 22:57:57 +00:00
|
|
|
|
|
|
|
|
except ImportError:
|
2012-03-14 06:48:53 +00:00
|
|
|
# Python 2.6 doesn't have skipTest. We'll just suffer without it.
|
2022-11-11 03:26:09 +00:00
|
|
|
if hasattr(self, "skipTest"):
|
2012-03-14 06:48:53 +00:00
|
|
|
self.skipTest("lxml is not installed")
|
2012-03-12 22:57:57 +00:00
|
|
|
|
2022-11-11 03:26:09 +00:00
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2010-02-21 18:44:39 +00:00
|
|
|
unittest.main()
|
