DictFlattener

saucebrush/filters.py

@@ -314,6 +314,17 @@ class Flattener(FieldFilter):
             result.append(rec)
         return result
 
+    
+class DictFlattener(Filter):
+    def __init__(self, keys, separator='_'):
+        super(DictFlattener, self).__init__()
+        self._keys = keys
+        self._separator = separator
+    
+    def process_record(self, record):
+        utils.flatten(record, keys=self._keys, separator=self._separator)
+
+
 class Unique(ConditionalFilter):
     """ Filter that ensures that all records passing through are unique.
     """

saucebrush/utils.py

@@ -33,7 +33,7 @@ def string_dig(element, separator=''):
                             for child in element.findAll(True)])
 
 
-def recursive_flatten(item, prefix=''):
+def flatten(item, prefix='', separator='_', keys=None):
     """
         Flatten nested dictionary into one with its keys concatenated together.
         
@@ -47,14 +47,18 @@ def recursive_flatten(item, prefix=''):
     if isinstance(item, dict):
         # don't prepend a leading _
         if prefix != '':
-            prefix += '_'
+            prefix += separator
         retval = {}
         for key, value in item.iteritems():
-            retval.update(recursive_flatten(value, prefix + key))
+            if (not keys) or (key in keys):
-        return retval
+                retval.update(flatten(value, prefix + key, separator, keys))
-    elif isinstance(item, (tuple, list)):
-        return {prefix: [recursive_flatten(i) for i in item]}
             else:
+                retval[prefix + key] = value
+        return retval
+    #elif isinstance(item, (tuple, list)):
+    #    return {prefix: [flatten(i, prefix, separator, keys) for i in item]}
+    else:
+        print item, prefix
         return {prefix: item}
 
 def dotted_key_lookup(dict_, dotted_key, default=KeyError, separator='.'):