Merge branch 'EvaSDK/allow-formatter-customizations'

Conflicts:
	README.md

Author: yyuu

.travis.yml

@@ -1,6 +1,5 @@
 language: python
 python:
-  - "2.6"
   - "2.7"
   - "3.2"
   - "3.3"

README.md

@@ -76,7 +76,9 @@ from fluent import handler
 
 logging.basicConfig(level=logging.INFO)
 l = logging.getLogger('fluent.test')
-l.addHandler(handler.FluentHandler('app.follow', host='host', port=24224))
+h = handler.FluentHandler('app.follow', host='host', port=24224)
+h.setFormatter(handler.FluentRecordFormatter())
+l.addHandler(h)
 l.info({
   'from': 'userA',
   'to': 'userB'

fluent/handler.py

@@ -2,6 +2,7 @@
 
 import logging
 import socket
+import sys
 
 try:
     import simplejson as json
@@ -16,35 +17,65 @@
 from fluent import sender
 
 
-class FluentRecordFormatter(object):
-    def __init__(self):
+class FluentRecordFormatter(logging.Formatter, object):
+    """ A structured formatter for Fluent.
+
+    Best used with server storing data in an ElasticSearch cluster for example.
+
+    :param fmt: a dict with format string as values to map to provided keys.
+    """
+    def __init__(self, fmt=None, datefmt=None):
+        super(FluentRecordFormatter, self).__init__(None, datefmt)
+
+        if not fmt:
+            self._fmt_dict = {
+                'sys_host': '%(hostname)s',
+                'sys_name': '%(name)s',
+                'sys_module': '%(module)s',
+            }
+        else:
+            self._fmt_dict = fmt
+
         self.hostname = socket.gethostname()
 
     def format(self, record):
-        data = {'sys_host': self.hostname,
-                'sys_name': record.name,
-                'sys_module': record.module,
-                # 'sys_lineno': record.lineno,
-                # 'sys_levelno': record.levelno,
-                # 'sys_levelname': record.levelname,
-                # 'sys_filename': record.filename,
-                # 'sys_funcname': record.funcName,
-                # 'sys_exc_info': record.exc_info,
-                }
-        # if 'sys_exc_info' in data and data['sys_exc_info']:
-        #    data['sys_exc_info'] = self.formatException(data['sys_exc_info'])
+        # Only needed for python2.6
+        if sys.version_info[0:2] <= (2, 6) and self.usesTime():
+            record.asctime = self.formatTime(record, self.datefmt)
+
+        # Compute attributes handled by parent class.
+        super(FluentRecordFormatter, self).format(record)
+        # Add ours
+        record.hostname = self.hostname
+        # Apply format
+        data = dict([(key, value % record.__dict__)
+                     for key, value in self._fmt_dict.items()])
 
         self._structuring(data, record.msg)
         return data
 
+    def usesTime(self):
+        return any([value.find('%(asctime)') >= 0
+                    for value in self._fmt_dict.values()])
+
     def _structuring(self, data, msg):
+        """ Melds `msg` into `data`.
+
+        :param data: dictionary to be sent to fluent server
+        :param msg: :class:`LogRecord`'s message to add to `data`.
+          `msg` can be a simple string for backward compatibility with
+          :mod:`logging` framework, a JSON encoded string or a dictionary
+          that will be merged into dictionary generated in :meth:`format.
+        """
         if isinstance(msg, dict):
             self._add_dic(data, msg)
-        elif isinstance(msg, str):
+        elif isinstance(msg, basestring):
             try:
                 self._add_dic(data, json.loads(str(msg)))
             except ValueError:
-                pass
+                self._add_dic(data, {'message': msg})
+        else:
+            self._add_dic(data, {'message': msg})
 
     @staticmethod
     def _add_dic(data, dic):

setup.py

@@ -12,7 +12,7 @@
 
 setup(
   name='fluent-logger',
-  version='0.3.5',
+  version='0.4.0.dev',
   description=desc,
   long_description=open(README).read(),
   package_dir={'fluent': 'fluent'},

tests/test_handler.py

@@ -44,3 +44,80 @@ def test_simple(self):
         eq('userB', data[0][2]['to'])
         self.assertTrue(data[0][1])
         self.assertTrue(isinstance(data[0][1], int))
+
+    def test_custom_fmt(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(
+            fluent.handler.FluentRecordFormatter(fmt={
+                'name': '%(name)s',
+                'lineno': '%(lineno)d',
+                'emitted_at': '%(asctime)s',
+            })
+        )
+        log.addHandler(handler)
+        log.info({'sample': 'value'})
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('name' in data[0][2])
+        self.assertEqual('fluent.test', data[0][2]['name'])
+        self.assertTrue('lineno' in data[0][2])
+        self.assertTrue('emitted_at' in data[0][2])
+
+    def test_json_encoded_message(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(fluent.handler.FluentRecordFormatter())
+        log.addHandler(handler)
+        log.info('{"key": "hello world!", "param": "value"}')
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('key' in data[0][2])
+        self.assertEqual('hello world!', data[0][2]['key'])
+
+    def test_unstructured_message(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(fluent.handler.FluentRecordFormatter())
+        log.addHandler(handler)
+        log.info('hello world')
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('message' in data[0][2])
+        self.assertEqual('hello world', data[0][2]['message'])
+
+    def test_non_string_simple_message(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(fluent.handler.FluentRecordFormatter())
+        log.addHandler(handler)
+        log.info(42)
+        handler.close()
+
+        data = self.get_data()
+        self.assertTrue('message' in data[0][2])
+
+    def test_non_string_dict_message(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(fluent.handler.FluentRecordFormatter())
+        log.addHandler(handler)
+        log.info({42: 'root'})
+        handler.close()
+
+        data = self.get_data()
+        # For some reason, non-string keys are ignored
+        self.assertFalse(42 in data[0][2])