Enable deserialization of different content types coming from the server.

Changelog.rst

@@ -16,6 +16,9 @@ Changelog
  * changed `perform_request` on `Connection` classes to return headers as well.
    This is a backwards incompatible change for people who have developed their own
    connection class.
+ * changed deserialization mechanics. Users who provided their own serializer
+   that didn't extend `JSONSerializer` need to specify a `mimetype` class
+   attribute.
 
 0.4.3
 -----

elasticsearch/serializer.py

@@ -2,9 +2,23 @@ import json
 from datetime import date, datetime
 from decimal import Decimal
 
-from .exceptions import SerializationError
+from .exceptions import SerializationError, ImproperlyConfigured
+
+class TextSerializer(object):
+    mimetype = 'text/plain'
+
+    def loads(self, s):
+        return s
+
+    def dumps(self, data):
+        if isinstance(data, (type(''), type(u''))):
+            return data
+
+        raise SerializationError('Cannot serialize %r into text.' % data)
 
 class JSONSerializer(object):
+    mimetype = 'application/json'
+
     def default(self, data):
         if isinstance(data, (date, datetime)):
             return data.isoformat()
@@ -28,4 +42,29 @@ class JSONSerializer(object):
         except (ValueError, TypeError) as e:
             raise SerializationError(data, e)
 
+DEFAULT_SERIALIZERS = {
+    JSONSerializer.mimetype: JSONSerializer(),
+    TextSerializer.mimetype: TextSerializer(),
+}
+
+class Deserializer(object):
+    def __init__(self, serializers, default_mimetype='application/json'):
+        try:
+            self.default = serializers[default_mimetype]
+        except KeyError:
+            raise ImproperlyConfigured('Cannot find default serializer (%s)' % default_mimetype)
+        self.serializers = serializers
+
+    def loads(self, s, mimetype=None):
+        if not mimetype:
+            deserializer = self.default
+        else:
+            # split out charset
+            mimetype = mimetype.split(';', 1)[0]
+            try:
+                deserializer = self.serializers[mimetype]
+            except KeyError:
+                raise SerializationError('Unknown mimetype, unable to deserialize: %s' % mimetype)
+
+        return deserializer.loads(s)
 

elasticsearch/transport.py

@@ -3,7 +3,7 @@ import time
 
 from .connection import Urllib3HttpConnection
 from .connection_pool import ConnectionPool
-from .serializer import JSONSerializer
+from .serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS
 from .exceptions import ConnectionError, TransportError, SerializationError
 
 # get ip/port from "inet[wind/127.0.0.1:9200]"
@@ -35,8 +35,9 @@ class Transport(object):
     def __init__(self, hosts, connection_class=Urllib3HttpConnection,
         connection_pool_class=ConnectionPool, host_info_callback=get_host_info,
         sniff_on_start=False, sniffer_timeout=None,
-        sniff_on_connection_fail=False, serializer=JSONSerializer(),
+        sniff_on_connection_fail=False, serializer=JSONSerializer(), serializers=None,
-        max_retries=3, send_get_body_as='GET', **kwargs):
+        default_mimetype='application/json', max_retries=3,
+        send_get_body_as='GET', **kwargs):
         """
         :arg hosts: list of dictionaries, each containing keyword arguments to
             create a `connection_class` instance
@@ -50,6 +51,10 @@ class Transport(object):
         :arg sniffer_timeout: number of seconds between automatic sniffs
         :arg sniff_on_connection_fail: flag controlling if connection failure triggers a sniff
         :arg serializer: serializer instance
+        :arg serializers: optional dict of serializer instances that will be
+            used for deserializing data coming from the server. (key is the mimetype)
+        :arg default_mimetype: when no mimetype is specified by the server
+            response assume this mimetype, defaults to `'application/json'`
         :arg max_retries: maximum number of retries before an exception is propagated
         :arg send_get_body_as: for GET requests with body this option allows
             you to specify an alternate way of execution for environments that
@@ -62,6 +67,16 @@ class Transport(object):
         options provided as part of the hosts parameter.
         """
 
+        # serialization config
+        _serializers = DEFAULT_SERIALIZERS.copy()
+        # if a serializer has been specified, use it for deserialization as well
+        _serializers[serializer.mimetype] = serializer
+        # if custom serializers map has been supplied, override the defaults with it
+        if serializers:
+            _serializers.update(serializers)
+        # create a deserializer with our config
+        self.deserializer = Deserializer(_serializers, default_mimetype)
+
         self.max_retries = max_retries
         self.send_get_body_as = send_get_body_as
 
@@ -155,7 +170,7 @@ class Transport(object):
                 try:
                     # use small timeout for the sniffing request, should be a fast api call
                     _, headers, node_info = c.perform_request('GET', '/_cluster/nodes', timeout=.1)
-                    node_info = self.serializer.loads(node_info)
+                    node_info = self.deserializer.loads(node_info, headers.get('content-type'))
                     break
                 except (ConnectionError, SerializationError):
                     pass
@@ -263,6 +278,6 @@ class Transport(object):
                 self.connection_pool.mark_live(connection)
                 data = None
                 if raw_data:
-                    data = self.serializer.loads(raw_data)
+                    data = self.deserializer.loads(raw_data, headers.get('content-type'))
                 return status, data
 

test_elasticsearch/test_serializer.py

@@ -4,8 +4,8 @@ import sys
 from datetime import datetime
 from decimal import Decimal
 
-from elasticsearch.serializer import JSONSerializer
+from elasticsearch.serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS, TextSerializer
-from elasticsearch.exceptions import SerializationError
+from elasticsearch.exceptions import SerializationError, ImproperlyConfigured
 
 from .test_cases import TestCase, SkipTest
 
@@ -28,3 +28,30 @@ class TestJSONSerializer(TestCase):
 
     def test_strings_are_left_untouched(self):
         self.assertEquals(u"你好", JSONSerializer().dumps(u"你好"))
+
+
+class TestTextSerializer(TestCase):
+    def test_strings_are_left_untouched(self):
+        self.assertEquals(u"你好", TextSerializer().dumps(u"你好"))
+
+    def test_raises_serialization_error_on_dump_error(self):
+        self.assertRaises(SerializationError, TextSerializer().dumps, {})
+
+
+class TestDeserializer(TestCase):
+    def setUp(self):
+        super(TestDeserializer, self).setUp()
+        self.de = Deserializer(DEFAULT_SERIALIZERS)
+
+    def test_deserializes_json_by_default(self):
+        self.assertEquals({"some": "data"}, self.de.loads('{"some":"data"}'))
+
+    def test_deserializes_text_with_correct_ct(self):
+        self.assertEquals('{"some":"data"}', self.de.loads('{"some":"data"}', 'text/plain'))
+        self.assertEquals('{"some":"data"}', self.de.loads('{"some":"data"}', 'text/plain; charset=whatever'))
+
+    def test_raises_serialization_error_on_unknown_mimetype(self):
+        self.assertRaises(SerializationError, self.de.loads, '{}', 'text/html')
+
+    def test_raises_improperly_configured_when_default_mimetype_cannot_be_deserialized(self):
+        self.assertRaises(ImproperlyConfigured, Deserializer, {})