diff --git a/Changelog.rst b/Changelog.rst index d0fb541a..f9ef3a6d 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -16,6 +16,9 @@ Changelog * changed `perform_request` on `Connection` classes to return headers as well. This is a backwards incompatible change for people who have developed their own connection class. + * changed deserialization mechanics. Users who provided their own serializer + that didn't extend `JSONSerializer` need to specify a `mimetype` class + attribute. 0.4.3 ----- diff --git a/elasticsearch/serializer.py b/elasticsearch/serializer.py index 970db113..23f7e890 100644 --- a/elasticsearch/serializer.py +++ b/elasticsearch/serializer.py @@ -2,9 +2,23 @@ import json from datetime import date, datetime from decimal import Decimal -from .exceptions import SerializationError +from .exceptions import SerializationError, ImproperlyConfigured + +class TextSerializer(object): + mimetype = 'text/plain' + + def loads(self, s): + return s + + def dumps(self, data): + if isinstance(data, (type(''), type(u''))): + return data + + raise SerializationError('Cannot serialize %r into text.' % data) class JSONSerializer(object): + mimetype = 'application/json' + def default(self, data): if isinstance(data, (date, datetime)): return data.isoformat() @@ -28,4 +42,29 @@ class JSONSerializer(object): except (ValueError, TypeError) as e: raise SerializationError(data, e) +DEFAULT_SERIALIZERS = { + JSONSerializer.mimetype: JSONSerializer(), + TextSerializer.mimetype: TextSerializer(), +} + +class Deserializer(object): + def __init__(self, serializers, default_mimetype='application/json'): + try: + self.default = serializers[default_mimetype] + except KeyError: + raise ImproperlyConfigured('Cannot find default serializer (%s)' % default_mimetype) + self.serializers = serializers + + def loads(self, s, mimetype=None): + if not mimetype: + deserializer = self.default + else: + # split out charset + mimetype = mimetype.split(';', 1)[0] + try: + deserializer = self.serializers[mimetype] + except KeyError: + raise SerializationError('Unknown mimetype, unable to deserialize: %s' % mimetype) + + return deserializer.loads(s) diff --git a/elasticsearch/transport.py b/elasticsearch/transport.py index 10591d15..3b5a7abc 100644 --- a/elasticsearch/transport.py +++ b/elasticsearch/transport.py @@ -3,7 +3,7 @@ import time from .connection import Urllib3HttpConnection from .connection_pool import ConnectionPool -from .serializer import JSONSerializer +from .serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS from .exceptions import ConnectionError, TransportError, SerializationError # get ip/port from "inet[wind/127.0.0.1:9200]" @@ -35,8 +35,9 @@ class Transport(object): def __init__(self, hosts, connection_class=Urllib3HttpConnection, connection_pool_class=ConnectionPool, host_info_callback=get_host_info, sniff_on_start=False, sniffer_timeout=None, - sniff_on_connection_fail=False, serializer=JSONSerializer(), - max_retries=3, send_get_body_as='GET', **kwargs): + sniff_on_connection_fail=False, serializer=JSONSerializer(), serializers=None, + default_mimetype='application/json', max_retries=3, + send_get_body_as='GET', **kwargs): """ :arg hosts: list of dictionaries, each containing keyword arguments to create a `connection_class` instance @@ -50,6 +51,10 @@ class Transport(object): :arg sniffer_timeout: number of seconds between automatic sniffs :arg sniff_on_connection_fail: flag controlling if connection failure triggers a sniff :arg serializer: serializer instance + :arg serializers: optional dict of serializer instances that will be + used for deserializing data coming from the server. (key is the mimetype) + :arg default_mimetype: when no mimetype is specified by the server + response assume this mimetype, defaults to `'application/json'` :arg max_retries: maximum number of retries before an exception is propagated :arg send_get_body_as: for GET requests with body this option allows you to specify an alternate way of execution for environments that @@ -62,6 +67,16 @@ class Transport(object): options provided as part of the hosts parameter. """ + # serialization config + _serializers = DEFAULT_SERIALIZERS.copy() + # if a serializer has been specified, use it for deserialization as well + _serializers[serializer.mimetype] = serializer + # if custom serializers map has been supplied, override the defaults with it + if serializers: + _serializers.update(serializers) + # create a deserializer with our config + self.deserializer = Deserializer(_serializers, default_mimetype) + self.max_retries = max_retries self.send_get_body_as = send_get_body_as @@ -155,7 +170,7 @@ class Transport(object): try: # use small timeout for the sniffing request, should be a fast api call _, headers, node_info = c.perform_request('GET', '/_cluster/nodes', timeout=.1) - node_info = self.serializer.loads(node_info) + node_info = self.deserializer.loads(node_info, headers.get('content-type')) break except (ConnectionError, SerializationError): pass @@ -263,6 +278,6 @@ class Transport(object): self.connection_pool.mark_live(connection) data = None if raw_data: - data = self.serializer.loads(raw_data) + data = self.deserializer.loads(raw_data, headers.get('content-type')) return status, data diff --git a/test_elasticsearch/test_serializer.py b/test_elasticsearch/test_serializer.py index 7a0f1d8f..3416e518 100644 --- a/test_elasticsearch/test_serializer.py +++ b/test_elasticsearch/test_serializer.py @@ -4,8 +4,8 @@ import sys from datetime import datetime from decimal import Decimal -from elasticsearch.serializer import JSONSerializer -from elasticsearch.exceptions import SerializationError +from elasticsearch.serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS, TextSerializer +from elasticsearch.exceptions import SerializationError, ImproperlyConfigured from .test_cases import TestCase, SkipTest @@ -28,3 +28,30 @@ class TestJSONSerializer(TestCase): def test_strings_are_left_untouched(self): self.assertEquals(u"你好", JSONSerializer().dumps(u"你好")) + + +class TestTextSerializer(TestCase): + def test_strings_are_left_untouched(self): + self.assertEquals(u"你好", TextSerializer().dumps(u"你好")) + + def test_raises_serialization_error_on_dump_error(self): + self.assertRaises(SerializationError, TextSerializer().dumps, {}) + + +class TestDeserializer(TestCase): + def setUp(self): + super(TestDeserializer, self).setUp() + self.de = Deserializer(DEFAULT_SERIALIZERS) + + def test_deserializes_json_by_default(self): + self.assertEquals({"some": "data"}, self.de.loads('{"some":"data"}')) + + def test_deserializes_text_with_correct_ct(self): + self.assertEquals('{"some":"data"}', self.de.loads('{"some":"data"}', 'text/plain')) + self.assertEquals('{"some":"data"}', self.de.loads('{"some":"data"}', 'text/plain; charset=whatever')) + + def test_raises_serialization_error_on_unknown_mimetype(self): + self.assertRaises(SerializationError, self.de.loads, '{}', 'text/html') + + def test_raises_improperly_configured_when_default_mimetype_cannot_be_deserialized(self): + self.assertRaises(ImproperlyConfigured, Deserializer, {})