From a54d7035bb6989d1111ba86b7be448b50a1cf160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Honza=20Kr=C3=A1l?= Date: Fri, 21 Feb 2014 16:53:56 +0100 Subject: [PATCH] Compatibility with python 3.2 Now all the python 2/3 compatibility code is in elasticsearch.compat Fixes #52 --- elasticsearch/client/__init__.py | 7 ++++--- elasticsearch/client/utils.py | 11 +++-------- elasticsearch/compat.py | 14 ++++++++++++++ elasticsearch/connection/http_requests.py | 5 +---- elasticsearch/connection/http_urllib3.py | 5 +---- elasticsearch/connection/memcached.py | 5 +---- elasticsearch/helpers.py | 5 +---- elasticsearch/serializer.py | 5 +++-- test_elasticsearch/test_client/__init__.py | 3 ++- test_elasticsearch/test_client/test_utils.py | 7 ++++--- test_elasticsearch/test_connection.py | 4 ++-- test_elasticsearch/test_serializer.py | 8 ++++---- test_elasticsearch/test_server/test_common.py | 7 ++++--- test_elasticsearch/test_server/test_memcached.py | 5 +++-- test_elasticsearch/test_transport.py | 3 ++- 15 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 elasticsearch/compat.py diff --git a/elasticsearch/client/__init__.py b/elasticsearch/client/__init__.py index abc9cd8c..d5702943 100644 --- a/elasticsearch/client/__init__.py +++ b/elasticsearch/client/__init__.py @@ -3,6 +3,7 @@ import logging from ..transport import Transport from ..exceptions import NotFoundError, TransportError +from ..compat import string_types from .indices import IndicesClient from .cluster import ClusterClient from .cat import CatClient @@ -22,13 +23,13 @@ def _normalize_hosts(hosts): return [{}] # passed in just one string - if isinstance(hosts, (type(''), type(u''))): + if isinstance(hosts, string_types): hosts = [hosts] out = [] # normalize hosts to dicts for i, host in enumerate(hosts): - if isinstance(host, (type(''), type(u''))): + if isinstance(host, string_types): host = host.strip('/') # remove schema information if '://' in host: @@ -130,7 +131,7 @@ class Elasticsearch(object): def _bulk_body(self, body): # if not passed in a string, serialize items and join by newline - if not isinstance(body, (type(''), type(u''))): + if not isinstance(body, string_types): body = '\n'.join(map(self.transport.serializer.dumps, body)) # bulk body must end with a newline diff --git a/elasticsearch/client/utils.py b/elasticsearch/client/utils.py index 6d9a1d68..c2755fb9 100644 --- a/elasticsearch/client/utils.py +++ b/elasticsearch/client/utils.py @@ -1,11 +1,6 @@ from datetime import date, datetime from functools import wraps -try: - # PY2 - from urllib import quote_plus -except ImportError: - # PY3 - from urllib.parse import quote_plus +from ..compat import string_types, quote_plus, u # parts of URL to be omitted SKIP_IN_PATH = (None, '', [], ()) @@ -18,7 +13,7 @@ def _escape(value): # make sequences into comma-separated stings if isinstance(value, (list, tuple)): - value = u','.join(value) + value = u(',').join(value) # dates and datetimes into isoformat elif isinstance(value, (date, datetime)): @@ -29,7 +24,7 @@ def _escape(value): value = str(value).lower() # encode strings to utf-8 - if isinstance(value, (type(''), type(u''))): + if isinstance(value, string_types): try: return value.encode('utf-8') except UnicodeDecodeError: diff --git a/elasticsearch/compat.py b/elasticsearch/compat.py new file mode 100644 index 00000000..0c3e628d --- /dev/null +++ b/elasticsearch/compat.py @@ -0,0 +1,14 @@ +import sys + +PY2 = sys.version_info[0] == 2 + +if PY2: + u = lambda s: s.decode('utf-8') + string_types = basestring, + from urllib import quote_plus, urlencode + from itertools import imap as map +else: + u = str + string_types = str, bytes + from urllib.parse import quote_plus, urlencode + map = map diff --git a/elasticsearch/connection/http_requests.py b/elasticsearch/connection/http_requests.py index 53a75496..fc702bb6 100644 --- a/elasticsearch/connection/http_requests.py +++ b/elasticsearch/connection/http_requests.py @@ -4,13 +4,10 @@ try: REQUESTS_AVAILABLE = True except ImportError: REQUESTS_AVAILABLE = False -try: - from urllib import urlencode -except ImportError: - from urllib.parse import urlencode from .base import Connection from ..exceptions import ConnectionError, ImproperlyConfigured +from ..compat import urlencode class RequestsHttpConnection(Connection): """ diff --git a/elasticsearch/connection/http_urllib3.py b/elasticsearch/connection/http_urllib3.py index f41d24bc..9f7b38e0 100644 --- a/elasticsearch/connection/http_urllib3.py +++ b/elasticsearch/connection/http_urllib3.py @@ -1,12 +1,9 @@ import time import urllib3 -try: - from urllib import urlencode -except ImportError: - from urllib.parse import urlencode from .base import Connection from ..exceptions import ConnectionError +from ..compat import urlencode class Urllib3HttpConnection(Connection): """ diff --git a/elasticsearch/connection/memcached.py b/elasticsearch/connection/memcached.py index 9f2ef236..bc1ff1e3 100644 --- a/elasticsearch/connection/memcached.py +++ b/elasticsearch/connection/memcached.py @@ -1,11 +1,8 @@ import time import json -try: - from urllib import urlencode -except ImportError: - from urllib.parse import urlencode from ..exceptions import TransportError, ConnectionError, ImproperlyConfigured +from ..compat import urlencode from .pooling import PoolingConnection class MemcachedConnection(PoolingConnection): diff --git a/elasticsearch/helpers.py b/elasticsearch/helpers.py index b789ca43..451126c0 100644 --- a/elasticsearch/helpers.py +++ b/elasticsearch/helpers.py @@ -1,11 +1,8 @@ from itertools import islice from operator import methodcaller -try: - from itertools import imap as map -except ImportError: - pass # python 3, use builtin map from elasticsearch.exceptions import ElasticsearchException +from elasticsearch.compat import map class BulkIndexError(ElasticsearchException): @property diff --git a/elasticsearch/serializer.py b/elasticsearch/serializer.py index 23f7e890..0bc2c5b9 100644 --- a/elasticsearch/serializer.py +++ b/elasticsearch/serializer.py @@ -3,6 +3,7 @@ from datetime import date, datetime from decimal import Decimal from .exceptions import SerializationError, ImproperlyConfigured +from .compat import string_types class TextSerializer(object): mimetype = 'text/plain' @@ -11,7 +12,7 @@ class TextSerializer(object): return s def dumps(self, data): - if isinstance(data, (type(''), type(u''))): + if isinstance(data, string_types): return data raise SerializationError('Cannot serialize %r into text.' % data) @@ -34,7 +35,7 @@ class JSONSerializer(object): def dumps(self, data): # don't serialize strings - if isinstance(data, (type(''), type(u''))): + if isinstance(data, string_types): return data try: diff --git a/test_elasticsearch/test_client/__init__.py b/test_elasticsearch/test_client/__init__.py index 7b9c68ed..356c3e45 100644 --- a/test_elasticsearch/test_client/__init__.py +++ b/test_elasticsearch/test_client/__init__.py @@ -1,6 +1,7 @@ from mock import patch from elasticsearch.client import _normalize_hosts, Elasticsearch +from elasticsearch.compat import u from ..test_cases import TestCase, ElasticsearchTestCase @@ -14,7 +15,7 @@ class TestNormalizeHosts(TestCase): def test_strings_are_parsed_for_port(self): self.assertEquals( [{"host": "elasticsearch.org", "port": 42}, {"host": "user:secret@elasticsearch.com"}], - _normalize_hosts(["elasticsearch.org:42", u"user:secret@elasticsearch.com"]) + _normalize_hosts(["elasticsearch.org:42", u("user:secret@elasticsearch.com")]) ) def test_dicts_are_left_unchanged(self): diff --git a/test_elasticsearch/test_client/test_utils.py b/test_elasticsearch/test_client/test_utils.py index bf42278b..a3040be9 100644 --- a/test_elasticsearch/test_client/test_utils.py +++ b/test_elasticsearch/test_client/test_utils.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- from elasticsearch.client.utils import _make_path +from elasticsearch.compat import PY2, u from ..test_cases import TestCase, SkipTest class TestMakePath(TestCase): def test_handles_unicode(self): - id = u"中文" + id = u("中文") self.assertEquals('/some-index/type/%E4%B8%AD%E6%96%87', _make_path('some-index', 'type', id)) def test_handles_utf_encoded_string(self): - if type('') is type(u''): + if not PY2: raise SkipTest('Only relevant for py2') - id = u"中文".encode('utf-8') + id = u("中文").encode('utf-8') self.assertEquals('/some-index/type/%E4%B8%AD%E6%96%87', _make_path('some-index', 'type', id)) diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index 52a977b9..ce836c00 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -54,7 +54,7 @@ class TestUrllib3Connection(TestCase): self.assertIsInstance(con.pool, urllib3.HTTPConnectionPool) class TestRequestsConnection(TestCase): - def _get_mock_connection(self, connection_params={}, status_code=200, response_body=u'{}'): + def _get_mock_connection(self, connection_params={}, status_code=200, response_body='{}'): con = RequestsHttpConnection(**connection_params) def _dummy_send(*args, **kwargs): dummy_response = Mock() @@ -71,7 +71,7 @@ class TestRequestsConnection(TestCase): def _get_request(self, connection, *args, **kwargs): status, headers, data = connection.perform_request(*args, **kwargs) self.assertEquals(200, status) - self.assertEquals(u'{}', data) + self.assertEquals('{}', data) timeout = kwargs.pop('timeout', connection.timeout) args, kwargs = connection.session.send.call_args diff --git a/test_elasticsearch/test_serializer.py b/test_elasticsearch/test_serializer.py index 3416e518..b7975050 100644 --- a/test_elasticsearch/test_serializer.py +++ b/test_elasticsearch/test_serializer.py @@ -11,12 +11,12 @@ from .test_cases import TestCase, SkipTest class TestJSONSerializer(TestCase): def test_datetime_serialization(self): - self.assertEquals(u'{"d": "2010-10-01T02:30:00"}', JSONSerializer().dumps({'d': datetime(2010, 10, 1, 2, 30)})) + self.assertEquals('{"d": "2010-10-01T02:30:00"}', JSONSerializer().dumps({'d': datetime(2010, 10, 1, 2, 30)})) def test_decimal_serialization(self): if sys.version_info[:2] == (2, 6): raise SkipTest("Float rounding is broken in 2.6.") - self.assertEquals(u'{"d": 3.8}', JSONSerializer().dumps({'d': Decimal('3.8')})) + self.assertEquals('{"d": 3.8}', JSONSerializer().dumps({'d': Decimal('3.8')})) def test_raises_serialization_error_on_dump_error(self): self.assertRaises(SerializationError, JSONSerializer().dumps, object()) @@ -27,12 +27,12 @@ class TestJSONSerializer(TestCase): self.assertRaises(SerializationError, JSONSerializer().loads, '{{') def test_strings_are_left_untouched(self): - self.assertEquals(u"你好", JSONSerializer().dumps(u"你好")) + self.assertEquals("你好", JSONSerializer().dumps("你好")) class TestTextSerializer(TestCase): def test_strings_are_left_untouched(self): - self.assertEquals(u"你好", TextSerializer().dumps(u"你好")) + self.assertEquals("你好", TextSerializer().dumps("你好")) def test_raises_serialization_error_on_dump_error(self): self.assertRaises(SerializationError, TextSerializer().dumps, {}) diff --git a/test_elasticsearch/test_server/test_common.py b/test_elasticsearch/test_server/test_common.py index 20ad6928..e842a027 100644 --- a/test_elasticsearch/test_server/test_common.py +++ b/test_elasticsearch/test_server/test_common.py @@ -9,6 +9,7 @@ from os.path import exists, join, dirname, pardir import yaml from elasticsearch import TransportError +from elasticsearch.compat import string_types from ..test_cases import SkipTest from . import ElasticTestCase, _get_version @@ -42,11 +43,11 @@ class YamlTestCase(ElasticTestCase): def _resolve(self, value): # resolve variables - if isinstance(value, (type(u''), type(''))) and value.startswith('$'): + if isinstance(value, string_types) and value.startswith('$'): value = value[1:] self.assertIn(value, self._state) value = self._state[value] - if isinstance(value, (type(u''), type(''))): + if isinstance(value, string_types): value = value.strip() return value @@ -172,7 +173,7 @@ class YamlTestCase(ElasticTestCase): value = self._lookup(path) expected = self._resolve(expected) - if isinstance(expected, (type(''), type(u''))) and \ + if isinstance(expected, string_types) and \ expected.startswith('/') and expected.endswith('/'): expected = re.compile(expected[1:-1], re.VERBOSE) self.assertTrue(expected.search(value)) diff --git a/test_elasticsearch/test_server/test_memcached.py b/test_elasticsearch/test_server/test_memcached.py index 79b320d0..24e7061c 100644 --- a/test_elasticsearch/test_server/test_memcached.py +++ b/test_elasticsearch/test_server/test_memcached.py @@ -2,6 +2,7 @@ from elasticsearch import Elasticsearch, MemcachedConnection, NotFoundError from elasticsearch.transport import ADDRESS_RE +from elasticsearch.compat import u from . import ElasticTestCase from ..test_cases import SkipTest @@ -34,8 +35,8 @@ class TestMemcachedConnection(ElasticTestCase): self.assertEquals({"answer": 42}, self.mc_client.get("test_index", doc_type="test_type", id=1)["_source"]) def test_unicode(self): - self.mc_client.index("test_index", "test_type", {"answer": u"你好"}, id=u"你好") - self.assertEquals({"answer": u"你好"}, self.mc_client.get("test_index", doc_type="test_type", id=u"你好")["_source"]) + self.mc_client.index("test_index", "test_type", {"answer": u("你好")}, id=u("你好")) + self.assertEquals({"answer": u("你好")}, self.mc_client.get("test_index", doc_type="test_type", id=u("你好"))["_source"]) def test_missing(self): self.assertRaises(NotFoundError, self.mc_client.get, "test_index", doc_type="test_type", id=42) diff --git a/test_elasticsearch/test_transport.py b/test_elasticsearch/test_transport.py index 6e406370..25ccf442 100644 --- a/test_elasticsearch/test_transport.py +++ b/test_elasticsearch/test_transport.py @@ -4,6 +4,7 @@ import time from elasticsearch.transport import Transport from elasticsearch.connection import Connection from elasticsearch.exceptions import ConnectionError +from elasticsearch.compat import u from .test_cases import TestCase @@ -54,7 +55,7 @@ class TestTransport(TestCase): def test_body_gets_encoded_into_bytes(self): t = Transport([{}], connection_class=DummyConnection) - t.perform_request('GET', '/', body=u'你好') + t.perform_request('GET', '/', body=u('你好')) self.assertEquals(1, len(t.get_connection().calls)) self.assertEquals(('GET', '/', None, b'\xe4\xbd\xa0\xe5\xa5\xbd'), t.get_connection().calls[0][0])