From cd769b98a53d1846349fe54dfb9918e5f90de4c3 Mon Sep 17 00:00:00 2001 From: Tyler James Harden Date: Tue, 15 Aug 2017 10:41:32 -0400 Subject: [PATCH] Additional fixes for UTF-8 surrogate escapes (#629) * Fixes non UTF-8 surrogateescapes Surrogate escapes in Unicode (non UTF-8 encoding) will be properly escaped with backslashes when encountered, versus breaking the transport layer. * Removes erroneous bytes decode and reraises Fixes to re-raise exceptions with different reasons Removes erroneous bytes decode where bytes are desired * Adds test for surrogate escapes in body Tests that a surrogate escape sequence is properly escaped with backslashes to produce valid UTF-8. * Use proper byte sequence for surrogate * Use if/else versus pass * Proper Unicode surrogate escape Use a Unicode Surrogate that properly escapes in both Python2 and Python3 * Passing test once surrogatepass is used Updating test to pass once surrogatepass is used * Use surrogatepass instead of backslashreplace This replicates behavior between Python 2 and Python 3 * Fixes whitespace * Simplifies with no exception block Since `surrogatepass` will only ever explicitly occur when there are surrogate bytes encountered, there is no need to let the error throw and catch it, also uses single-quotes for consistency. * Fixes Unicode Surrogate Escapes in request logging This is the same fix as accepted for ElasticSearch requests, except applied to the request logging mechanisms. --- elasticsearch/connection/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/connection/base.py b/elasticsearch/connection/base.py index 62d62f65..3ea18fd5 100644 --- a/elasticsearch/connection/base.py +++ b/elasticsearch/connection/base.py @@ -76,7 +76,7 @@ class Connection(object): # body has already been serialized to utf-8, deserialize it for logging # TODO: find a better way to avoid (de)encoding the body back and forth if body: - body = body.decode('utf-8') + body = body.decode('utf-8', 'ignore') logger.info( '%s %s [status:%s request:%.3fs]', method, full_url, @@ -100,7 +100,7 @@ class Connection(object): # body has already been serialized to utf-8, deserialize it for logging # TODO: find a better way to avoid (de)encoding the body back and forth if body: - body = body.decode('utf-8') + body = body.decode('utf-8', 'ignore') logger.debug('> %s', body)