diff --git a/docs/index.rst b/docs/index.rst index 9b1754c6..30e8ea3a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -230,6 +230,27 @@ description of the options. .. _certifi: http://certifiio.readthedocs.io/en/latest/ +Connecting via Cloud ID +~~~~~~~~~~~~~~~~~~~~~~~ + +Cloud ID is an easy way to configure your client to work +with your Elastic Cloud deployment. Combine the ``cloud_id`` +with either ``http_auth`` or ``api_key`` to authenticate +with your Elastic Cloud deployment. + +Using ``cloud_id`` enables TLS verification and HTTP compression by default +and sets the port to ``443`` unless otherwise overwritten via the ``port`` parameter +or the port value encoded within ``cloud_id``. Using Cloud ID also disables sniffing. + +.. code-block:: python + + from elasticsearch import Elasticsearch + + es = Elasticsearch( + cloud_id="cluster-1:dXMa5Fx...", + http_auth=("elastic", ""), + ) + APIKey Authentication ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/elasticsearch/connection/base.py b/elasticsearch/connection/base.py index 9b2344e4..2781a59a 100644 --- a/elasticsearch/connection/base.py +++ b/elasticsearch/connection/base.py @@ -44,7 +44,7 @@ class Connection(object): def __init__( self, host="localhost", - port=9200, + port=None, use_ssl=False, url_prefix="", timeout=10, @@ -59,20 +59,28 @@ class Connection(object): if cloud_id: try: _, cloud_id = cloud_id.split(":") - parent_dn, es_uuid, _ = ( + parent_dn, es_uuid = ( binascii.a2b_base64(cloud_id.encode("utf-8")) .decode("utf-8") - .split("$") + .split("$")[:2] ) - except ValueError: + if ":" in parent_dn: + parent_dn, _, parent_port = parent_dn.rpartition(":") + if port is None and parent_port != "443": + port = int(parent_port) + except (ValueError, IndexError): raise ImproperlyConfigured("'cloud_id' is not properly formatted") host = "%s.%s" % (es_uuid, parent_dn) - port = 9243 use_ssl = True if http_compress is None: http_compress = True + # If cloud_id isn't set and port is default then use 9200. + # Cloud should use '443' by default via the 'https' scheme. + elif port is None: + port = 9200 + # Work-around if the implementing class doesn't # define the headers property before calling super().__init__() if not hasattr(self, "headers"): @@ -102,7 +110,9 @@ class Connection(object): self.hostname = host self.port = port - self.host = "%s://%s:%s" % (scheme, host, port) + self.host = "%s://%s" % (scheme, host) + if self.port is not None: + self.host += ":%s" % self.port if url_prefix: url_prefix = "/" + url_prefix.strip("/") self.url_prefix = url_prefix diff --git a/elasticsearch/connection/http_requests.py b/elasticsearch/connection/http_requests.py index 9612f13e..d88fe70b 100644 --- a/elasticsearch/connection/http_requests.py +++ b/elasticsearch/connection/http_requests.py @@ -45,7 +45,7 @@ class RequestsHttpConnection(Connection): def __init__( self, host="localhost", - port=9200, + port=None, http_auth=None, use_ssl=False, verify_certs=True, @@ -93,12 +93,7 @@ class RequestsHttpConnection(Connection): http_auth = tuple(http_auth.split(":", 1)) self.session.auth = http_auth - self.base_url = "http%s://%s:%d%s" % ( - "s" if self.use_ssl else "", - self.hostname, - self.port, - self.url_prefix, - ) + self.base_url = "%s%s" % (self.host, self.url_prefix,) self.session.verify = verify_certs if not client_key: self.session.cert = client_cert @@ -118,7 +113,7 @@ class RequestsHttpConnection(Connection): if self.use_ssl and not verify_certs and ssl_show_warn: warnings.warn( "Connecting to %s using SSL with verify_certs=False is insecure." - % self.base_url + % self.host ) def perform_request( diff --git a/elasticsearch/connection/http_urllib3.py b/elasticsearch/connection/http_urllib3.py index 81223926..7db782fb 100644 --- a/elasticsearch/connection/http_urllib3.py +++ b/elasticsearch/connection/http_urllib3.py @@ -82,7 +82,7 @@ class Urllib3HttpConnection(Connection): def __init__( self, host="localhost", - port=9200, + port=None, http_auth=None, use_ssl=False, verify_certs=VERIFY_CERTS_DEFAULT, @@ -186,7 +186,7 @@ class Urllib3HttpConnection(Connection): if ssl_show_warn: warnings.warn( "Connecting to %s using SSL with verify_certs=False is insecure." - % host + % self.host ) if not ssl_show_warn: urllib3.disable_warnings() diff --git a/elasticsearch/transport.py b/elasticsearch/transport.py index ccf599db..268b5f36 100644 --- a/elasticsearch/transport.py +++ b/elasticsearch/transport.py @@ -128,6 +128,11 @@ class Transport(object): # retain the original connection instances for sniffing self.seed_connections = self.connection_pool.connections[:] + # Don't enable sniffing on Cloud instances. + if kwargs.get("cloud_id", False): + sniff_on_start = False + sniff_on_connection_fail = False + # sniffing data self.sniffer_timeout = sniffer_timeout self.sniff_on_connection_fail = sniff_on_connection_fail diff --git a/test_elasticsearch/test_connection.py b/test_elasticsearch/test_connection.py index 436a142a..3d709a93 100644 --- a/test_elasticsearch/test_connection.py +++ b/test_elasticsearch/test_connection.py @@ -14,7 +14,11 @@ from elasticsearch.exceptions import ( RequestError, NotFoundError, ) -from elasticsearch.connection import RequestsHttpConnection, Urllib3HttpConnection +from elasticsearch.connection import ( + Connection, + RequestsHttpConnection, + Urllib3HttpConnection, +) from elasticsearch import __versionstr__ from .test_cases import TestCase, SkipTest @@ -24,6 +28,66 @@ def gzip_decompress(data): return buf.read() +class TestBaseConnection(TestCase): + def test_parse_cloud_id(self): + # Embedded port in cloud_id + con = Connection( + cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5" + ) + self.assertEqual( + con.host, + "https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:9243", + ) + self.assertEqual(con.port, 9243) + self.assertEqual( + con.hostname, + "c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com", + ) + + # Embedded port but overridden + con = Connection( + cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5", + port=443, + ) + self.assertEqual( + con.host, + "https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:443", + ) + self.assertEqual(con.port, 443) + self.assertEqual( + con.hostname, + "c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com", + ) + + # Port is 443, removed by default. + con = Connection( + cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSRlN2RlOWYxMzQ1ZTQ0OTAyODNkOTAzYmU1YjZmOTE5ZQ==" + ) + self.assertEqual( + con.host, + "https://e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com", + ) + self.assertEqual(con.port, None) + self.assertEqual( + con.hostname, + "e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com", + ) + + # No port, contains Kibana UUID + con = Connection( + cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSQ4YWY3ZWUzNTQyMGY0NThlOTAzMDI2YjQwNjQwODFmMiQyMDA2MTU1NmM1NDA0OTg2YmZmOTU3ZDg0YTZlYjUxZg==" + ) + self.assertEqual( + con.host, + "https://8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com", + ) + self.assertEqual(con.port, None) + self.assertEqual( + con.hostname, + "8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com", + ) + + class TestUrllib3Connection(TestCase): def _get_mock_connection(self, connection_params={}, response_body=b"{}"): con = Urllib3HttpConnection(**connection_params) @@ -61,41 +125,54 @@ class TestUrllib3Connection(TestCase): def test_http_cloud_id(self): con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n" + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==" ) self.assertTrue(con.use_ssl) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" + ) + self.assertEquals(con.port, None) + self.assertEquals( + con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" + ) + self.assertTrue(con.http_compress) + + con = Urllib3HttpConnection( + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", + port=9243, + ) + self.assertEquals( + con.host, + "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243", ) self.assertEquals(con.port, 9243) self.assertEquals( - con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com" + con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) - self.assertTrue(con.http_compress) def test_api_key_auth(self): # test with tuple con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", api_key=("elastic", "changeme1"), ) self.assertEquals( con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=" ) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) # test with base64 encoded string con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=", ) self.assertEquals( con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=" ) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) def test_no_http_compression(self): @@ -140,18 +217,18 @@ class TestUrllib3Connection(TestCase): # 'http_compress' will be 'True' by default for connections with # 'cloud_id' set but should prioritize user-defined values. con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", ) self.assertEquals(con.http_compress, True) con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", http_compress=False, ) self.assertEquals(con.http_compress, False) con = Urllib3HttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", http_compress=True, ) self.assertEquals(con.http_compress, True) @@ -219,7 +296,7 @@ class TestUrllib3Connection(TestCase): con = Urllib3HttpConnection(use_ssl=True, verify_certs=False) self.assertEquals(1, len(w)) self.assertEquals( - "Connecting to localhost using SSL with verify_certs=False is insecure.", + "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.", str(w[0].message), ) @@ -327,41 +404,54 @@ class TestRequestsConnection(TestCase): def test_http_cloud_id(self): con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n" + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==" ) self.assertTrue(con.use_ssl) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" + ) + self.assertEquals(con.port, None) + self.assertEquals( + con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" + ) + self.assertTrue(con.http_compress) + + con = RequestsHttpConnection( + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", + port=9243, + ) + self.assertEquals( + con.host, + "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243", ) self.assertEquals(con.port, 9243) self.assertEquals( - con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com" + con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) - self.assertTrue(con.http_compress) def test_api_key_auth(self): # test with tuple con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", api_key=("elastic", "changeme1"), ) self.assertEquals( con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE=" ) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) # test with base64 encoded string con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=", ) self.assertEquals( con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI=" ) self.assertEquals( - con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243" + con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io" ) def test_no_http_compression(self): @@ -401,18 +491,18 @@ class TestRequestsConnection(TestCase): # 'http_compress' will be 'True' by default for connections with # 'cloud_id' set but should prioritize user-defined values. con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", ) self.assertEquals(con.http_compress, True) con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", http_compress=False, ) self.assertEquals(con.http_compress, False) con = RequestsHttpConnection( - cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n", + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", http_compress=True, ) self.assertEquals(con.http_compress, True) @@ -424,7 +514,7 @@ class TestRequestsConnection(TestCase): ) self.assertEquals(1, len(w)) self.assertEquals( - "Connecting to https://localhost:9200/url using SSL with verify_certs=False is insecure.", + "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.", str(w[0].message), ) diff --git a/test_elasticsearch/test_transport.py b/test_elasticsearch/test_transport.py index 443db719..4dbb79ce 100644 --- a/test_elasticsearch/test_transport.py +++ b/test_elasticsearch/test_transport.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals import time +from mock import patch from elasticsearch.transport import Transport, get_host_info from elasticsearch.connection import Connection @@ -346,3 +347,15 @@ class TestTransport(TestCase): t.connection_pool.connection_opts[0][1], {"host": "somehost.tld", "port": 123}, ) + + @patch("elasticsearch.transport.Transport.sniff_hosts") + def test_sniffing_disabled_on_cloud_instances(self, sniff_hosts): + t = Transport( + [{}], + sniff_on_start=True, + sniff_on_connection_fail=True, + cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==", + ) + + self.assertFalse(t.sniff_on_connection_fail) + self.assertIs(sniff_hosts.call_args, None) # Assert not called.