[7.x] Update Cloud ID default port, parsing of Cloud ID

docs/index.rst

@@ -230,6 +230,27 @@ description of the options.
 
 .. _certifi: http://certifiio.readthedocs.io/en/latest/
 
+Connecting via Cloud ID
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Cloud ID is an easy way to configure your client to work
+with your Elastic Cloud deployment. Combine the ``cloud_id``
+with either ``http_auth`` or ``api_key`` to authenticate
+with your Elastic Cloud deployment.
+
+Using ``cloud_id`` enables TLS verification and HTTP compression by default
+and sets the port to ``443`` unless otherwise overwritten via the ``port`` parameter
+or the port value encoded within ``cloud_id``.  Using Cloud ID also disables sniffing.
+
+.. code-block:: python
+
+    from elasticsearch import Elasticsearch
+
+    es = Elasticsearch(
+        cloud_id="cluster-1:dXMa5Fx...",
+        http_auth=("elastic", "<password>"),
+    )
+
 APIKey Authentication
 ~~~~~~~~~~~~~~~~~~~~~~
 

elasticsearch/connection/base.py

@@ -44,7 +44,7 @@ class Connection(object):
     def __init__(
         self,
         host="localhost",
-        port=9200,
+        port=None,
         use_ssl=False,
         url_prefix="",
         timeout=10,
@@ -59,20 +59,28 @@ class Connection(object):
         if cloud_id:
             try:
                 _, cloud_id = cloud_id.split(":")
-                parent_dn, es_uuid, _ = (
+                parent_dn, es_uuid = (
                     binascii.a2b_base64(cloud_id.encode("utf-8"))
                     .decode("utf-8")
-                    .split("$")
+                    .split("$")[:2]
                 )
-            except ValueError:
+                if ":" in parent_dn:
+                    parent_dn, _, parent_port = parent_dn.rpartition(":")
+                    if port is None and parent_port != "443":
+                        port = int(parent_port)
+            except (ValueError, IndexError):
                 raise ImproperlyConfigured("'cloud_id' is not properly formatted")
 
             host = "%s.%s" % (es_uuid, parent_dn)
-            port = 9243
             use_ssl = True
             if http_compress is None:
                 http_compress = True
 
+        # If cloud_id isn't set and port is default then use 9200.
+        # Cloud should use '443' by default via the 'https' scheme.
+        elif port is None:
+            port = 9200
+
         # Work-around if the implementing class doesn't
         # define the headers property before calling super().__init__()
         if not hasattr(self, "headers"):
@@ -102,7 +110,9 @@ class Connection(object):
 
         self.hostname = host
         self.port = port
-        self.host = "%s://%s:%s" % (scheme, host, port)
+        self.host = "%s://%s" % (scheme, host)
+        if self.port is not None:
+            self.host += ":%s" % self.port
         if url_prefix:
             url_prefix = "/" + url_prefix.strip("/")
         self.url_prefix = url_prefix

elasticsearch/connection/http_requests.py

@@ -45,7 +45,7 @@ class RequestsHttpConnection(Connection):
     def __init__(
         self,
         host="localhost",
-        port=9200,
+        port=None,
         http_auth=None,
         use_ssl=False,
         verify_certs=True,
@@ -93,12 +93,7 @@ class RequestsHttpConnection(Connection):
                 http_auth = tuple(http_auth.split(":", 1))
             self.session.auth = http_auth
 
-        self.base_url = "http%s://%s:%d%s" % (
+        self.base_url = "%s%s" % (self.host, self.url_prefix,)
-            "s" if self.use_ssl else "",
-            self.hostname,
-            self.port,
-            self.url_prefix,
-        )
         self.session.verify = verify_certs
         if not client_key:
             self.session.cert = client_cert
@@ -118,7 +113,7 @@ class RequestsHttpConnection(Connection):
         if self.use_ssl and not verify_certs and ssl_show_warn:
             warnings.warn(
                 "Connecting to %s using SSL with verify_certs=False is insecure."
-                % self.base_url
+                % self.host
             )
 
     def perform_request(

elasticsearch/connection/http_urllib3.py

@@ -82,7 +82,7 @@ class Urllib3HttpConnection(Connection):
     def __init__(
         self,
         host="localhost",
-        port=9200,
+        port=None,
         http_auth=None,
         use_ssl=False,
         verify_certs=VERIFY_CERTS_DEFAULT,
@@ -186,7 +186,7 @@ class Urllib3HttpConnection(Connection):
                 if ssl_show_warn:
                     warnings.warn(
                         "Connecting to %s using SSL with verify_certs=False is insecure."
-                        % host
+                        % self.host
                     )
                 if not ssl_show_warn:
                     urllib3.disable_warnings()

elasticsearch/transport.py

@@ -128,6 +128,11 @@ class Transport(object):
         # retain the original connection instances for sniffing
         self.seed_connections = self.connection_pool.connections[:]
 
+        # Don't enable sniffing on Cloud instances.
+        if kwargs.get("cloud_id", False):
+            sniff_on_start = False
+            sniff_on_connection_fail = False
+
         # sniffing data
         self.sniffer_timeout = sniffer_timeout
         self.sniff_on_connection_fail = sniff_on_connection_fail

test_elasticsearch/test_connection.py

@@ -14,7 +14,11 @@ from elasticsearch.exceptions import (
     RequestError,
     NotFoundError,
 )
-from elasticsearch.connection import RequestsHttpConnection, Urllib3HttpConnection
+from elasticsearch.connection import (
+    Connection,
+    RequestsHttpConnection,
+    Urllib3HttpConnection,
+)
 from elasticsearch import __versionstr__
 from .test_cases import TestCase, SkipTest
 
@@ -24,6 +28,66 @@ def gzip_decompress(data):
     return buf.read()
 
 
+class TestBaseConnection(TestCase):
+    def test_parse_cloud_id(self):
+        # Embedded port in cloud_id
+        con = Connection(
+            cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5"
+        )
+        self.assertEqual(
+            con.host,
+            "https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:9243",
+        )
+        self.assertEqual(con.port, 9243)
+        self.assertEqual(
+            con.hostname,
+            "c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com",
+        )
+
+        # Embedded port but overridden
+        con = Connection(
+            cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5",
+            port=443,
+        )
+        self.assertEqual(
+            con.host,
+            "https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:443",
+        )
+        self.assertEqual(con.port, 443)
+        self.assertEqual(
+            con.hostname,
+            "c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com",
+        )
+
+        # Port is 443, removed by default.
+        con = Connection(
+            cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSRlN2RlOWYxMzQ1ZTQ0OTAyODNkOTAzYmU1YjZmOTE5ZQ=="
+        )
+        self.assertEqual(
+            con.host,
+            "https://e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com",
+        )
+        self.assertEqual(con.port, None)
+        self.assertEqual(
+            con.hostname,
+            "e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com",
+        )
+
+        # No port, contains Kibana UUID
+        con = Connection(
+            cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSQ4YWY3ZWUzNTQyMGY0NThlOTAzMDI2YjQwNjQwODFmMiQyMDA2MTU1NmM1NDA0OTg2YmZmOTU3ZDg0YTZlYjUxZg=="
+        )
+        self.assertEqual(
+            con.host,
+            "https://8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com",
+        )
+        self.assertEqual(con.port, None)
+        self.assertEqual(
+            con.hostname,
+            "8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com",
+        )
+
+
 class TestUrllib3Connection(TestCase):
     def _get_mock_connection(self, connection_params={}, response_body=b"{}"):
         con = Urllib3HttpConnection(**connection_params)
@@ -61,41 +125,54 @@ class TestUrllib3Connection(TestCase):
 
     def test_http_cloud_id(self):
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n"
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng=="
         )
         self.assertTrue(con.use_ssl)
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+        self.assertEquals(con.port, None)
+        self.assertEquals(
+            con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+        self.assertTrue(con.http_compress)
+
+        con = Urllib3HttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            port=9243,
+        )
+        self.assertEquals(
+            con.host,
+            "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243",
         )
         self.assertEquals(con.port, 9243)
         self.assertEquals(
-            con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
+            con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
-        self.assertTrue(con.http_compress)
 
     def test_api_key_auth(self):
         # test with tuple
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             api_key=("elastic", "changeme1"),
         )
         self.assertEquals(
             con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
         )
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
 
         # test with base64 encoded string
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
         )
         self.assertEquals(
             con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
         )
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
 
     def test_no_http_compression(self):
@@ -140,18 +217,18 @@ class TestUrllib3Connection(TestCase):
         # 'http_compress' will be 'True' by default for connections with
         # 'cloud_id' set but should prioritize user-defined values.
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
         )
         self.assertEquals(con.http_compress, True)
 
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             http_compress=False,
         )
         self.assertEquals(con.http_compress, False)
 
         con = Urllib3HttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             http_compress=True,
         )
         self.assertEquals(con.http_compress, True)
@@ -219,7 +296,7 @@ class TestUrllib3Connection(TestCase):
             con = Urllib3HttpConnection(use_ssl=True, verify_certs=False)
             self.assertEquals(1, len(w))
             self.assertEquals(
-                "Connecting to localhost using SSL with verify_certs=False is insecure.",
+                "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.",
                 str(w[0].message),
             )
 
@@ -327,41 +404,54 @@ class TestRequestsConnection(TestCase):
 
     def test_http_cloud_id(self):
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n"
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng=="
         )
         self.assertTrue(con.use_ssl)
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+        self.assertEquals(con.port, None)
+        self.assertEquals(
+            con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
+        )
+        self.assertTrue(con.http_compress)
+
+        con = RequestsHttpConnection(
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+            port=9243,
+        )
+        self.assertEquals(
+            con.host,
+            "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243",
         )
         self.assertEquals(con.port, 9243)
         self.assertEquals(
-            con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
+            con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
-        self.assertTrue(con.http_compress)
 
     def test_api_key_auth(self):
         # test with tuple
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             api_key=("elastic", "changeme1"),
         )
         self.assertEquals(
             con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
         )
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
 
         # test with base64 encoded string
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
         )
         self.assertEquals(
             con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
         )
         self.assertEquals(
-            con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
+            con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
         )
 
     def test_no_http_compression(self):
@@ -401,18 +491,18 @@ class TestRequestsConnection(TestCase):
         # 'http_compress' will be 'True' by default for connections with
         # 'cloud_id' set but should prioritize user-defined values.
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
         )
         self.assertEquals(con.http_compress, True)
 
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             http_compress=False,
         )
         self.assertEquals(con.http_compress, False)
 
         con = RequestsHttpConnection(
-            cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
             http_compress=True,
         )
         self.assertEquals(con.http_compress, True)
@@ -424,7 +514,7 @@ class TestRequestsConnection(TestCase):
             )
             self.assertEquals(1, len(w))
             self.assertEquals(
-                "Connecting to https://localhost:9200/url using SSL with verify_certs=False is insecure.",
+                "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.",
                 str(w[0].message),
             )
 

test_elasticsearch/test_transport.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 import time
+from mock import patch
 
 from elasticsearch.transport import Transport, get_host_info
 from elasticsearch.connection import Connection
@@ -346,3 +347,15 @@ class TestTransport(TestCase):
             t.connection_pool.connection_opts[0][1],
             {"host": "somehost.tld", "port": 123},
         )
+
+    @patch("elasticsearch.transport.Transport.sniff_hosts")
+    def test_sniffing_disabled_on_cloud_instances(self, sniff_hosts):
+        t = Transport(
+            [{}],
+            sniff_on_start=True,
+            sniff_on_connection_fail=True,
+            cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
+        )
+
+        self.assertFalse(t.sniff_on_connection_fail)
+        self.assertIs(sniff_hosts.call_args, None)  # Assert not called.