actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[7.x] Update Cloud ID default port, parsing of Cloud ID

Browse Source
This commit is contained in:
Seth Michael Larson
2020-03-12 07:02:04 -05:00
committed by GitHub
parent 9219c92779
commit ae74e1d0e6
7 changed files with 175 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/index.rst
+21
View File
@@ -230,6 +230,27 @@ description of the options.
.. _certifi: http://certifiio.readthedocs.io/en/latest/
Connecting via Cloud ID
~~~~~~~~~~~~~~~~~~~~~~~
Cloud ID is an easy way to configure your client to work
with your Elastic Cloud deployment. Combine the ``cloud_id``
with either ``http_auth`` or ``api_key`` to authenticate
with your Elastic Cloud deployment.
Using ``cloud_id`` enables TLS verification and HTTP compression by default
and sets the port to ``443`` unless otherwise overwritten via the ``port`` parameter
or the port value encoded within ``cloud_id``. Using Cloud ID also disables sniffing.
.. code-block:: python
from elasticsearch import Elasticsearch
es = Elasticsearch(
cloud_id="cluster-1:dXMa5Fx...",
http_auth=("elastic", "<password>"),
)
APIKey Authentication
~~~~~~~~~~~~~~~~~~~~~~
elasticsearch/connection/base.py
+16 -6
View File
@@ -44,7 +44,7 @@ class Connection(object):
def __init__(
self,
host="localhost",
port=9200,
port=None,
use_ssl=False,
url_prefix="",
timeout=10,
@@ -59,20 +59,28 @@ class Connection(object):
if cloud_id:
try:
_, cloud_id = cloud_id.split(":")
parent_dn, es_uuid, _ = (
parent_dn, es_uuid = (
binascii.a2b_base64(cloud_id.encode("utf-8"))
.decode("utf-8")
.split("$")
.split("$")[:2]
)
except ValueError:
if ":" in parent_dn:
parent_dn, _, parent_port = parent_dn.rpartition(":")
if port is None and parent_port != "443":
port = int(parent_port)
except (ValueError, IndexError):
raise ImproperlyConfigured("'cloud_id' is not properly formatted")
host = "%s.%s" % (es_uuid, parent_dn)
port = 9243
use_ssl = True
if http_compress is None:
http_compress = True
# If cloud_id isn't set and port is default then use 9200.
# Cloud should use '443' by default via the 'https' scheme.
elif port is None:
port = 9200
# Work-around if the implementing class doesn't
# define the headers property before calling super().__init__()
if not hasattr(self, "headers"):
@@ -102,7 +110,9 @@ class Connection(object):
self.hostname = host
self.port = port
self.host = "%s://%s:%s" % (scheme, host, port)
self.host = "%s://%s" % (scheme, host)
if self.port is not None:
self.host += ":%s" % self.port
if url_prefix:
url_prefix = "/" + url_prefix.strip("/")
self.url_prefix = url_prefix
elasticsearch/connection/http_requests.py
+3 -8
View File
@@ -45,7 +45,7 @@ class RequestsHttpConnection(Connection):
def __init__(
self,
host="localhost",
port=9200,
port=None,
http_auth=None,
use_ssl=False,
verify_certs=True,
@@ -93,12 +93,7 @@ class RequestsHttpConnection(Connection):
http_auth = tuple(http_auth.split(":", 1))
self.session.auth = http_auth
self.base_url = "http%s://%s:%d%s" % (
"s" if self.use_ssl else "",
self.hostname,
self.port,
self.url_prefix,
)
self.base_url = "%s%s" % (self.host, self.url_prefix,)
self.session.verify = verify_certs
if not client_key:
self.session.cert = client_cert
@@ -118,7 +113,7 @@ class RequestsHttpConnection(Connection):
if self.use_ssl and not verify_certs and ssl_show_warn:
warnings.warn(
"Connecting to %s using SSL with verify_certs=False is insecure."
% self.base_url
% self.host
)
def perform_request(
elasticsearch/connection/http_urllib3.py
+2 -2
View File
@@ -82,7 +82,7 @@ class Urllib3HttpConnection(Connection):
def __init__(
self,
host="localhost",
port=9200,
port=None,
http_auth=None,
use_ssl=False,
verify_certs=VERIFY_CERTS_DEFAULT,
@@ -186,7 +186,7 @@ class Urllib3HttpConnection(Connection):
if ssl_show_warn:
warnings.warn(
"Connecting to %s using SSL with verify_certs=False is insecure."
% host
% self.host
)
if not ssl_show_warn:
urllib3.disable_warnings()
elasticsearch/transport.py
+5
View File
@@ -128,6 +128,11 @@ class Transport(object):
# retain the original connection instances for sniffing
self.seed_connections = self.connection_pool.connections[:]
# Don't enable sniffing on Cloud instances.
if kwargs.get("cloud_id", False):
sniff_on_start = False
sniff_on_connection_fail = False
# sniffing data
self.sniffer_timeout = sniffer_timeout
self.sniff_on_connection_fail = sniff_on_connection_fail
test_elasticsearch/test_connection.py
+115 -25
View File
@@ -14,7 +14,11 @@ from elasticsearch.exceptions import (
RequestError,
NotFoundError,
)
from elasticsearch.connection import RequestsHttpConnection, Urllib3HttpConnection
from elasticsearch.connection import (
Connection,
RequestsHttpConnection,
Urllib3HttpConnection,
)
from elasticsearch import __versionstr__
from .test_cases import TestCase, SkipTest
@@ -24,6 +28,66 @@ def gzip_decompress(data):
return buf.read()
class TestBaseConnection(TestCase):
def test_parse_cloud_id(self):
# Embedded port in cloud_id
con = Connection(
cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5"
)
self.assertEqual(
con.host,
"https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:9243",
)
self.assertEqual(con.port, 9243)
self.assertEqual(
con.hostname,
"c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com",
)
# Embedded port but overridden
con = Connection(
cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJGM2NjM3ZjMxMmM1MjQzY2RhN2RlZDZlOTllM2QyYzE5",
port=443,
)
self.assertEqual(
con.host,
"https://c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com:443",
)
self.assertEqual(con.port, 443)
self.assertEqual(
con.hostname,
"c6637f312c5243cda7ded6e99e3d2c19.westeurope.azure.elastic-cloud.com",
)
# Port is 443, removed by default.
con = Connection(
cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSRlN2RlOWYxMzQ1ZTQ0OTAyODNkOTAzYmU1YjZmOTE5ZQ=="
)
self.assertEqual(
con.host,
"https://e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com",
)
self.assertEqual(con.port, None)
self.assertEqual(
con.hostname,
"e7de9f1345e4490283d903be5b6f919e.westeurope.azure.elastic-cloud.com",
)
# No port, contains Kibana UUID
con = Connection(
cloud_id="cluster:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbSQ4YWY3ZWUzNTQyMGY0NThlOTAzMDI2YjQwNjQwODFmMiQyMDA2MTU1NmM1NDA0OTg2YmZmOTU3ZDg0YTZlYjUxZg=="
)
self.assertEqual(
con.host,
"https://8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com",
)
self.assertEqual(con.port, None)
self.assertEqual(
con.hostname,
"8af7ee35420f458e903026b4064081f2.westeurope.azure.elastic-cloud.com",
)
class TestUrllib3Connection(TestCase):
def _get_mock_connection(self, connection_params={}, response_body=b"{}"):
con = Urllib3HttpConnection(**connection_params)
@@ -61,41 +125,54 @@ class TestUrllib3Connection(TestCase):
def test_http_cloud_id(self):
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n"
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng=="
)
self.assertTrue(con.use_ssl)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertEquals(con.port, None)
self.assertEquals(
con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertTrue(con.http_compress)
con = Urllib3HttpConnection(
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
port=9243,
)
self.assertEquals(
con.host,
"https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243",
)
self.assertEquals(con.port, 9243)
self.assertEquals(
con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertTrue(con.http_compress)
def test_api_key_auth(self):
# test with tuple
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
api_key=("elastic", "changeme1"),
)
self.assertEquals(
con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
# test with base64 encoded string
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
)
self.assertEquals(
con.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
def test_no_http_compression(self):
@@ -140,18 +217,18 @@ class TestUrllib3Connection(TestCase):
# 'http_compress' will be 'True' by default for connections with
# 'cloud_id' set but should prioritize user-defined values.
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
)
self.assertEquals(con.http_compress, True)
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
http_compress=False,
)
self.assertEquals(con.http_compress, False)
con = Urllib3HttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
http_compress=True,
)
self.assertEquals(con.http_compress, True)
@@ -219,7 +296,7 @@ class TestUrllib3Connection(TestCase):
con = Urllib3HttpConnection(use_ssl=True, verify_certs=False)
self.assertEquals(1, len(w))
self.assertEquals(
"Connecting to localhost using SSL with verify_certs=False is insecure.",
"Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.",
str(w[0].message),
)
@@ -327,41 +404,54 @@ class TestRequestsConnection(TestCase):
def test_http_cloud_id(self):
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n"
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng=="
)
self.assertTrue(con.use_ssl)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertEquals(con.port, None)
self.assertEquals(
con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertTrue(con.http_compress)
con = RequestsHttpConnection(
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
port=9243,
)
self.assertEquals(
con.host,
"https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io:9243",
)
self.assertEquals(con.port, 9243)
self.assertEquals(
con.hostname, "0fd50f62320ed6539f6cb48e1b68.example.cloud.com"
con.hostname, "4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
self.assertTrue(con.http_compress)
def test_api_key_auth(self):
# test with tuple
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
api_key=("elastic", "changeme1"),
)
self.assertEquals(
con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTE="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
# test with base64 encoded string
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
api_key="ZWxhc3RpYzpjaGFuZ2VtZTI=",
)
self.assertEquals(
con.session.headers["authorization"], "ApiKey ZWxhc3RpYzpjaGFuZ2VtZTI="
)
self.assertEquals(
con.host, "https://0fd50f62320ed6539f6cb48e1b68.example.cloud.com:9243"
con.host, "https://4fa8821e75634032bed1cf22110e2f97.us-east-1.aws.found.io"
)
def test_no_http_compression(self):
@@ -401,18 +491,18 @@ class TestRequestsConnection(TestCase):
# 'http_compress' will be 'True' by default for connections with
# 'cloud_id' set but should prioritize user-defined values.
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
)
self.assertEquals(con.http_compress, True)
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
http_compress=False,
)
self.assertEquals(con.http_compress, False)
con = RequestsHttpConnection(
cloud_id="foobar:ZXhhbXBsZS5jbG91ZC5jb20kMGZkNTBmNjIzMjBlZDY1MzlmNmNiNDhlMWI2OCRhYzUzOTVhODgz\nNDU2NmM5ZjE1Y2Q4ZTQ5MGE=\n",
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
http_compress=True,
)
self.assertEquals(con.http_compress, True)
@@ -424,7 +514,7 @@ class TestRequestsConnection(TestCase):
)
self.assertEquals(1, len(w))
self.assertEquals(
"Connecting to https://localhost:9200/url using SSL with verify_certs=False is insecure.",
"Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure.",
str(w[0].message),
)
test_elasticsearch/test_transport.py
+13
View File
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import time
from mock import patch
from elasticsearch.transport import Transport, get_host_info
from elasticsearch.connection import Connection
@@ -346,3 +347,15 @@ class TestTransport(TestCase):
t.connection_pool.connection_opts[0][1],
{"host": "somehost.tld", "port": 123},
)
@patch("elasticsearch.transport.Transport.sniff_hosts")
def test_sniffing_disabled_on_cloud_instances(self, sniff_hosts):
t = Transport(
[{}],
sniff_on_start=True,
sniff_on_connection_fail=True,
cloud_id="cluster:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5NyQ0ZmE4ODIxZTc1NjM0MDMyYmVkMWNmMjIxMTBlMmY5Ng==",
)
self.assertFalse(t.sniff_on_connection_fail)
self.assertIs(sniff_hosts.call_args, None) # Assert not called.