Abstract away service name (#268)

* Abstract away service name

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Compuute x-amz-content-256 header

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Fix async signing

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Adds types-six to dependencies

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Optionally remove Content-Length

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Fix dict typo

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Remove requirement for x-amz-content-sha256 header

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Remove deletion of content-length

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Fix capitalization

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

* Adding unit tests

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

CHANGELOG.md

@@ -26,6 +26,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Updated getting started to user guide ([#233](https://github.com/opensearch-project/opensearch-py/pull/233))
 - Updated CA certificate handling to check OpenSSL environment variables before defaulting to certifi ([#196](https://github.com/opensearch-project/opensearch-py/pull/196))
 - Updates `master` to `cluster_manager` to be inclusive ([#242](https://github.com/opensearch-project/opensearch-py/pull/242))
+- Support a custom signing service name for AWS SigV4 ([#268](https://github.com/opensearch-project/opensearch-py/pull/268))
 ### Deprecated
 
 ### Removed

USER_GUIDE.md

@@ -407,8 +407,9 @@ import boto3
 
 host = '' # cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com
 region = 'us-west-2'
+service = 'es' # 'aoss' for OpenSearch Serverless
 credentials = boto3.Session().get_credentials()
-auth = AWSV4SignerAuth(credentials, region)
+auth = AWSV4SignerAuth(credentials, region, service)
 index_name = 'python-test-index3'
 
 client = OpenSearch(
@@ -450,8 +451,9 @@ import boto3
 
 host = '' # cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com
 region = 'us-west-2'
+service = 'es' # 'aoss' for OpenSearch Serverless
 credentials = boto3.Session().get_credentials()
-auth = AWSV4SignerAsyncAuth(credentials, region)
+auth = AWSV4SignerAsyncAuth(credentials, region, service)
 index_name = 'python-test-index3'
 
 client = OpenSearch(

docs/source/conf.py

@@ -17,9 +17,9 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'OpenSearch Python Client'
+project = "OpenSearch Python Client"
-copyright = 'OpenSearch Project Contributors'
+copyright = "OpenSearch Project Contributors"
-author = 'OpenSearch Project Contributors'
+author = "OpenSearch Project Contributors"
 
 
 # -- General configuration ---------------------------------------------------
@@ -38,7 +38,7 @@ extensions = [
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -51,12 +51,12 @@ exclude_patterns = []
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # -- additional settings -------------------------------------------------
 intersphinx_mapping = {

noxfile.py

@@ -57,7 +57,7 @@ def format(session):
 
 @nox.session()
 def lint(session):
-    session.install("flake8", "black", "mypy", "isort", "types-requests")
+    session.install("flake8", "black", "mypy", "isort", "types-requests", "types-six")
 
     session.run("isort", "--check", "--profile=black", *SOURCE_FILES)
     session.run("black", "--target-version=py33", "--check", *SOURCE_FILES)

opensearchpy/_async/helpers.py

@@ -204,7 +204,7 @@ async def async_streaming_bulk(
                         raise_on_error,
                         ignore_status,
                         *args,
-                        **kwargs
+                        **kwargs,
                     ),
                 ):
 
@@ -471,5 +471,5 @@ async def async_reindex(
         target_client,
         _change_doc_index(docs, target_index),
         chunk_size=chunk_size,
-        **kwargs
+        **kwargs,
     )

opensearchpy/helpers/asyncsigner.py

@@ -9,8 +9,6 @@
 
 import sys
 
-OPENSEARCH_SERVICE = "es"
-
 PY3 = sys.version_info[0] == 3
 
 
@@ -19,7 +17,7 @@ class AWSV4SignerAsyncAuth:
     AWS V4 Request Signer for Async Requests.
     """
 
-    def __init__(self, credentials, region):  # type: ignore
+    def __init__(self, credentials, region, service="es"):  # type: ignore
         if not credentials:
             raise ValueError("Credentials cannot be empty")
         self.credentials = credentials
@@ -28,6 +26,10 @@ class AWSV4SignerAsyncAuth:
             raise ValueError("Region cannot be empty")
         self.region = region
 
+        if not service:
+            raise ValueError("Service name cannot be empty")
+        self.service = service
+
     def __call__(self, method, url, query_string, body):  # type: ignore
         return self._sign_request(method, url, query_string, body)  # type: ignore
 
@@ -37,17 +39,17 @@ class AWSV4SignerAsyncAuth:
         :param prepared_request: unsigned headers
         :return: signed headers
         """
+
         from botocore.auth import SigV4Auth
         from botocore.awsrequest import AWSRequest
 
         # create an AWS request object and sign it using SigV4Auth
-        print("".join([url, query_string]))
         aws_request = AWSRequest(
             method=method,
             url="".join([url, query_string]),
-            data=body,
         )
-        sig_v4_auth = SigV4Auth(self.credentials, OPENSEARCH_SERVICE, self.region)
+
+        sig_v4_auth = SigV4Auth(self.credentials, self.service, self.region)
         sig_v4_auth.add_auth(aws_request)
 
         # copy the headers from AWS request object into the prepared_request

opensearchpy/helpers/signer.py

@@ -11,8 +11,6 @@ import sys
 
 import requests
 
-OPENSEARCH_SERVICE = "es"
-
 PY3 = sys.version_info[0] == 3
 
 if PY3:
@@ -50,7 +48,7 @@ class AWSV4SignerAuth(requests.auth.AuthBase):
     AWS V4 Request Signer for Requests.
     """
 
-    def __init__(self, credentials, region):  # type: ignore
+    def __init__(self, credentials, region, service="es"):  # type: ignore
         if not credentials:
             raise ValueError("Credentials cannot be empty")
         self.credentials = credentials
@@ -59,6 +57,10 @@ class AWSV4SignerAuth(requests.auth.AuthBase):
             raise ValueError("Region cannot be empty")
         self.region = region
 
+        if not service:
+            raise ValueError("Service name cannot be empty")
+        self.service = service
+
     def __call__(self, request):  # type: ignore
         return self._sign_request(request)  # type: ignore
 
@@ -78,9 +80,9 @@ class AWSV4SignerAuth(requests.auth.AuthBase):
         aws_request = AWSRequest(
             method=prepared_request.method.upper(),
             url=url,
-            data=prepared_request.body,
         )
-        sig_v4_auth = SigV4Auth(self.credentials, OPENSEARCH_SERVICE, self.region)
+
+        sig_v4_auth = SigV4Auth(self.credentials, self.service, self.region)
         sig_v4_auth.add_auth(aws_request)
 
         # copy the headers from AWS request object into the prepared_request

out/opensearchpy/plugins/alerting.pyi

@@ -7,19 +7,67 @@
 # Modifications Copyright OpenSearch Contributors. See
 # GitHub history for details.
 
-from ..client.utils import NamespacedClient as NamespacedClient, query_params as query_params
+from ..client.utils import (
+    NamespacedClient as NamespacedClient,
+    query_params as query_params,
+)
 from typing import Any, Union
 
 class AlertingClient(NamespacedClient):
-    def search_monitor(self, body, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    def search_monitor(
-    def get_monitor(self, monitor_id, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+        self, body, params: Any | None = ..., headers: Any | None = ...
-    def run_monitor(self, monitor_id, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    ) -> Union[bool, Any]: ...
-    def create_monitor(self, body: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    def get_monitor(
-    def update_monitor(self, monitor_id, body: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+        self, monitor_id, params: Any | None = ..., headers: Any | None = ...
-    def delete_monitor(self, monitor_id, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    ) -> Union[bool, Any]: ...
-    def get_destination(self, destination_id: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    def run_monitor(
-    def create_destination(self, body: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+        self, monitor_id, params: Any | None = ..., headers: Any | None = ...
-    def update_destination(self, destination_id, body: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    ) -> Union[bool, Any]: ...
-    def delete_destination(self, destination_id, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+    def create_monitor(
-    def get_alerts(self, params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+        self,
-    def acknowledge_alert(self, monitor_id, body: Any | None = ..., params: Any | None = ..., headers: Any | None = ...) -> Union[bool, Any]: ...
+        body: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...
+    def update_monitor(
+        self,
+        monitor_id,
+        body: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...
+    def delete_monitor(
+        self, monitor_id, params: Any | None = ..., headers: Any | None = ...
+    ) -> Union[bool, Any]: ...
+    def get_destination(
+        self,
+        destination_id: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...
+    def create_destination(
+        self,
+        body: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...
+    def update_destination(
+        self,
+        destination_id,
+        body: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...
+    def delete_destination(
+        self, destination_id, params: Any | None = ..., headers: Any | None = ...
+    ) -> Union[bool, Any]: ...
+    def get_alerts(
+        self, params: Any | None = ..., headers: Any | None = ...
+    ) -> Union[bool, Any]: ...
+    def acknowledge_alert(
+        self,
+        monitor_id,
+        body: Any | None = ...,
+        params: Any | None = ...,
+        headers: Any | None = ...,
+    ) -> Union[bool, Any]: ...

test_opensearchpy/test_async/test_asyncsigner.py

@@ -63,3 +63,18 @@ class TestAsyncSigner(TestCase):
 
         with pytest.raises(ValueError) as e:
             assert str(e.value) == "Credentials cannot be empty"
+
+    @pytest.mark.skipif(
+        sys.version_info < (3, 6), reason="AWSV4SignerAsyncAuth requires python3.6+"
+    )
+    async def test_aws_signer_async_when_service_is_specified(self):
+        region = "us-west-2"
+        service = "aoss"
+
+        from opensearchpy.helpers.asyncsigner import AWSV4SignerAsyncAuth
+
+        auth = AWSV4SignerAsyncAuth(self.mock_session(), region, service)
+        headers = auth("GET", "http://localhost")
+        self.assertIn("Authorization", headers)
+        self.assertIn("X-Amz-Date", headers)
+        self.assertIn("X-Amz-Security-Token", headers)

test_opensearchpy/test_connection.py

@@ -333,6 +333,9 @@ class TestUrllib3Connection(TestCase):
         self.assertIn("X-Amz-Date", prepared_request.headers)
         self.assertIn("X-Amz-Security-Token", prepared_request.headers)
 
+    @pytest.mark.skipif(
+        sys.version_info < (3, 6), reason="AWSV4SignerAuth requires python3.6+"
+    )
     def test_aws_signer_when_region_is_null(self):
         session = self.mock_session()
 
@@ -346,6 +349,9 @@ class TestUrllib3Connection(TestCase):
             AWSV4SignerAuth(session, "")
         assert str(e.value) == "Region cannot be empty"
 
+    @pytest.mark.skipif(
+        sys.version_info < (3, 6), reason="AWSV4SignerAuth requires python3.6+"
+    )
     def test_aws_signer_when_credentials_is_null(self):
         region = "us-west-1"
 
@@ -359,6 +365,26 @@ class TestUrllib3Connection(TestCase):
             AWSV4SignerAuth("", region)
         assert str(e.value) == "Credentials cannot be empty"
 
+    @pytest.mark.skipif(
+        sys.version_info < (3, 6), reason="AWSV4SignerAuth requires python3.6+"
+    )
+    def test_aws_signer_when_service_is_specified(self):
+        region = "us-west-1"
+        service = "aoss"
+
+        import requests
+
+        from opensearchpy.helpers.signer import AWSV4SignerAuth
+
+        auth = AWSV4SignerAuth(self.mock_session(), region, service)
+        con = RequestsHttpConnection(http_auth=auth)
+        prepared_request = requests.Request("GET", "http://localhost").prepare()
+        auth(prepared_request)
+        self.assertEqual(auth, con.session.auth)
+        self.assertIn("Authorization", prepared_request.headers)
+        self.assertIn("X-Amz-Date", prepared_request.headers)
+        self.assertIn("X-Amz-Security-Token", prepared_request.headers)
+
     def mock_session(self):
         access_key = uuid.uuid4().hex
         secret_key = uuid.uuid4().hex