made custom headers be available to async aws signer (#863)

* made custom headers be available to async aws signer

Signed-off-by: Bruno Murino <brunomurino@users.noreply.github.com>

* updated changelog

Signed-off-by: Bruno Murino <brunomurino@users.noreply.github.com>

* added tests for using host header for AWS request signature on both sync and async clients

Signed-off-by: Bruno Murino <brunomurino@users.noreply.github.com>

* added documentation guide about aws auth when accessing via tunnel

Signed-off-by: Bruno Murino <brunomurino@users.noreply.github.com>

---------

Signed-off-by: Bruno Murino <brunomurino@users.noreply.github.com>

CHANGELOG.md

@@ -3,6 +3,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Added option to pass custom headers to 'AWSV4SignerAsyncAuth' ([863](https://github.com/opensearch-project/opensearch-py/pull/863))
 ### Updated APIs
 ### Changed
 ### Deprecated

guides/auth.md

@@ -2,6 +2,7 @@
   - [IAM Authentication](#iam-authentication)
   - [IAM Authentication with a Synchronous Client](#iam-authentication-with-a-synchronous-client)
   - [IAM Authentication with an Async Client](#iam-authentication-with-an-async-client)
+  - [IAM Authentication via Tunnel](#iam-authentication-via-tunnel)
   - [Kerberos](#kerberos)
 
 # Authentication
@@ -104,6 +105,43 @@ async def search():
 search()
 ```
 
+## IAM Authentication via Tunnel
+
+If you're accessing OpenSearch via SSH or SSM tunnel, then you need to specify the Host to be used for signing the AWS requests by passing a "Host" header, like so:
+
+
+```python
+from opensearchpy import OpenSearch, RequestsHttpConnection, RequestsAWSV4SignerAuth, AsyncOpenSearch, AsyncHttpConnection, AWSV4SignerAsyncAuth
+import boto3
+
+host = 'localhost' # local endpoint used by the SSH/SSM tunnel
+port = 8443
+signature_host = 'my-test-domain.eu-west-1.es.amazonaws.com:443' # this needs to be the real host provided by AWS
+region = 'eu-west-1'
+service = 'es' # 'aoss' for OpenSearch Serverless
+credentials = boto3.Session().get_credentials()
+
+# Sync
+client = OpenSearch(
+    hosts = [{'host': host, 'port': port, 'headers': {'host': signature_host}}],
+    http_auth = RequestsAWSV4SignerAuth(credentials, region, service),
+    use_ssl = True,
+    verify_certs = True,
+    connection_class = RequestsHttpConnection,
+    pool_maxsize = 20
+)
+
+# Async
+async_client = AsyncOpenSearch(
+    hosts = [{'host': host, 'port': port, 'headers': {'host': signature_host}}],
+    http_auth = AWSV4SignerAsyncAuth(credentials, region, service),
+    use_ssl = True,
+    verify_certs = True,
+    connection_class = AsyncHttpConnection
+)
+
+```
+
 ## Kerberos
 
 There are several python packages that provide Kerberos support over HTTP, such as [requests-kerberos](http://pypi.org/project/requests-kerberos) and [requests-gssapi](https://pypi.org/project/requests-gssapi). The following example shows how to setup Kerberos authentication. 

opensearchpy/helpers/asyncsigner.py

@@ -8,6 +8,7 @@
 # GitHub history for details.
 
 from typing import Any, Dict, Optional, Union
+from urllib.parse import parse_qs, urlencode, urlparse
 
 
 class AWSV4SignerAsyncAuth:
@@ -34,8 +35,9 @@ class AWSV4SignerAsyncAuth:
         url: str,
         query_string: Optional[str] = None,
         body: Optional[Union[str, bytes]] = None,
+        headers: Optional[Dict[str, str]] = None,
     ) -> Dict[str, str]:
-        return self._sign_request(method, url, query_string, body)
+        return self._sign_request(method, url, query_string, body, headers)
 
     def _sign_request(
         self,
@@ -43,6 +45,7 @@ class AWSV4SignerAsyncAuth:
         url: str,
         query_string: Optional[str],
         body: Optional[Union[str, bytes]],
+        headers: Optional[Dict[str, str]],
     ) -> Dict[str, str]:
         """
         This method helps in signing the request by injecting the required headers.
@@ -53,10 +56,12 @@ class AWSV4SignerAsyncAuth:
         from botocore.auth import SigV4Auth
         from botocore.awsrequest import AWSRequest
 
+        signature_host = self._fetch_url(url, headers or dict())
+
         # create an AWS request object and sign it using SigV4Auth
         aws_request = AWSRequest(
             method=method,
-            url=url,
+            url=signature_host,
             data=body,
         )
 
@@ -80,3 +85,26 @@ class AWSV4SignerAsyncAuth:
 
         # copy the headers from AWS request object into the prepared_request
         return dict(aws_request.headers.items())
+
+    def _fetch_url(self, url: str, headers: Optional[Dict[str, str]]) -> str:
+        """
+        This is a util method that helps in reconstructing the request url.
+        :param prepared_request: unsigned request
+        :return: reconstructed url
+        """
+        parsed_url = urlparse(url)
+        path = parsed_url.path or "/"
+
+        # fetch the query string if present in the request
+        querystring = ""
+        if parsed_url.query:
+            querystring = "?" + urlencode(
+                parse_qs(parsed_url.query, keep_blank_values=True), doseq=True
+            )
+
+        # fetch the host information from headers
+        headers = {key.lower(): value for key, value in (headers or dict()).items()}
+        location = headers.get("host") or parsed_url.netloc
+
+        # construct the url and return
+        return parsed_url.scheme + "://" + location + path + querystring

opensearchpy/helpers/signer.py

@@ -92,14 +92,14 @@ class RequestsAWSV4SignerAuth(requests.auth.AuthBase):
         prepared_request.headers.update(
             self.signer.sign(
                 prepared_request.method,
-                self._fetch_url(prepared_request),  # type: ignore
+                self._fetch_url(prepared_request),
                 prepared_request.body,
             )
         )
 
         return prepared_request
 
-    def _fetch_url(self, prepared_request):  # type: ignore
+    def _fetch_url(self, prepared_request: requests.PreparedRequest) -> str:
         """
         This is a util method that helps in reconstructing the request url.
         :param prepared_request: unsigned request
@@ -112,7 +112,7 @@ class RequestsAWSV4SignerAuth(requests.auth.AuthBase):
         querystring = ""
         if url.query:
             querystring = "?" + urlencode(
-                parse_qs(url.query, keep_blank_values=True), doseq=True
+                parse_qs(url.query, keep_blank_values=True), doseq=True  # type: ignore
             )
 
         # fetch the host information from headers
@@ -122,7 +122,7 @@ class RequestsAWSV4SignerAuth(requests.auth.AuthBase):
         location = headers.get("host") or url.netloc
 
         # construct the url and return
-        return url.scheme + "://" + location + path + querystring
+        return url.scheme + "://" + location + path + querystring  # type: ignore
 
 
 # Deprecated: use RequestsAWSV4SignerAuth

test_opensearchpy/test_async/test_signer.py

@@ -77,6 +77,20 @@ class TestAsyncSigner:
         assert "X-Amz-Security-Token" in headers
         assert "X-Amz-Content-SHA256" in headers
 
+    async def test_aws_signer_async_fetch_url_with_querystring(self) -> None:
+        region = "us-west-2"
+        service = "aoss"
+
+        from opensearchpy.helpers.asyncsigner import AWSV4SignerAsyncAuth
+
+        auth = AWSV4SignerAsyncAuth(self.mock_session(), region, service)
+
+        signature_host = auth._fetch_url(
+            "http://localhost/?foo=bar", headers={"host": "otherhost"}
+        )
+
+        assert signature_host == "http://otherhost/?foo=bar"
+
 
 class TestAsyncSignerWithFrozenCredentials(TestAsyncSigner):
     def mock_session(self, disable_get_frozen: bool = True) -> Mock:
@@ -143,6 +157,7 @@ class TestAsyncSignerWithSpecialCharacters:
                 url: str,
                 query_string: Optional[str] = None,
                 body: Optional[Union[str, bytes]] = None,
+                headers: Optional[Dict[str, str]] = None,
             ) -> Dict[str, str]:
                 nonlocal signed_url
                 signed_url = url

test_opensearchpy/test_connection/test_requests_http_connection.py

@@ -457,6 +457,23 @@ class TestRequestsHttpConnection(TestCase):
 
         return dummy_session
 
+    def test_aws_signer_fetch_url_with_querystring(self) -> None:
+        region = "us-west-2"
+
+        import requests
+
+        from opensearchpy.helpers.signer import RequestsAWSV4SignerAuth
+
+        auth = RequestsAWSV4SignerAuth(self.mock_session(), region)
+
+        prepared_request = requests.Request(
+            "GET", "http://localhost/?foo=bar", headers={"host": "otherhost:443"}
+        ).prepare()
+
+        signature_host = auth._fetch_url(prepared_request)
+
+        assert signature_host == "http://otherhost:443/?foo=bar"
+
     def test_aws_signer_as_http_auth(self) -> None:
         region = "us-west-2"