Pylint integration updates (#643)

* updated files with docstrings to pass pylint

Signed-off-by: Mark Cohen <[email protected]>

* updated samples to prepare for enabling missing-docstring linter; will continue to work on this before committing setup.cfg

Signed-off-by: Mark Cohen <[email protected]>

* removed missing-function-docstring from setup.cfg so the linter doesn't fail while work on docstrings continues

Signed-off-by: Mark Cohen <[email protected]>

* corrected unnecessary return docstring values

Signed-off-by: Mark Cohen <[email protected]>

* fixing failure in 'black' on reformatting

Signed-off-by: Mark Cohen <[email protected]>

* updated utils to pass missing-function-docstring tests

Signed-off-by: Mark Cohen <[email protected]>

* updated functions with missing docstrings or pylint ignore instructions; added a utility to automatically add these ignore instructions to most functions that should be self-describing; rolled back some automatically generated code mistakenly changed

Signed-off-by: Mark Cohen <[email protected]>

* * ignoring opensearchpy for pylint and then added it back to noxfile.py
* fixed some lints; created a feature flag for newer dynamic pylint so now lints can be fixed first in legacy code and then enabled by multiple people
* extracted a method for per-folder linting
* updated noxfile.lint_per_folder with type hints
* enabled unspecified-encoding in pylint
* added disable missing-function-docstring pragma to test_clients.py in test_async and test_server
* added more encodings to pass unspecified-encoding pylint tests
* updated changelog
Signed-off-by: Mark Cohen <[email protected]>

* updated CHANGELOG.md entry
removed the feature flag for pylint lint_per_folder
fixed failures from mypy and pylint
removed pylint MESSAGE CONTROL config from setup.cfg after relocating to lint_per_folder method
Signed-off-by: Mark Cohen <[email protected]>

* removed pylint ignore missing-function-docstring

Signed-off-by: Mark Cohen <[email protected]>

* added pylint.extensions.docparams plugin

updated some docstrings to correct parameters

removed pylint from setup.cfg

Signed-off-by: Mark Cohen <[email protected]>

* added four lints for opensearchpy/

Signed-off-by: Mark Cohen <[email protected]>

* adding await back to client.info() call

Signed-off-by: Mark Cohen <[email protected]>

* updated TODOs as requested

renamed test_opensearchpy.test_async.test_server.test_helpers.conftest.setup_ubq_tests to setup_update_by_query_tests

added
OpenSearch-main/rest-api-spec/src/main/resources/rest-api-spec/test/indices/stats/50_noop_update[0]
to skip tests list

run_tests.py catches a CalledProcessError when the git repo already exists and the command to add the origin fails in fetch_opensearch_repo()

Signed-off-by: Mark Cohen <[email protected]>

---------

Signed-off-by: Mark Cohen <[email protected]>

test_opensearchpy/test_async/test_connection.py

@@ -88,7 +88,8 @@ class TestAIOHttpConnection:
             # it means SSLContext is not available for that version of python
             # and we should skip this test.
             pytest.skip(
-                "Test test_ssl_context is skipped cause SSLContext is not available for this version of Python"
+                "Test test_ssl_context is skipped cause SSLContext is "
+                "not available for this version of Python"
             )
 
         con = AIOHttpConnection(use_ssl=True, ssl_context=context)
@@ -202,8 +203,8 @@ class TestAIOHttpConnection:
             con = AIOHttpConnection(use_ssl=True, verify_certs=False)
             assert 1 == len(w)
             assert (
-                "Connecting to https://localhost:9200 using SSL with verify_certs=False is insecure."
-                == str(w[0].message)
+                "Connecting to https://localhost:9200 using SSL with "
+                "verify_certs=False is insecure." == str(w[0].message)
             )
 
         assert con.use_ssl
@@ -379,13 +380,17 @@ class TestConnectionHttpServer:
 
     @classmethod
     def setup_class(cls) -> None:
-        # Start server
+        """
+        Start server
+        """
         cls.server = TestHTTPServer(port=8081)
         cls.server.start()
 
     @classmethod
     def teardown_class(cls) -> None:
-        # Stop server
+        """
+        stop server
+        """
         cls.server.stop()
 
     async def httpserver(self, conn: Any, **kwargs: Any) -> Any:

test_opensearchpy/test_async/test_helpers/conftest.py

@@ -22,6 +22,10 @@ pytestmark: MarkDecorator = pytest.mark.asyncio
 
 @fixture  # type: ignore
 async def mock_client(dummy_response: Any) -> Any:
+    """
+    yields a mock client with the dummy_response param
+    :param dummy_response: any kind of response for test
+    """
     client = Mock()
     client.search.return_value = dummy_response
     await add_connection("mock", client)

test_opensearchpy/test_async/test_plugins_client.py

@@ -26,5 +26,6 @@ class TestPluginsClient:
             client.plugins.__init__(client)  # type: ignore
             assert (
                 str(w[0].message)
-                == "Cannot load `alerting` directly to AsyncOpenSearch as it already exists. Use `AsyncOpenSearch.plugin.alerting` instead."
+                == "Cannot load `alerting` directly to AsyncOpenSearch as it already exists. Use "
+                "`AsyncOpenSearch.plugin.alerting` instead."
             )

test_opensearchpy/test_async/test_server/__init__.py

@@ -34,13 +34,19 @@ from ...utils import wipe_cluster
 
 
 class AsyncOpenSearchTestCase(IsolatedAsyncioTestCase):  # type: ignore
-    async def asyncSetUp(self) -> None:  # pylint: disable=invalid-name
+    async def asyncSetUp(
+        self,
+    ) -> None:
+        # pylint: disable=invalid-name,missing-function-docstring
         self.client = await get_test_client(
             verify_certs=False, http_auth=("admin", "admin")
         )
         await add_connection("default", self.client)
 
-    async def asyncTearDown(self) -> None:  # pylint: disable=invalid-name
+    async def asyncTearDown(
+        self,
+    ) -> None:
+        # pylint: disable=invalid-name,missing-function-docstring
         wipe_cluster(self.client)
         if self.client:
             await self.client.close()

test_opensearchpy/test_async/test_server/test_clients.py

@@ -60,7 +60,9 @@ class TestYarlMissing:
     async def test_aiohttp_connection_works_without_yarl(
         self, async_client: Any, monkeypatch: Any
     ) -> None:
-        # This is a defensive test case for if aiohttp suddenly stops using yarl.
+        """
+        This is a defensive test case for if aiohttp suddenly stops using yarl.
+        """
         from opensearchpy._async import http_aiohttp
 
         monkeypatch.setattr(http_aiohttp, "yarl", False)

test_opensearchpy/test_async/test_server/test_helpers/conftest.py

@@ -43,12 +43,24 @@ async def client() -> Any:
 
 @fixture(scope="function")  # type: ignore
 async def opensearch_version(client: Any) -> Any:
+    """
+    yields the version of the OpenSearch cluster
+    :param client: client connection to OpenSearch
+    :return: yields major version number
+    """
     info = await client.info()
     print(info)
-    yield tuple(
-        int(x)
-        for x in re.match(r"^([0-9.]+)", info["version"]["number"]).group(1).split(".")  # type: ignore
-    )
+    yield (int(x) async for x in match_version(info))
+
+
+async def match_version(info: Any) -> Any:
+    """
+    matches the full semver server version with the given info
+    :param info: response from the OpenSearch cluster
+    """
+    match = re.match(r"^([0-9.]+)", info["version"]["number"])
+    assert match is not None
+    yield match.group(1).split(".")
 
 
 @fixture  # type: ignore
@@ -60,7 +72,9 @@ async def write_client(client: Any) -> Any:
 
 @fixture  # type: ignore
 async def data_client(client: Any) -> Any:
-    # create mappings
+    """
+    create mappings
+    """
     await create_git_index(client, "git")
     await create_flat_git_index(client, "flat-git")
     # load data
@@ -73,6 +87,11 @@ async def data_client(client: Any) -> Any:
 
 @fixture  # type: ignore
 async def pull_request(write_client: Any) -> Any:
+    """
+    create dummy pull request instance
+    :param write_client: #todo not used
+    :return: instance of PullRequest
+    """
     await PullRequest.init()
     pr = PullRequest(
         _id=42,
@@ -96,7 +115,12 @@ async def pull_request(write_client: Any) -> Any:
 
 
 @fixture  # type: ignore
-async def setup_ubq_tests(client: Any) -> str:
+async def setup_update_by_query_tests(client: Any) -> str:
+    """
+    sets up update by query tests
+    :param client:
+    :return: an index name
+    """
     index = "test-git"
     await create_git_index(client, index)
     await async_bulk(client, TEST_GIT_DATA, raise_on_error=True, refresh=True)

test_opensearchpy/test_async/test_server/test_helpers/test_actions.py

@@ -60,6 +60,9 @@ class FailingBulkClient(object):
         self._fail_with = fail_with
 
     async def bulk(self, *args: Any, **kwargs: Any) -> Any:
+        """
+        increments number of times called and, when it equals fail_at, raises self.fail_with
+        """
         self._called += 1
         if self._called in self._fail_at:
             raise self._fail_with

test_opensearchpy/test_async/test_server/test_helpers/test_faceted_search.py

@@ -56,6 +56,10 @@ class MetricSearch(AsyncFacetedSearch):
 
 @pytest.fixture(scope="function")  # type: ignore
 def commit_search_cls(opensearch_version: Any) -> Any:
+    """
+    :param opensearch_version the semver version of OpenSearch
+    :return: an AsyncFacetedSearch for git commits
+    """
     interval_kwargs = {"fixed_interval": "1d"}
 
     class CommitSearch(AsyncFacetedSearch):
@@ -102,6 +106,10 @@ def repo_search_cls(opensearch_version: Any) -> Any:
 
 @pytest.fixture(scope="function")  # type: ignore
 def pr_search_cls(opensearch_version: Any) -> Any:
+    """
+    :param opensearch_version: not used here... #TODO remove this parameter?
+    :return: an AsyncFacetedSearch for pull requests
+    """
     interval_type = "calendar_interval"
 
     class PRSearch(AsyncFacetedSearch):

test_opensearchpy/test_async/test_server/test_helpers/test_update_by_query.py

@@ -19,9 +19,9 @@ pytestmark: MarkDecorator = pytest.mark.asyncio
 
 
 async def test_update_by_query_no_script(
-    write_client: Any, setup_ubq_tests: Any
+    write_client: Any, setup_update_by_query_tests: Any
 ) -> None:
-    index = setup_ubq_tests
+    index = setup_update_by_query_tests
 
     ubq = (
         AsyncUpdateByQuery(using=write_client)
@@ -40,9 +40,9 @@ async def test_update_by_query_no_script(
 
 
 async def test_update_by_query_with_script(
-    write_client: Any, setup_ubq_tests: Any
+    write_client: Any, setup_update_by_query_tests: Any
 ) -> None:
-    index = setup_ubq_tests
+    index = setup_update_by_query_tests
 
     ubq = (
         AsyncUpdateByQuery(using=write_client)
@@ -59,9 +59,9 @@ async def test_update_by_query_with_script(
 
 
 async def test_delete_by_query_with_script(
-    write_client: Any, setup_ubq_tests: Any
+    write_client: Any, setup_update_by_query_tests: Any
 ) -> None:
-    index = setup_ubq_tests
+    index = setup_update_by_query_tests
 
     ubq = (
         AsyncUpdateByQuery(using=write_client)

test_opensearchpy/test_async/test_server/test_rest_api_spec.py

@@ -54,6 +54,10 @@ OPENSEARCH_VERSION = None
 
 
 async def await_if_coro(x: Any) -> Any:
+    """
+    awaits if x is a coroutine
+    :return: x
+    """
     if inspect.iscoroutine(x):
         return await x
     return x

test_opensearchpy/test_async/test_server_secured/test_security_plugin.py

@@ -40,13 +40,15 @@ class TestSecurityPlugin(IsolatedAsyncioTestCase):  # type: ignore
     USER_NAME = "test-user"
     USER_CONTENT = {"password": "opensearchpy@123", "opendistro_security_roles": []}
 
-    async def asyncSetUp(self) -> None:  # pylint: disable=invalid-name
+    async def asyncSetUp(self) -> None:
+        # pylint: disable=invalid-name, missing-function-docstring
         self.client = await get_test_client(
             verify_certs=False, http_auth=("admin", "admin")
         )
         await add_connection("default", self.client)
 
-    async def asyncTearDown(self) -> None:  # pylint: disable=invalid-name
+    async def asyncTearDown(self) -> None:
+        # pylint disable=invalid-name
         if self.client:
             await self.client.close()
 

test_opensearchpy/test_async/test_transport.py

@@ -449,8 +449,10 @@ class TestTransport:
         assert event_loop.time() - 1 < t.last_sniff < event_loop.time() + 0.01
 
     async def test_sniff_7x_publish_host(self) -> None:
-        # Test the response shaped when a 7.x node has publish_host set
-        # and the returend data is shaped in the fqdn/ip:port format.
+        """
+        Test the response shaped when a 7.x node has publish_host set
+        and the returned data is shaped in the fqdn/ip:port format.
+        """
         t: Any = AsyncTransport(
             [{"data": CLUSTER_NODES_7X_PUBLISH_HOST}],
             connection_class=DummyConnection,