Pylint integration updates (#643)

* updated files with docstrings to pass pylint Signed-off-by: Mark Cohen <markcoh@amazon.com> * updated samples to prepare for enabling missing-docstring linter; will continue to work on this before committing setup.cfg Signed-off-by: Mark Cohen <markcoh@amazon.com> * removed missing-function-docstring from setup.cfg so the linter doesn't fail while work on docstrings continues Signed-off-by: Mark Cohen <markcoh@amazon.com> * corrected unnecessary return docstring values Signed-off-by: Mark Cohen <markcoh@amazon.com> * fixing failure in 'black' on reformatting Signed-off-by: Mark Cohen <markcoh@amazon.com> * updated utils to pass missing-function-docstring tests Signed-off-by: Mark Cohen <markcoh@amazon.com> * updated functions with missing docstrings or pylint ignore instructions; added a utility to automatically add these ignore instructions to most functions that should be self-describing; rolled back some automatically generated code mistakenly changed Signed-off-by: Mark Cohen <markcoh@amazon.com> * * ignoring opensearchpy for pylint and then added it back to noxfile.py * fixed some lints; created a feature flag for newer dynamic pylint so now lints can be fixed first in legacy code and then enabled by multiple people * extracted a method for per-folder linting * updated noxfile.lint_per_folder with type hints * enabled unspecified-encoding in pylint * added disable missing-function-docstring pragma to test_clients.py in test_async and test_server * added more encodings to pass unspecified-encoding pylint tests * updated changelog Signed-off-by: Mark Cohen <markcoh@amazon.com> * updated CHANGELOG.md entry removed the feature flag for pylint lint_per_folder fixed failures from mypy and pylint removed pylint MESSAGE CONTROL config from setup.cfg after relocating to lint_per_folder method Signed-off-by: Mark Cohen <markcoh@amazon.com> * removed pylint ignore missing-function-docstring Signed-off-by: Mark Cohen <markcoh@amazon.com> * added pylint.extensions.docparams plugin updated some docstrings to correct parameters removed pylint from setup.cfg Signed-off-by: Mark Cohen <markcoh@amazon.com> * added four lints for opensearchpy/ Signed-off-by: Mark Cohen <markcoh@amazon.com> * adding await back to client.info() call Signed-off-by: Mark Cohen <markcoh@amazon.com> * updated TODOs as requested renamed test_opensearchpy.test_async.test_server.test_helpers.conftest.setup_ubq_tests to setup_update_by_query_tests added OpenSearch-main/rest-api-spec/src/main/resources/rest-api-spec/test/indices/stats/50_noop_update[0] to skip tests list run_tests.py catches a CalledProcessError when the git repo already exists and the command to add the origin fails in fetch_opensearch_repo() Signed-off-by: Mark Cohen <markcoh@amazon.com> --------- Signed-off-by: Mark Cohen <markcoh@amazon.com>
2024-01-19 13:36:05 -05:00
parent 2ab3a40307
commit 0ddbf8cafa
47 changed files with 563 additions and 218 deletions
@@ -36,8 +36,9 @@ async def index_records(client: Any, index_name: str, item_count: int) -> None:

 async def test_async(client_count: int = 1, item_count: int = 1) -> None:
    """
-    asynchronously index with item_count records and run client_count clients. This function can be used to
-    test balancing the number of items indexed with the number of documents.
+    asynchronously index with item_count records and run client_count
+    clients. This function can be used to test balancing the number of
+    items indexed with the number of documents.
    """
    host = "localhost"
    port = 9200
@@ -22,13 +22,12 @@ from opensearchpy import OpenSearch

 def get_info(client: Any, request_count: int) -> float:
    """get info from client"""
-    tt: float = 0
-    for n in range(request_count):
+    total_time: float = 0
+    for request in range(request_count):
        start = time.time() * 1000
        client.info()
-        total_time = time.time() * 1000 - start
-        tt += total_time
-    return tt
+        total_time += time.time() * 1000 - start
+    return total_time


 def test(thread_count: int = 1, request_count: int = 1, client_count: int = 1) -> None:
@@ -71,8 +70,8 @@ def test(thread_count: int = 1, request_count: int = 1, client_count: int = 1) -
        thread.start()

    latency = 0
-    for t in threads:
-        latency += t.join()
+    for thread in threads:
+        latency += thread.join()

    print(f"latency={latency}")

@@ -23,29 +23,29 @@ from opensearchpy import OpenSearch, Urllib3HttpConnection

 def index_records(client: Any, index_name: str, item_count: int) -> Any:
    """bulk index item_count records into index_name"""
-    tt = 0
-    for n in range(10):
+    total_time = 0
+    for iteration in range(10):
        data: Any = []
-        for i in range(item_count):
+        for item in range(item_count):
            data.append(
                json.dumps({"index": {"_index": index_name, "_id": str(uuid.uuid4())}})
            )
-            data.append(json.dumps({"value": i}))
+            data.append(json.dumps({"value": item}))
        data = "\n".join(data)

        start = time.time() * 1000
-        rc = client.bulk(data)
-        if rc["errors"]:
-            raise Exception(rc["errors"])
+        response = client.bulk(data)
+        if response["errors"]:
+            raise Exception(response["errors"])

-        server_time = rc["took"]
-        total_time = time.time() * 1000 - start
+        server_time = response["took"]
+        this_time = time.time() * 1000 - start

-        if total_time < server_time:
-            raise Exception(f"total={total_time} < server={server_time}")
+        if this_time < server_time:
+            raise Exception(f"total={this_time} < server={server_time}")

-        tt += total_time - server_time
-    return tt
+        total_time += this_time - server_time
+    return total_time


 def test(thread_count: int = 1, item_count: int = 1, client_count: int = 1) -> None:
@@ -105,8 +105,8 @@ def test(thread_count: int = 1, item_count: int = 1, client_count: int = 1) -> N
        thread.start()

    latency = 0
-    for t in threads:
-        latency += t.join()
+    for thread in threads:
+        latency += thread.join()

    clients[0].indices.refresh(index=index_name)
    count = clients[0].count(index=index_name)