Incorporated API generation into CI workflow and fixed 'generate' nox session (#660)

Signed-off-by: saimedhi <[email protected]>

.github/workflows/ci.yml

@@ -19,6 +19,21 @@ jobs:
       - name: Lint the code
         run: nox -s lint
   
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python3.7 -m pip install nox
+      - name: Run the api generator
+        run: nox -s generate
+
   test-build-distribution:
     runs-on: ubuntu-latest
     steps:

CHANGELOG.md

@@ -8,6 +8,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added pylint `pointless-statement` ([#611](https://github.com/opensearch-project/opensearch-py/pull/611))
 - Added a log collection guide ([#579](https://github.com/opensearch-project/opensearch-py/pull/579))
 - Added GHA release ([#614](https://github.com/opensearch-project/opensearch-py/pull/614))
+- Incorporated API generation into CI workflow and fixed 'generate' nox session ([#660](https://github.com/opensearch-project/opensearch-py/pull/660))
 ### Changed
 - Updated the `get_policy` API in the index_management plugin to allow the policy_id argument as optional ([#633](https://github.com/opensearch-project/opensearch-py/pull/633))
 ### Deprecated

noxfile.py

@@ -74,7 +74,7 @@ def format(session: Any) -> None:
     session.run("black", *SOURCE_FILES)
     session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
 
-    lint(session)
+    session.notify("lint")
 
 
 @nox.session(python=["3.7"])  # type: ignore
@@ -146,4 +146,4 @@ def generate(session: Any) -> None:
     """
     session.install("-rdev-requirements.txt")
     session.run("python", "utils/generate_api.py")
-    format(session)
+    session.notify("format")

opensearchpy/_async/client/__init__.py

@@ -1610,6 +1610,7 @@ class AsyncOpenSearch(Client):
         "rest_total_hits_as_int",
         "routing",
         "scroll",
+        "search_pipeline",
         "search_type",
         "seq_no_primary_term",
         "size",
@@ -1704,6 +1705,8 @@ class AsyncOpenSearch(Client):
         :arg routing: Comma-separated list of specific routing values.
         :arg scroll: Specify how long a consistent view of the index
             should be maintained for scrolled search.
+        :arg search_pipeline: Customizable sequence of processing stages
+            applied to search queries.
         :arg search_type: Search operation type. Valid choices are
             query_then_fetch, dfs_query_then_fetch.
         :arg seq_no_primary_term: Specify whether to return sequence

opensearchpy/_async/client/cat.py

@@ -70,7 +70,7 @@ class CatClient(NamespacedClient):
             "GET", _make_path("_cat", "aliases", name), params=params, headers=headers
         )
 
-    @query_params()
+    @query_params("bytes", "format", "h", "help", "s", "v")
     async def all_pit_segments(
         self,
         params: Any = None,
@@ -79,6 +79,16 @@ class CatClient(NamespacedClient):
         """
         Lists all active point-in-time segments.
 
+
+        :arg bytes: The unit in which to display byte values. Valid
+            choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
+        :arg format: A short version of the Accept header, e.g. json,
+            yaml.
+        :arg h: Comma-separated list of column names to display.
+        :arg help: Return help information. Default is false.
+        :arg s: Comma-separated list of column names or column aliases
+            to sort by.
+        :arg v: Verbose mode. Display column headers. Default is false.
         """
         return await self.transport.perform_request(
             "GET", "/_cat/pit_segments/_all", params=params, headers=headers
@@ -503,7 +513,7 @@ class CatClient(NamespacedClient):
             "GET", "/_cat/pending_tasks", params=params, headers=headers
         )
 
-    @query_params()
+    @query_params("bytes", "format", "h", "help", "s", "v")
     async def pit_segments(
         self,
         body: Any = None,
@@ -514,6 +524,15 @@ class CatClient(NamespacedClient):
         List segments for one or several PITs.
 
 
+        :arg bytes: The unit in which to display byte values. Valid
+            choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
+        :arg format: A short version of the Accept header, e.g. json,
+            yaml.
+        :arg h: Comma-separated list of column names to display.
+        :arg help: Return help information. Default is false.
+        :arg s: Comma-separated list of column names or column aliases
+            to sort by.
+        :arg v: Verbose mode. Display column headers. Default is false.
         """
         return await self.transport.perform_request(
             "GET", "/_cat/pit_segments", params=params, headers=headers, body=body
@@ -632,15 +651,20 @@ class CatClient(NamespacedClient):
 
     @query_params(
         "active_only",
+        "allow_no_indices",
         "bytes",
         "completed_only",
         "detailed",
+        "expand_wildcards",
         "format",
         "h",
         "help",
+        "ignore_throttled",
+        "ignore_unavailable",
         "s",
         "shards",
         "time",
+        "timeout",
         "v",
     )
     async def segment_replication(
@@ -658,21 +682,32 @@ class CatClient(NamespacedClient):
             names to limit the returned information.
         :arg active_only: If `true`, the response only includes ongoing
             segment replication events. Default is false.
+        :arg allow_no_indices: Whether to ignore if a wildcard indices
+            expression resolves into no concrete indices. (This includes `_all`
+            string or when no indices have been specified).
         :arg bytes: The unit in which to display byte values. Valid
             choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
         :arg completed_only: If `true`, the response only includes
             latest completed segment replication events. Default is false.
         :arg detailed: If `true`, the response includes detailed
             information about segment replications. Default is false.
+        :arg expand_wildcards: Whether to expand wildcard expression to
+            concrete indices that are open, closed or both. Valid choices are all,
+            open, closed, hidden, none.
         :arg format: A short version of the Accept header, e.g. json,
             yaml.
         :arg h: Comma-separated list of column names to display.
         :arg help: Return help information. Default is false.
+        :arg ignore_throttled: Whether specified concrete, expanded or
+            aliased indices should be ignored when throttled.
+        :arg ignore_unavailable: Whether specified concrete indices
+            should be ignored when unavailable (missing or closed).
         :arg s: Comma-separated list of column names or column aliases
             to sort by.
         :arg shards: Comma-separated list of shards to display.
         :arg time: The unit in which to display time values. Valid
             choices are d, h, m, s, ms, micros, nanos.
+        :arg timeout: Operation timeout.
         :arg v: Verbose mode. Display column headers. Default is false.
         """
         return await self.transport.perform_request(

opensearchpy/_async/client/indices.py

@@ -166,7 +166,12 @@ class IndicesClient(NamespacedClient):
         )
 
     @query_params(
-        "cluster_manager_timeout", "master_timeout", "timeout", "wait_for_active_shards"
+        "cluster_manager_timeout",
+        "master_timeout",
+        "task_execution_timeout",
+        "timeout",
+        "wait_for_active_shards",
+        "wait_for_completion",
     )
     async def clone(
         self,
@@ -189,9 +194,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the cloned index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:
@@ -260,8 +269,10 @@ class IndicesClient(NamespacedClient):
         "expand_wildcards",
         "ignore_unavailable",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     async def open(
         self,
@@ -287,9 +298,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Sets the number of active shards to
             wait for before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         if index in SKIP_IN_PATH:
             raise ValueError("Empty value passed for a required argument 'index'.")
@@ -1245,6 +1260,7 @@ class IndicesClient(NamespacedClient):
         "ignore_unavailable",
         "max_num_segments",
         "only_expunge_deletes",
+        "wait_for_completion",
     )
     async def forcemerge(
         self,
@@ -1272,6 +1288,8 @@ class IndicesClient(NamespacedClient):
             be merged into (default: dynamic).
         :arg only_expunge_deletes: Specify whether the operation should
             only expunge deleted documents.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         return await self.transport.perform_request(
             "POST", _make_path(index, "_forcemerge"), params=params, headers=headers
@@ -1281,8 +1299,10 @@ class IndicesClient(NamespacedClient):
         "cluster_manager_timeout",
         "copy_settings",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     async def shrink(
         self,
@@ -1307,9 +1327,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the shrunken index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:
@@ -1327,8 +1351,10 @@ class IndicesClient(NamespacedClient):
         "cluster_manager_timeout",
         "copy_settings",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     async def split(
         self,
@@ -1354,9 +1380,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the shrunken index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:

opensearchpy/_async/client/nodes.py

@@ -122,7 +122,8 @@ class NodesClient(NamespacedClient):
             nodes.
         :arg metric: Limit the information returned to the specified
             metrics. Valid choices are _all, breaker, fs, http, indices, jvm, os,
-            process, thread_pool, transport, discovery, indexing_pressure.
+            process, thread_pool, transport, discovery, indexing_pressure,
+            search_pipeline.
         :arg index_metric: Limit the information returned for `indices`
             metric to the specific index metrics. Isn't used if `indices` (or `all`)
             metric isn't specified. Valid choices are _all, store, indexing, get,

opensearchpy/client/__init__.py

@@ -1610,6 +1610,7 @@ class OpenSearch(Client):
         "rest_total_hits_as_int",
         "routing",
         "scroll",
+        "search_pipeline",
         "search_type",
         "seq_no_primary_term",
         "size",
@@ -1704,6 +1705,8 @@ class OpenSearch(Client):
         :arg routing: Comma-separated list of specific routing values.
         :arg scroll: Specify how long a consistent view of the index
             should be maintained for scrolled search.
+        :arg search_pipeline: Customizable sequence of processing stages
+            applied to search queries.
         :arg search_type: Search operation type. Valid choices are
             query_then_fetch, dfs_query_then_fetch.
         :arg seq_no_primary_term: Specify whether to return sequence

opensearchpy/client/cat.py

@@ -70,7 +70,7 @@ class CatClient(NamespacedClient):
             "GET", _make_path("_cat", "aliases", name), params=params, headers=headers
         )
 
-    @query_params()
+    @query_params("bytes", "format", "h", "help", "s", "v")
     def all_pit_segments(
         self,
         params: Any = None,
@@ -79,6 +79,16 @@ class CatClient(NamespacedClient):
         """
         Lists all active point-in-time segments.
 
+
+        :arg bytes: The unit in which to display byte values. Valid
+            choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
+        :arg format: A short version of the Accept header, e.g. json,
+            yaml.
+        :arg h: Comma-separated list of column names to display.
+        :arg help: Return help information. Default is false.
+        :arg s: Comma-separated list of column names or column aliases
+            to sort by.
+        :arg v: Verbose mode. Display column headers. Default is false.
         """
         return self.transport.perform_request(
             "GET", "/_cat/pit_segments/_all", params=params, headers=headers
@@ -503,7 +513,7 @@ class CatClient(NamespacedClient):
             "GET", "/_cat/pending_tasks", params=params, headers=headers
         )
 
-    @query_params()
+    @query_params("bytes", "format", "h", "help", "s", "v")
     def pit_segments(
         self,
         body: Any = None,
@@ -514,6 +524,15 @@ class CatClient(NamespacedClient):
         List segments for one or several PITs.
 
 
+        :arg bytes: The unit in which to display byte values. Valid
+            choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
+        :arg format: A short version of the Accept header, e.g. json,
+            yaml.
+        :arg h: Comma-separated list of column names to display.
+        :arg help: Return help information. Default is false.
+        :arg s: Comma-separated list of column names or column aliases
+            to sort by.
+        :arg v: Verbose mode. Display column headers. Default is false.
         """
         return self.transport.perform_request(
             "GET", "/_cat/pit_segments", params=params, headers=headers, body=body
@@ -632,15 +651,20 @@ class CatClient(NamespacedClient):
 
     @query_params(
         "active_only",
+        "allow_no_indices",
         "bytes",
         "completed_only",
         "detailed",
+        "expand_wildcards",
         "format",
         "h",
         "help",
+        "ignore_throttled",
+        "ignore_unavailable",
         "s",
         "shards",
         "time",
+        "timeout",
         "v",
     )
     def segment_replication(
@@ -658,21 +682,32 @@ class CatClient(NamespacedClient):
             names to limit the returned information.
         :arg active_only: If `true`, the response only includes ongoing
             segment replication events. Default is false.
+        :arg allow_no_indices: Whether to ignore if a wildcard indices
+            expression resolves into no concrete indices. (This includes `_all`
+            string or when no indices have been specified).
         :arg bytes: The unit in which to display byte values. Valid
             choices are b, k, kb, m, mb, g, gb, t, tb, p, pb.
         :arg completed_only: If `true`, the response only includes
             latest completed segment replication events. Default is false.
         :arg detailed: If `true`, the response includes detailed
             information about segment replications. Default is false.
+        :arg expand_wildcards: Whether to expand wildcard expression to
+            concrete indices that are open, closed or both. Valid choices are all,
+            open, closed, hidden, none.
         :arg format: A short version of the Accept header, e.g. json,
             yaml.
         :arg h: Comma-separated list of column names to display.
         :arg help: Return help information. Default is false.
+        :arg ignore_throttled: Whether specified concrete, expanded or
+            aliased indices should be ignored when throttled.
+        :arg ignore_unavailable: Whether specified concrete indices
+            should be ignored when unavailable (missing or closed).
         :arg s: Comma-separated list of column names or column aliases
             to sort by.
         :arg shards: Comma-separated list of shards to display.
         :arg time: The unit in which to display time values. Valid
             choices are d, h, m, s, ms, micros, nanos.
+        :arg timeout: Operation timeout.
         :arg v: Verbose mode. Display column headers. Default is false.
         """
         return self.transport.perform_request(

opensearchpy/client/indices.py

@@ -166,7 +166,12 @@ class IndicesClient(NamespacedClient):
         )
 
     @query_params(
-        "cluster_manager_timeout", "master_timeout", "timeout", "wait_for_active_shards"
+        "cluster_manager_timeout",
+        "master_timeout",
+        "task_execution_timeout",
+        "timeout",
+        "wait_for_active_shards",
+        "wait_for_completion",
     )
     def clone(
         self,
@@ -189,9 +194,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the cloned index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:
@@ -260,8 +269,10 @@ class IndicesClient(NamespacedClient):
         "expand_wildcards",
         "ignore_unavailable",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     def open(
         self,
@@ -287,9 +298,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Sets the number of active shards to
             wait for before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         if index in SKIP_IN_PATH:
             raise ValueError("Empty value passed for a required argument 'index'.")
@@ -1245,6 +1260,7 @@ class IndicesClient(NamespacedClient):
         "ignore_unavailable",
         "max_num_segments",
         "only_expunge_deletes",
+        "wait_for_completion",
     )
     def forcemerge(
         self,
@@ -1272,6 +1288,8 @@ class IndicesClient(NamespacedClient):
             be merged into (default: dynamic).
         :arg only_expunge_deletes: Specify whether the operation should
             only expunge deleted documents.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         return self.transport.perform_request(
             "POST", _make_path(index, "_forcemerge"), params=params, headers=headers
@@ -1281,8 +1299,10 @@ class IndicesClient(NamespacedClient):
         "cluster_manager_timeout",
         "copy_settings",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     def shrink(
         self,
@@ -1307,9 +1327,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the shrunken index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:
@@ -1327,8 +1351,10 @@ class IndicesClient(NamespacedClient):
         "cluster_manager_timeout",
         "copy_settings",
         "master_timeout",
+        "task_execution_timeout",
         "timeout",
         "wait_for_active_shards",
+        "wait_for_completion",
     )
     def split(
         self,
@@ -1354,9 +1380,13 @@ class IndicesClient(NamespacedClient):
         :arg master_timeout (Deprecated: To promote inclusive language,
             use 'cluster_manager_timeout' instead.): Operation timeout for
             connection to master node.
+        :arg task_execution_timeout: Explicit task execution timeout,
+            only useful when wait_for_completion is false, defaults to 1h.
         :arg timeout: Operation timeout.
         :arg wait_for_active_shards: Set the number of active shards to
             wait for on the shrunken index before the operation returns.
+        :arg wait_for_completion: Should this request wait until the
+            operation has completed before returning. Default is True.
         """
         for param in (index, target):
             if param in SKIP_IN_PATH:

opensearchpy/client/nodes.py

@@ -122,7 +122,8 @@ class NodesClient(NamespacedClient):
             nodes.
         :arg metric: Limit the information returned to the specified
             metrics. Valid choices are _all, breaker, fs, http, indices, jvm, os,
-            process, thread_pool, transport, discovery, indexing_pressure.
+            process, thread_pool, transport, discovery, indexing_pressure,
+            search_pipeline.
         :arg index_metric: Limit the information returned for `indices`
             metric to the specific index metrics. Isn't used if `indices` (or `all`)
             metric isn't specified. Valid choices are _all, store, indexing, get,