actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge .pyi type stubs inline (#563)

Browse Source 
* Merged types into .py code.

Signed-off-by: dblock <[email protected]>

* Fix: nox -rs generate.

Signed-off-by: dblock <[email protected]>

* Updated CHANGELOG.

Signed-off-by: dblock <[email protected]>

* Use lowest common python version for lint.

Signed-off-by: dblock <[email protected]>

* Fix: don't typeshed.

Signed-off-by: dblock <[email protected]>

* Removed unneeded comment.

Signed-off-by: dblock <[email protected]>

* Simplify OPENSEARCH_URL.

Signed-off-by: dblock <[email protected]>

* Fix: positional ignore_status used as chunk_size.

Signed-off-by: dblock <[email protected]>

* Fix: parse version string.

Signed-off-by: dblock <[email protected]>

* Remove future annotations for Python 3.6.

Signed-off-by: dblock <[email protected]>

* Fix: types in documentation.

Signed-off-by: dblock <[email protected]>

* Improve CHANGELOG text.

Signed-off-by: dblock <[email protected]>

* Re-added missing separator.

Signed-off-by: dblock <[email protected]>

* Remove duplicate licenses.

Signed-off-by: dblock <[email protected]>

* Get rid of Optional[Any].

Signed-off-by: dblock <[email protected]>

* Fix docs with AsyncOpenSearch.

Signed-off-by: dblock <[email protected]>

* Fix: undo comment.

Signed-off-by: dblock <[email protected]>

---------

Signed-off-by: dblock <[email protected]>
This commit is contained in:
Daniel (dB.) Doubrovkine
2023-11-06 10:08:19 -08:00
committed by GitHub
parent 0d8a23dd78
commit dcb79cc322
268 changed files with 6218 additions and 16378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/build-dists.py
+10 -6
View File
@@ -52,7 +52,7 @@ def set_tmp_dir():
tmp_dir = None
def run(*argv, expect_exit_code=0):
def run(*argv, expect_exit_code: int = 0) -> None:
global tmp_dir
if tmp_dir is None:
os.chdir(base_dir)
@@ -70,7 +70,7 @@ def run(*argv, expect_exit_code=0):
exit(exit_code or 1)
def test_dist(dist):
def test_dist(dist) -> None:
with set_tmp_dir() as tmp_dir:
dist_name = re.match(
r"^(opensearchpy\d*)-",
@@ -180,7 +180,7 @@ def test_dist(dist):
)
def main():
def main() -> None:
run("git", "checkout", "--", "setup.py", "opensearchpy/")
run("rm", "-rf", "build/", "dist/*", "*.egg-info", ".eggs")
run("python", "setup.py", "sdist", "bdist_wheel")
@@ -188,9 +188,13 @@ def main():
# Grab the major version to be used as a suffix.
version_path = os.path.join(base_dir, "opensearchpy/_version.py")
with open(version_path) as f:
version = re.search(
r"^__versionstr__\s+=\s+[\"\']([^\"\']+)[\"\']", f.read(), re.M
).group(1)
data = f.read()
m = re.search(r"^__versionstr__: str\s+=\s+[\"\']([^\"\']+)[\"\']", data, re.M)
if m:
version = m.group(1)
else:
raise Exception(f"Invalid version {data}")
major_version = version.split(".")[0]
# If we're handed a version from the build manager we
utils/generate-api.py
+19 -42
View File
@@ -78,7 +78,7 @@ jinja_env = Environment(
)
def blacken(filename):
def blacken(filename) -> None:
runner = CliRunner()
result = runner.invoke(black.main, [str(filename)])
assert result.exit_code == 0, result.output
@@ -90,29 +90,20 @@ def is_valid_url(url):
class Module:
def __init__(self, namespace, is_pyi=False):
def __init__(self, namespace) -> None:
self.namespace = namespace
self.is_pyi = is_pyi
self._apis = []
self.parse_orig()
if not is_pyi:
self.pyi = Module(namespace, is_pyi=True)
self.pyi.orders = self.orders[:]
def add(self, api):
def add(self, api) -> None:
self._apis.append(api)
def parse_orig(self):
self.orders = []
self.header = ""
if self.is_pyi is True:
self.header = "from typing import Any, Collection, MutableMapping, Optional, Tuple, Union\n\n"
self.header = "from typing import Any, Collection, Optional, Tuple, Union\n\n"
namespace_new = "".join(word.capitalize() for word in self.namespace.split("_"))
self.header = (
self.header + "class " + namespace_new + "Client(NamespacedClient):"
)
self.header += "class " + namespace_new + "Client(NamespacedClient):"
if os.path.exists(self.filepath):
with open(self.filepath) as f:
content = f.read()
@@ -127,12 +118,10 @@ class Module:
for line in content.split("\n"):
header_lines.append(line)
if line.startswith("class"):
if (
"security.py" in str(self.filepath)
and not self.filepath.suffix == ".pyi"
):
if "security.py" in str(self.filepath):
# TODO: FIXME, import code
header_lines.append(
" from ._patch import health_check, update_audit_config"
" from ._patch import health_check, update_audit_config # type: ignore"
)
break
self.header = "\n".join(header_lines)
@@ -146,10 +135,10 @@ class Module:
except ValueError:
return len(self.orders)
def sort(self):
def sort(self) -> None:
self._apis.sort(key=self._position)
def dump(self):
def dump(self) -> None:
self.sort()
# This code snippet adds headers to each generated module indicating that the code is generated.
@@ -244,22 +233,15 @@ class Module:
with open(self.filepath, "w") as f:
f.write(file_content)
if not self.is_pyi:
self.pyi.dump()
@property
def filepath(self):
return (
CODE_ROOT
/ f"opensearchpy/_async/client/{self.namespace}.py{'i' if self.is_pyi else ''}"
)
return CODE_ROOT / f"opensearchpy/_async/client/{self.namespace}.py"
class API:
def __init__(self, namespace, name, definition, is_pyi=False):
def __init__(self, namespace, name, definition) -> None:
self.namespace = namespace
self.name = name
self.is_pyi = is_pyi
# overwrite the dict to maintain key order
definition["params"] = {
@@ -429,13 +411,10 @@ class API:
return required
def to_python(self):
if self.is_pyi:
t = jinja_env.get_template("base_pyi")
else:
try:
t = jinja_env.get_template(f"overrides/{self.namespace}/{self.name}")
except TemplateNotFound:
t = jinja_env.get_template("base")
try:
t = jinja_env.get_template(f"overrides/{self.namespace}/{self.name}")
except TemplateNotFound:
t = jinja_env.get_template("base")
return t.render(
api=self,
@@ -658,7 +637,6 @@ def read_modules():
modules[namespace] = Module(namespace)
modules[namespace].add(API(namespace, name, api))
modules[namespace].pyi.add(API(namespace, name, api, is_pyi=True))
return modules
@@ -697,10 +675,9 @@ def dump_modules(modules):
filepaths = []
for root, _, filenames in os.walk(CODE_ROOT / "opensearchpy/_async"):
for filename in filenames:
if filename.rpartition(".")[-1] in (
"py",
"pyi",
) and not filename.startswith("utils.py"):
if filename.rpartition(".")[-1] in ("py",) and not filename.startswith(
"utils.py"
):
filepaths.append(os.path.join(root, filename))
unasync.unasync_files(filepaths, rules)
utils/license-headers.py
+2 -2
View File
@@ -48,7 +48,7 @@ def find_files_to_fix(sources: List[str]) -> Iterator[str]:
def does_file_need_fix(filepath: str) -> bool:
if not re.search(r"\.pyi?$", filepath):
if not re.search(r"\.py$", filepath):
return False
existing_header = ""
with open(filepath, mode="r") as f:
@@ -78,7 +78,7 @@ def add_header_to_file(filepath: str) -> None:
print(f"Fixed {os.path.relpath(filepath, os.getcwd())}")
def main():
def main() -> None:
mode = sys.argv[1]
assert mode in ("fix", "check")
sources = [os.path.abspath(x) for x in sys.argv[2:]]
utils/templates/base
+2 -2
View File
@@ -1,6 +1,6 @@
@query_params({{ api.query_params|map("tojson")|join(", ")}})
async def {{ api.name }}(self, {% include "func_params" %}):
@query_params({{ api.query_params|map("tojson")|join(", ")}})
async def {{ api.name }}(self, {% include "func_params" %}) -> Any:
"""
{% if api.description %}
{{ api.description|replace("\n", " ")|wordwrap(wrapstring="\n ") }}
utils/templates/base_pyi
-2
View File
@@ -1,2 +0,0 @@
async def {{ api.name }}(self, {% include "func_params_pyi" %}) -> {% if api.method == 'HEAD' %}bool{% else %}Any{% endif %}: ...
utils/templates/func_params
+6 -5
View File
@@ -1,14 +1,15 @@
{% for p, info in api.all_parts.items() %}
{% if info.required %}{{ p }}, {% endif %}
{% if info.required %}{{ p }}: {{ info.type }}, {% endif %}
{% endfor %}
{% if api.body %}
body{% if not api.body.required %}=None{% endif %},
body{% if not api.body.required %}: Any=None{% else %}: Any{% endif %},
{% endif %}
{% for p, info in api.all_parts.items() %}
{% if not info.required %}{{ p }}=None, {% endif %}
{% if not info.required and not info.type == 'Any' %}{{ p }}: Optional[{{ info.type }}]=None, {% endif %}
{% if not info.required and info.type == 'Any' %}{{ p }}: {{ info.type }}=None, {% endif %}
{% endfor %}
params=None,
headers=None
params: Any=None,
headers: Any=None,
utils/templates/func_params_pyi
-26
View File
@@ -1,26 +0,0 @@
{% for p, info in api.all_parts.items() %}
{% if info.required %}{{ p }}: {{ info.type }}, {% endif %}
{% endfor %}
*,
{% if api.body %}
body{% if not api.body.required %}: Optional[Any]=...{% else %}: Any{% endif %},
{% endif %}
{% for p, info in api.all_parts.items() %}
{% if not info.required %}{{ p }}: Optional[{{ info.type }}]=..., {% endif %}
{% endfor %}
{% for p in api.query_params %}
{{ p }}: Optional[Any]=...,
{% endfor %}
{% for p, p_type in global_query_params.items() %}
{% if p not in api.all_func_params %}
{{ p }}: {{ p_type }}=...,
{% endif %}
{% endfor %}
params: Optional[MutableMapping[str, Any]]=...,
headers: Optional[MutableMapping[str, str]]=...,