diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cba0fedf..dd90f22a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ +# Python Elasticsearch Client + If you have a bugfix or new feature that you would like to contribute to elasticsearch-py, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that @@ -8,6 +10,32 @@ We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. +## API Code Generation + +All the API methods (any method in `elasticsearch.client` classes decorated +with `@query_params`) are actually auto-generated from the +[rest-api-spec](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec/src/main/resources/rest-api-spec/api) +found in the `Elasticsearch` repository. Any changes to those methods should be +done either by submitting a PR to Elasticsearch itself (in case of adding or +modifying any of the API methods) or to the [Generate +Script](https://github.com/elastic/elasticsearch-py/blob/master/generate_api.py). + +To run the code generation make sure you have pre-requisites installed: + +* by running `pip install -e '.[develop]'` +* having the [elasticsearch](https://github.com/elastic/elasticsearch) repo + cloned on the same level as `elasticsearch-py` and switched to appropriate + version + +Then you should be able to run the code generation by invoking: + +``` +python generate_api.py +``` + + +## Contributing Code Changes + The process for contributing to any of the Elasticsearch repositories is similar. 1. Please make sure you have signed the [Contributor License diff --git a/generate_api.py b/generate_api.py new file mode 100644 index 00000000..4f4243b1 --- /dev/null +++ b/generate_api.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python + +import os +import json +import re +from itertools import chain + +import black +from click.testing import CliRunner +from pathlib import Path + +from jinja2 import Environment, DictLoader, ChoiceLoader, TemplateNotFound + + +# line to look for in the original source file +SEPARATOR = " # AUTO-GENERATED-API-DEFINITIONS #" +# global substitutions for python keywords +SUBSTITUTIONS = {"type": "doc_type", "from": "from_"} +# api path(s) +CODE_ROOT = Path(__file__).absolute().parent +BASE_PATH = ( + CODE_ROOT.parent + / "elasticsearch" + / "rest-api-spec" + / "src" + / "main" + / "resources" + / "rest-api-spec" + / "api" +) +XPACK_PATH = ( + CODE_ROOT.parent + / "elasticsearch" + / "x-pack" + / "plugin" + / "src" + / "test" + / "resources" + / "rest-api-spec" + / "api" +) + + +BASE_TEMPLATES = { + "url": """{% if api.url_parts.0 %}_make_path({{ api.url_parts.1|join(", ")}}){% else %}{{ api.url_parts.1|tojson }}{% endif %}""", + "required": """{% if api.required_parts.1 %} + for param in ({{ api.required_parts|join(", ")}}): + if param in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument.") + + {% elif api.required_parts %} + if {{ api.required_parts.0 }} in SKIP_IN_PATH: + raise ValueError("Empty value passed for a required argument '{{ api.required_parts.0 }}'.") + + {% endif %}""", + "substitutions": """{% for p, info in api.params %} + {% if p in substitutions and p not in api.url_parts.1 %} + # {{ substitutions[p] }} is a reserved word so it cannot be used, use {{ p }} instead + if "{{ p }}" in params: + params["{{ substitutions[p] }}"] = params.pop("{{ p }}") + + {% endif %} + {% endfor %}""", + "base": ''' + @query_params({{ api.query_params|map("tojson")|join(", ")}}) + def {{ api.name }}(self, {{ api.func_params|join(", ") }}): + """ + {% if api.description %} + {{ api.description|replace("\n", " ")|wordwrap(wrapstring="\n ") }} + {% endif %} + {% if api.doc_url %} + `<{{ api.doc_url }}>`_ + {% endif %} + {% if api.params %} + + {% for p, info in api.params %} + {% filter wordwrap(72, wrapstring="\n ") %} + :arg {{ p }}: {{ info.description }} {% if info.options %}Valid choices: {{ info.options|join(", ") }}{% endif %} {% if info.default %}Default: {{ info.default }}{% endif %} + {% endfilter %} + + {% endfor %} + {% endif %} + """ + {% include "substitutions" %} + {% include "required" %} + {% if api.body.body.serialize == "bulk" %} + body = self._bulk_body(body) + {% endif %} + {% block request %} + return self.transport.perform_request("{{ api.method }}", {% include "url" %}, params=params{% if api.body.body %}, body=body{% endif %}) + {% endblock %} + ''', +} + +OVERRIDE_TEMPLATES = { + "__init__.ping": """ + {% extends "base" %} + {% block request %} + try: + {{ super()|trim }} + except TransportError: + return False + {% endblock %} + """, + "__init__.scroll": """ + {% extends "base" %} + {% block request %} + if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: + raise ValueError("You need to supply scroll_id or body.") + elif scroll_id and not body: + body = {"scroll_id": scroll_id} + elif scroll_id: + params["scroll_id"] = scroll_id + + return self.transport.perform_request("{{ api.method }}", "/_search/scroll", params=params, body=body) + {% endblock %} + """, + "__init__.clear_scroll": """ + {% extends "base" %} + {% block request %} + if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: + raise ValueError("You need to supply scroll_id or body.") + elif scroll_id and not body: + body = {"scroll_id": [scroll_id]} + elif scroll_id: + params["scroll_id"] = scroll_id + + return self.transport.perform_request("{{ api.method }}", "/_search/scroll", params=params, body=body) + {% endblock %} + """, +} + +jinja_env = Environment( + loader=ChoiceLoader((DictLoader(OVERRIDE_TEMPLATES), DictLoader(BASE_TEMPLATES))), + trim_blocks=True, + lstrip_blocks=True, +) + + +def blacken(filename): + runner = CliRunner() + result = runner.invoke(black.main, [str(filename)]) + assert result.exit_code == 0, result.output + + +class Module: + def __init__(self, namespace): + self.namespace = namespace + self._apis = [] + self.parse_orig() + + def add(self, api): + self._apis.append(api) + if api.name in self.descriptions and not api.description: + api.description = self.descriptions[api.name] + + def parse_orig(self): + self.descriptions = {} + self.orders = [] + self.header = "class C:" + fname = CODE_ROOT / "elasticsearch" / "client" / f"{self.namespace}.py" + if os.path.exists(fname): + with open(fname) as f: + content = f.read() + header_lines = [] + for line in content.split("\n"): + header_lines.append(line) + if line == SEPARATOR: + break + # no separator found + else: + header_lines = [] + for line in content.split("\n"): + header_lines.append(line) + if line.startswith('class'): + break + self.header = "\n".join(header_lines) + defined_apis = re.findall( + r'\n def ([a-z_]+)\([^\n]*\n *"""\n *([\w\W]*?)(?:`<|""")', + content, + re.MULTILINE, + ) + self.descriptions = dict( + map(lambda i: (i[0], i[1].strip()), defined_apis) + ) + self.orders = list(map(lambda x: x[0], defined_apis)) + + def _position(self, api): + try: + return self.orders.index(api.name) + except ValueError: + return len(self.orders) + + def sort(self): + self._apis.sort(key=self._position) + + def dump(self): + self.sort() + fname = CODE_ROOT / "elasticsearch" / "client" / f"{self.namespace}.py" + with open(fname, "w") as f: + f.write(self.header) + for api in self._apis: + f.write(api.to_python()) + blacken(fname) + + +class API: + def __init__(self, namespace, name, definition): + self.namespace = namespace + self.name = name + + # overwrite the dict to maintain key order + definition["params"] = { + SUBSTITUTIONS.get(p, p): v for p, v in definition.get("params", {}).items() + } + + self._def = definition + self.description = "" + self.doc_url = "" + if isinstance(definition["documentation"], str): + self.doc_url = definition["documentation"] + else: + # set as attribute so it may be overriden by Module.add + self.description = definition["documentation"].get("description", "") + self.doc_url = definition["documentation"].get("url", "") + + def _all_parts(self): + parts = {} + for url in self._def["url"]["paths"]: + parts.update(url.get("parts", {})) + + for p in parts: + parts[p]["required"] = all( + p in url.get("parts", {}) for url in self._def["url"]["paths"] + ) + + for k, sub in SUBSTITUTIONS.items(): + if k in parts: + parts[sub] = parts.pop(k) + + dynamic, components = self.url_parts + + def ind(item): + try: + return components.index(item[0]) + except ValueError: + return len(components) + + parts = dict(sorted(parts.items(), key=ind)) + return parts + + @property + def params(self): + parts = self._all_parts() + return chain( + ((p, parts[p]) for p in parts if parts[p]["required"]), + self.body.items() if self.body["body"] else (), + ((p, parts[p]) for p in parts if not parts[p]["required"]), + sorted(self._def.get("params", {}).items()), + ) + + @property + def body(self): + b = self._def.get("body", {}) + if b: + b.setdefault("required", False) + return {"body": b} + + @property + def query_params(self): + return sorted(self._def.get("params", {}).keys()) + + @property + def path(self): + return max( + (path for path in self._def["url"]["paths"]), + key=lambda p: len(re.findall(r"\{([^}]+)\}", p["path"])), + ) + + @property + def method(self): + return self.path["methods"][0] + + @property + def url_parts(self): + path = self.path["path"] + + dynamic = "{" in path + if not dynamic: + return dynamic, path + + parts = [] + for part in path.split("/"): + if not part: + continue + + if part[0] == "{": + part = part[1:-1] + parts.append(SUBSTITUTIONS.get(part, part)) + else: + parts.append(f"'{part}'") + + return dynamic, parts + + @property + def func_params(self): + parts = self._all_parts() + return chain( + (p for p in parts if parts[p]["required"]), + (b for b in self.body if self.body[b].get("required")), + (f"{b}=None" for b in self.body if not self.body[b].get("required", True)), + (f"{p}=None" for p in parts if not parts[p]["required"]), + ("params=None",), + ) + + @property + def required_parts(self): + parts = self._all_parts() + return [p for p in parts if parts[p]["required"]] + [ + b for b in self.body if self.body[b].get("required") + ] + + def to_python(self): + try: + t = jinja_env.get_template(f"{self.namespace}.{self.name}") + except TemplateNotFound: + t = jinja_env.get_template("base") + return t.render( + api=self, substitutions={v: k for k, v in SUBSTITUTIONS.items()} + ) + + +def read_modules(): + modules = {} + + for path in (BASE_PATH, XPACK_PATH): + for f in sorted(os.listdir(path)): + name, ext = f.rsplit(".", 1) + + if ext != "json" or name == "_common": + continue + + with open(path / f) as api_def: + api = json.load(api_def)[name] + + namespace = "__init__" + if "." in name: + namespace, name = name.rsplit(".", 1) + + if namespace not in modules: + modules[namespace] = Module(namespace) + + modules[namespace].add(API(namespace, name, api)) + + return modules + + +def dump_modules(modules): + for mod in modules.values(): + mod.dump() + + +if __name__ == "__main__": + dump_modules(read_modules()) diff --git a/setup.py b/setup.py index 5436d941..4bcaf589 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,9 @@ tests_require = [ "nosexcover", ] +docs_require = ["sphinx<1.7", "sphinx_rtd_theme"] +generate_require = ["black", "jinja2"] + # use external unittest for 2.6 if sys.version_info[:2] == (2, 6): install_requires.append("unittest2") @@ -56,7 +59,8 @@ setup( test_suite="test_elasticsearch.run_tests.run_all", tests_require=tests_require, extras_require={ - "develop": tests_require + ["sphinx<1.7", "sphinx_rtd_theme"], + "develop": tests_require + docs_require + generate_require, + "docs": docs_require, "requests": ["requests>=2.4.0, <3.0.0"], }, )