[7.x] Switch to Nox, add build_dist script

.github/workflows/ci.yml

@@ -35,9 +35,9 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          python3.7 -m pip install tox
+          python3.7 -m pip install nox
       - name: Lint the code
-        run: tox -e lint
+        run: nox -s lint
 
   docs:
     runs-on: ubuntu-latest
@@ -50,9 +50,9 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          python3.7 -m pip install tox
+          python3.7 -m pip install nox
       - name: Build the docs
-        run: tox -e docs
+        run: nox -s docs
 
   test-linux:
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -30,7 +30,7 @@ To run the code generation make sure you have pre-requisites installed:
 Then you should be able to run the code generation by invoking:
 
 ```
-python utils/generate_api.py
+$ python utils/generate_api.py
 ```
 
 
@@ -45,10 +45,17 @@ The process for contributing to any of the Elasticsearch repositories is similar
    assure our users of the origin and continuing existence of the code. You only
    need to sign the CLA once.
 
-2. Run the test suite to ensure your changes do not break existing code:
+2. Run the linter and test suite to ensure your changes do not break existing code:
 
     ````
-    python setup.py test
+    # Install Nox for task management
+    $ python -m pip install nox
+    
+    # Auto-format and lint your changes
+    $ nox -s blacken
+   
+    # Run the test suite
+    $ python setup.py test
     ````
 
    See the README file in `test_elasticsearch` directory for more information on
@@ -68,4 +75,3 @@ The process for contributing to any of the Elasticsearch repositories is similar
 Then sit back and wait. There will probably be a discussion about the pull
 request and, if any changes are needed, we would love to work with you to get
 your pull request merged into elasticsearch-py.
-

noxfile.py

@@ -0,0 +1,49 @@
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+import nox
+
+
+SOURCE_FILES = (
+    "setup.py",
+    "noxfile.py",
+    "elasticsearch/",
+    "test_elasticsearch/",
+    "utils/",
+)
+
+
+@nox.session(python=["2.7", "3.4", "3.5", "3.6", "3.7", "3.8"])
+def test(session):
+    session.install(".")
+    session.install("-r", "dev-requirements.txt")
+
+    session.run("python", "setup.py", "test")
+
+
+@nox.session()
+def blacken(session):
+    session.install("black")
+
+    session.run("black", "--target-version=py27", *SOURCE_FILES)
+    session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
+
+    lint(session)
+
+
+@nox.session()
+def lint(session):
+    session.install("flake8", "black")
+
+    session.run("black", "--target-version=py27", "--check", *SOURCE_FILES)
+    session.run("flake8", *SOURCE_FILES)
+    session.run("python", "utils/license_headers.py", "check", *SOURCE_FILES)
+
+
+@nox.session()
+def docs(session):
+    session.install(".")
+    session.install("-rdev-requirements.txt", "sphinx-rtd-theme")
+
+    session.run("sphinx-build", "docs/", "docs/_build", "-b", "html")

tox.ini

@@ -1,52 +0,0 @@
-[tox]
-envlist = pypy,py27,py34,py35,py36,py37,py38,lint,docs
-[testenv]
-whitelist_externals = git
-deps =
-    -r dev-requirements.txt
-commands =
-    python setup.py test
-
-[testenv:blacken]
-deps =
-    black
-commands =
-    black --target-version=py27 \
-        elasticsearch/ \
-        test_elasticsearch/ \
-        setup.py
-    python utils/license_headers.py fix \
-        elasticsearch/ \
-        test_elasticsearch/ \
-        utils \
-        docs \
-        setup.py
-
-[testenv:lint]
-deps =
-    flake8
-    black
-commands =
-    black --target-version=py27 --check \
-        elasticsearch/ \
-        test_elasticsearch/ \
-        setup.py
-    flake8 \
-        elasticsearch/ \
-        test_elasticsearch/ \
-        utils \
-        docs \
-        setup.py
-    python utils/license_headers.py check \
-        elasticsearch/ \
-        test_elasticsearch/ \
-        utils \
-        docs \
-        setup.py
-
-[testenv:docs]
-deps =
-    sphinx-rtd-theme
-    -r dev-requirements.txt
-commands =
-    sphinx-build docs/ docs/_build -b html

utils/build_dists.py

@@ -0,0 +1,129 @@
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+"""A command line tool for building and verifying releases
+Can be used for building both 'elasticsearch' and 'elasticsearchX' dists.
+Only requires 'name' in 'setup.py' and the directory to be changed.
+"""
+
+import tempfile
+import os
+import shlex
+import re
+import contextlib
+import shutil
+
+
+base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+tmp_dir = None
+
+
+@contextlib.contextmanager
+def set_tmp_dir():
+    global tmp_dir
+    tmp_dir = tempfile.mkdtemp()
+    yield tmp_dir
+    shutil.rmtree(tmp_dir)
+    tmp_dir = None
+
+
+def run(*argv, expect_exit_code=0):
+    global tmp_dir
+    if tmp_dir is None:
+        os.chdir(base_dir)
+    else:
+        os.chdir(tmp_dir)
+
+    cmd = " ".join(shlex.quote(x) for x in argv)
+    print("$ " + cmd)
+    exit_code = os.system(cmd)
+    if exit_code != expect_exit_code:
+        print(
+            "Command exited incorrectly: should have been %d was %d"
+            % (expect_exit_code, exit_code)
+        )
+        exit(exit_code or 1)
+
+
+def test_dist(dist):
+    with set_tmp_dir() as tmp_dir:
+        dist_name = re.match(r"^(elasticsearch\d*)-", os.path.basename(dist)).group(1)
+
+        # Build the venv and install the dist
+        run("python", "-m", "venv", os.path.join(tmp_dir, "venv"))
+        venv_python = os.path.join(tmp_dir, "venv/bin/python")
+        run(venv_python, "-m", "pip", "install", "-U", "pip")
+        run(venv_python, "-m", "pip", "install", dist)
+
+        # Test the sync namespaces
+        run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
+        )
+        run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
+        )
+
+        # Ensure that async is not available yet
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name} import AsyncElasticsearch",
+            expect_exit_code=256,
+        )
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name}.helpers import async_scan, async_bulk, async_streaming_bulk, async_reindex",
+            expect_exit_code=256,
+        )
+
+        # Install aiohttp and see that async is now available
+        run(venv_python, "-m", "pip", "install", "aiohttp")
+        run(venv_python, "-c", f"from {dist_name} import AsyncElasticsearch")
+
+        # Ensure that the namespaces are correct for the dist
+        for suffix in ("", "1", "2", "5", "6", "7", "8", "9", "10"):
+            distx_name = f"elasticsearch{suffix}"
+            run(
+                venv_python,
+                "-c",
+                f"import {distx_name}",
+                expect_exit_code=256 if distx_name != dist_name else 0,
+            )
+
+        # Uninstall the dist, see that we can't import things anymore
+        run(venv_python, "-m", "pip", "uninstall", "--yes", dist_name)
+        run(
+            venv_python,
+            "-c",
+            f"from {dist_name} import Elasticsearch",
+            expect_exit_code=256,
+        )
+
+
+def main():
+    run("rm", "-rf", "build/", "dist/", "*.egg-info", ".eggs")
+    run("python", "setup.py", "sdist", "bdist_wheel")
+
+    for dist in os.listdir(os.path.join(base_dir, "dist")):
+        test_dist(os.path.join(base_dir, "dist", dist))
+
+    # After this run 'python -m twine upload dist/*'
+    print(
+        "\n\n"
+        "===============================\n\n"
+        "    * Releases are ready! *\n\n"
+        "$ python -m twine upload dist/*\n\n"
+        "==============================="
+    )
+
+
+if __name__ == "__main__":
+    main()

utils/generate_api.py

@@ -96,9 +96,7 @@ class Module:
                             break
                 self.header = "\n".join(header_lines)
                 self.orders = re.findall(
-                    r'\n    (?:async )?def ([a-z_]+)\(',
+                    r"\n    (?:async )?def ([a-z_]+)\(", content, re.MULTILINE
-                    content,
-                    re.MULTILINE
                 )
 
     def _position(self, api):
@@ -153,7 +151,11 @@ class API:
 
             # Try setting doc refs like 'current' and 'master' to our branches ref.
             if BRANCH_NAME is not None:
-                revised_url = re.sub("/elasticsearch/reference/[^/]+/", f"/elasticsearch/reference/{BRANCH_NAME}/", self.doc_url)
+                revised_url = re.sub(
+                    "/elasticsearch/reference/[^/]+/",
+                    f"/elasticsearch/reference/{BRANCH_NAME}/",
+                    self.doc_url,
+                )
                 if is_valid_url(revised_url):
                     self.doc_url = revised_url
                 else:
@@ -199,9 +201,11 @@ class API:
         return chain(
             ((p, parts[p]) for p in parts if parts[p]["required"]),
             (("body", self.body),) if self.body else (),
-            ((p, parts[p]) for p in parts
+            (
-             if not parts[p]["required"]
+                (p, parts[p])
-             and p not in params),
+                for p in parts
+                if not parts[p]["required"] and p not in params
+            ),
             sorted(params.items(), key=lambda x: (x[0] not in parts, x[0])),
         )
 
@@ -320,7 +324,7 @@ def dump_modules(modules):
         unasync.Rule(
             fromdir="/elasticsearch/_async/client/",
             todir="/elasticsearch/client/",
-            additional_replacements=additional_replacements
+            additional_replacements=additional_replacements,
         ),
     ]
 

utils/license_headers.py

@@ -13,15 +13,12 @@ from typing import List, Iterator
 from itertools import chain
 
 
-lines_to_keep = [
+lines_to_keep = ["# -*- coding: utf-8 -*-\n", "#!/usr/bin/env python\n"]
-    "# -*- coding: utf-8 -*-\n",
-    "#!/usr/bin/env python\n"
-]
 license_header_lines = [
     "# Licensed to Elasticsearch B.V under one or more agreements.\n",
     "# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.\n",
     "# See the LICENSE file in the project root for more information\n",
-    "\n"
+    "\n",
 ]
 
 
@@ -51,7 +48,9 @@ def does_file_need_fix(filepath: str) -> bool:
                 break
             elif line not in lines_to_keep:
                 return True
-        for header_line, line in zip(license_header_lines, chain((first_license_line,), f)):
+        for header_line, line in zip(
+            license_header_lines, chain((first_license_line,), f)
+        ):
             if line != header_line:
                 return True
     return False
@@ -85,7 +84,10 @@ def main():
         if no_license_headers:
             print("No license header found in:")
             cwd = os.getcwd()
-            [print(f" - {os.path.relpath(filepath, cwd)}") for filepath in no_license_headers]
+            [
+                print(f" - {os.path.relpath(filepath, cwd)}")
+                for filepath in no_license_headers
+            ]
             sys.exit(1)
         else:
             print("All files had license header")