actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[7.x] Switch to Nox, add build_dist script

Browse Source
This commit is contained in:
Seth Michael Larson
2020-06-24 14:25:28 -05:00
parent d73d7a0d8f
commit 4473f9a659
7 changed files with 213 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+4 -4
View File
@@ -35,9 +35,9 @@ jobs:
python-version: 3.7
- name: Install dependencies
run: |
python3.7 -m pip install tox
python3.7 -m pip install nox
- name: Lint the code
run: tox -e lint
run: nox -s lint
docs:
runs-on: ubuntu-latest
@@ -50,9 +50,9 @@ jobs:
python-version: 3.7
- name: Install dependencies
run: |
python3.7 -m pip install tox
python3.7 -m pip install nox
- name: Build the docs
run: tox -e docs
run: nox -s docs
test-linux:
runs-on: ubuntu-latest
CONTRIBUTING.md
+10 -4
View File
@@ -30,7 +30,7 @@ To run the code generation make sure you have pre-requisites installed:
Then you should be able to run the code generation by invoking:
```
python utils/generate_api.py
$ python utils/generate_api.py
```
@@ -45,10 +45,17 @@ The process for contributing to any of the Elasticsearch repositories is similar
assure our users of the origin and continuing existence of the code. You only
need to sign the CLA once.
2. Run the test suite to ensure your changes do not break existing code:
2. Run the linter and test suite to ensure your changes do not break existing code:
````
python setup.py test
# Install Nox for task management
$ python -m pip install nox
# Auto-format and lint your changes
$ nox -s blacken
# Run the test suite
$ python setup.py test
````
See the README file in `test_elasticsearch` directory for more information on
@@ -68,4 +75,3 @@ The process for contributing to any of the Elasticsearch repositories is similar
Then sit back and wait. There will probably be a discussion about the pull
request and, if any changes are needed, we would love to work with you to get
your pull request merged into elasticsearch-py.
noxfile.py
+49
View File
@@ -0,0 +1,49 @@
# Licensed to Elasticsearch B.V under one or more agreements.
# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
# See the LICENSE file in the project root for more information
import nox
SOURCE_FILES = (
"setup.py",
"noxfile.py",
"elasticsearch/",
"test_elasticsearch/",
"utils/",
)
@nox.session(python=["2.7", "3.4", "3.5", "3.6", "3.7", "3.8"])
def test(session):
session.install(".")
session.install("-r", "dev-requirements.txt")
session.run("python", "setup.py", "test")
@nox.session()
def blacken(session):
session.install("black")
session.run("black", "--target-version=py27", *SOURCE_FILES)
session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
lint(session)
@nox.session()
def lint(session):
session.install("flake8", "black")
session.run("black", "--target-version=py27", "--check", *SOURCE_FILES)
session.run("flake8", *SOURCE_FILES)
session.run("python", "utils/license_headers.py", "check", *SOURCE_FILES)
@nox.session()
def docs(session):
session.install(".")
session.install("-rdev-requirements.txt", "sphinx-rtd-theme")
session.run("sphinx-build", "docs/", "docs/_build", "-b", "html")
tox.ini
-52
View File
@@ -1,52 +0,0 @@
[tox]
envlist = pypy,py27,py34,py35,py36,py37,py38,lint,docs
[testenv]
whitelist_externals = git
deps =
-r dev-requirements.txt
commands =
python setup.py test
[testenv:blacken]
deps =
black
commands =
black --target-version=py27 \
elasticsearch/ \
test_elasticsearch/ \
setup.py
python utils/license_headers.py fix \
elasticsearch/ \
test_elasticsearch/ \
utils \
docs \
setup.py
[testenv:lint]
deps =
flake8
black
commands =
black --target-version=py27 --check \
elasticsearch/ \
test_elasticsearch/ \
setup.py
flake8 \
elasticsearch/ \
test_elasticsearch/ \
utils \
docs \
setup.py
python utils/license_headers.py check \
elasticsearch/ \
test_elasticsearch/ \
utils \
docs \
setup.py
[testenv:docs]
deps =
sphinx-rtd-theme
-r dev-requirements.txt
commands =
sphinx-build docs/ docs/_build -b html
utils/build_dists.py
+129
View File
@@ -0,0 +1,129 @@
# Licensed to Elasticsearch B.V under one or more agreements.
# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
# See the LICENSE file in the project root for more information
"""A command line tool for building and verifying releases
Can be used for building both 'elasticsearch' and 'elasticsearchX' dists.
Only requires 'name' in 'setup.py' and the directory to be changed.
"""
import tempfile
import os
import shlex
import re
import contextlib
import shutil
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
tmp_dir = None
@contextlib.contextmanager
def set_tmp_dir():
global tmp_dir
tmp_dir = tempfile.mkdtemp()
yield tmp_dir
shutil.rmtree(tmp_dir)
tmp_dir = None
def run(*argv, expect_exit_code=0):
global tmp_dir
if tmp_dir is None:
os.chdir(base_dir)
else:
os.chdir(tmp_dir)
cmd = " ".join(shlex.quote(x) for x in argv)
print("$ " + cmd)
exit_code = os.system(cmd)
if exit_code != expect_exit_code:
print(
"Command exited incorrectly: should have been %d was %d"
% (expect_exit_code, exit_code)
)
exit(exit_code or 1)
def test_dist(dist):
with set_tmp_dir() as tmp_dir:
dist_name = re.match(r"^(elasticsearch\d*)-", os.path.basename(dist)).group(1)
# Build the venv and install the dist
run("python", "-m", "venv", os.path.join(tmp_dir, "venv"))
venv_python = os.path.join(tmp_dir, "venv/bin/python")
run(venv_python, "-m", "pip", "install", "-U", "pip")
run(venv_python, "-m", "pip", "install", dist)
# Test the sync namespaces
run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
run(
venv_python,
"-c",
f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
)
run(venv_python, "-c", f"from {dist_name} import Elasticsearch")
run(
venv_python,
"-c",
f"from {dist_name}.helpers import scan, bulk, streaming_bulk, reindex",
)
# Ensure that async is not available yet
run(
venv_python,
"-c",
f"from {dist_name} import AsyncElasticsearch",
expect_exit_code=256,
)
run(
venv_python,
"-c",
f"from {dist_name}.helpers import async_scan, async_bulk, async_streaming_bulk, async_reindex",
expect_exit_code=256,
)
# Install aiohttp and see that async is now available
run(venv_python, "-m", "pip", "install", "aiohttp")
run(venv_python, "-c", f"from {dist_name} import AsyncElasticsearch")
# Ensure that the namespaces are correct for the dist
for suffix in ("", "1", "2", "5", "6", "7", "8", "9", "10"):
distx_name = f"elasticsearch{suffix}"
run(
venv_python,
"-c",
f"import {distx_name}",
expect_exit_code=256 if distx_name != dist_name else 0,
)
# Uninstall the dist, see that we can't import things anymore
run(venv_python, "-m", "pip", "uninstall", "--yes", dist_name)
run(
venv_python,
"-c",
f"from {dist_name} import Elasticsearch",
expect_exit_code=256,
)
def main():
run("rm", "-rf", "build/", "dist/", "*.egg-info", ".eggs")
run("python", "setup.py", "sdist", "bdist_wheel")
for dist in os.listdir(os.path.join(base_dir, "dist")):
test_dist(os.path.join(base_dir, "dist", dist))
# After this run 'python -m twine upload dist/*'
print(
"\n\n"
"===============================\n\n"
" * Releases are ready! *\n\n"
"$ python -m twine upload dist/*\n\n"
"==============================="
)
if __name__ == "__main__":
main()
utils/generate_api.py
+12 -8
View File
@@ -96,9 +96,7 @@ class Module:
break
self.header = "\n".join(header_lines)
self.orders = re.findall(
r'\n (?:async )?def ([a-z_]+)\(',
content,
re.MULTILINE
r"\n (?:async )?def ([a-z_]+)\(", content, re.MULTILINE
)
def _position(self, api):
@@ -153,7 +151,11 @@ class API:
# Try setting doc refs like 'current' and 'master' to our branches ref.
if BRANCH_NAME is not None:
revised_url = re.sub("/elasticsearch/reference/[^/]+/", f"/elasticsearch/reference/{BRANCH_NAME}/", self.doc_url)
revised_url = re.sub(
"/elasticsearch/reference/[^/]+/",
f"/elasticsearch/reference/{BRANCH_NAME}/",
self.doc_url,
)
if is_valid_url(revised_url):
self.doc_url = revised_url
else:
@@ -199,9 +201,11 @@ class API:
return chain(
((p, parts[p]) for p in parts if parts[p]["required"]),
(("body", self.body),) if self.body else (),
((p, parts[p]) for p in parts
if not parts[p]["required"]
and p not in params),
(
(p, parts[p])
for p in parts
if not parts[p]["required"] and p not in params
),
sorted(params.items(), key=lambda x: (x[0] not in parts, x[0])),
)
@@ -320,7 +324,7 @@ def dump_modules(modules):
unasync.Rule(
fromdir="/elasticsearch/_async/client/",
todir="/elasticsearch/client/",
additional_replacements=additional_replacements
additional_replacements=additional_replacements,
),
]
utils/license_headers.py
+9 -7
View File
@@ -13,15 +13,12 @@ from typing import List, Iterator
from itertools import chain
lines_to_keep = [
"# -*- coding: utf-8 -*-\n",
"#!/usr/bin/env python\n"
]
lines_to_keep = ["# -*- coding: utf-8 -*-\n", "#!/usr/bin/env python\n"]
license_header_lines = [
"# Licensed to Elasticsearch B.V under one or more agreements.\n",
"# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.\n",
"# See the LICENSE file in the project root for more information\n",
"\n"
"\n",
]
@@ -51,7 +48,9 @@ def does_file_need_fix(filepath: str) -> bool:
break
elif line not in lines_to_keep:
return True
for header_line, line in zip(license_header_lines, chain((first_license_line,), f)):
for header_line, line in zip(
license_header_lines, chain((first_license_line,), f)
):
if line != header_line:
return True
return False
@@ -85,7 +84,10 @@ def main():
if no_license_headers:
print("No license header found in:")
cwd = os.getcwd()
[print(f" - {os.path.relpath(filepath, cwd)}") for filepath in no_license_headers]
[
print(f" - {os.path.relpath(filepath, cwd)}")
for filepath in no_license_headers
]
sys.exit(1)
else:
print("All files had license header")