[7.x] Actually dockerize repo, update build-dists for release manager

.ci/Dockerfile

@@ -1,7 +1,10 @@
 ARG PYTHON_VERSION=3.8
 FROM python:${PYTHON_VERSION}
 
-COPY dev-requirements.txt /tmp
-RUN python -m pip install -r /tmp/dev-requirements.txt
-
 WORKDIR /code/elasticsearch-py
+COPY . .
+RUN python -m pip install \
+    --no-cache-dir \
+    --disable-pip-version-check \
+    -e . \
+    -r dev-requirements.txt

.ci/make.sh

@@ -5,11 +5,12 @@ set -eo pipefail
 BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 BASE_DIR="$( dirname "$BASE_DIR" )"
 
-if [[ "$1" != "release" ]]; then
-  echo "Must be called ./.ci/make.sh release [version]"
-  exit 1
+if [[ "$1" == "release" ]]; then
+  python $BASE_DIR/utils/build-dists.py $2
+  mkdir -p $BASE_DIR/.ci/output
+  cp $BASE_DIR/dist/* $BASE_DIR/.ci/output/
+  exit 0
 fi
 
-python $BASE_DIR/utils/build_dists.py
-mkdir -p $BASE_DIR/.ci/output
-cp $BASE_DIR/dist/* $BASE_DIR/.ci/output/
+echo "Must be called with '.ci/make.sh [command]"
+exit 1

.ci/run-repository.sh

@@ -40,6 +40,5 @@ docker run \
   --env "TEST_TYPE=server" \
   --name elasticsearch-py \
   --rm \
-  --volume `pwd`:/code/elasticsearch-py \
   elastic/elasticsearch-py \
   python setup.py test

.dockerignore

@@ -3,3 +3,5 @@ example
 venv
 .git
 .tox
+.nox
+.*_cache

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
           python3.7 -m pip install setuptools wheel twine
       - name: Build packages
         run: |
-          python3.7 utils/build_dists.py
+          python3.7 utils/build-dists.py
       - name: Check packages
         run: |
           set -exo pipefail;

dev-requirements.txt

@@ -14,6 +14,7 @@ pandas
 pyyaml<5.3
 
 black; python_version>="3.6"
+twine
 
 # Requirements for testing [async] extra
 aiohttp; python_version>="3.6"

elasticsearch/__init__.py

@@ -18,13 +18,13 @@
 # flake8: noqa
 from __future__ import absolute_import
 
-__versionstr__ = "7.10.0+dev"
-
 import re
 import sys
 import logging
 import warnings
 
+from ._version import __versionstr__
+
 _major, _minor, _patch = [
     int(x) for x in re.search(r"^(\d+)\.(\d+)\.(\d+)", __versionstr__).groups()
 ]

elasticsearch/_version.py

@@ -0,0 +1,18 @@
+#  Licensed to Elasticsearch B.V. under one or more contributor
+#  license agreements. See the NOTICE file distributed with
+#  this work for additional information regarding copyright
+#  ownership. Elasticsearch B.V. licenses this file to you under
+#  the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+# 	http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+__versionstr__ = "7.11.0+dev"

noxfile.py

@@ -40,7 +40,7 @@ def blacken(session):
     session.install("black")
 
     session.run("black", "--target-version=py27", *SOURCE_FILES)
-    session.run("python", "utils/license_headers.py", "fix", *SOURCE_FILES)
+    session.run("python", "utils/license-headers.py", "fix", *SOURCE_FILES)
 
     lint(session)
 
@@ -51,7 +51,7 @@ def lint(session):
 
     session.run("black", "--target-version=py27", "--check", *SOURCE_FILES)
     session.run("flake8", *SOURCE_FILES)
-    session.run("python", "utils/license_headers.py", "check", *SOURCE_FILES)
+    session.run("python", "utils/license-headers.py", "check", *SOURCE_FILES)
 
     # Workaround to make '-r' to still work despite uninstalling aiohttp below.
     session.run("python", "-m", "pip", "install", "aiohttp")

setup.py

@@ -23,7 +23,7 @@ from setuptools import setup, find_packages
 package_name = "elasticsearch"
 base_dir = abspath(dirname(__file__))
 
-with open(join(base_dir, package_name, "__init__.py")) as f:
+with open(join(base_dir, package_name, "_version.py")) as f:
     package_version = re.search(
         r"__versionstr__\s+=\s+[\"\']([^\"\']+)[\"\']", f.read()
     ).group(1)

utils/build-dists.py

@@ -23,6 +23,7 @@ Only requires 'name' in 'setup.py' and the directory to be changed.
 import tempfile
 import os
 import shlex
+import sys
 import re
 import contextlib
 import shutil
@@ -163,11 +164,50 @@ def main():
     run("rm", "-rf", "build/", "dist/", "*.egg-info", ".eggs")
 
     # Grab the major version to be used as a suffix.
-    setup_py_path = os.path.join(base_dir, "setup.py")
-    with open(os.path.join(base_dir, "elasticsearch/__init__.py")) as f:
-        major_version = re.search(
-            r"^__versionstr__\s+=\s+[\"\'](\d+)\.", f.read(), re.M
+    version_path = os.path.join(base_dir, "elasticsearch/_version.py")
+    with open(version_path) as f:
+        version = re.search(
+            r"^__versionstr__\s+=\s+[\"\']([^\"\']+)[\"\']", f.read(), re.M
         ).group(1)
+    major_version = version.split(".")[0]
+
+    # If we're handed a version from the build manager we
+    # should check that the version is correct or write
+    # a new one.
+    if len(sys.argv) >= 2:
+        # 'build_version' is what the release manager wants,
+        # 'expect_version' is what we're expecting to compare
+        # the package version to before building the dists.
+        build_version = expect_version = sys.argv[1]
+
+        # '-SNAPSHOT' means we're making a pre-release.
+        if "-SNAPSHOT" in build_version:
+            # If there's no +dev already (as is the case on dev
+            # branches like 7.x, master) then we need to add one.
+            if not version.endswith("+dev"):
+                version = version + "+dev"
+            expect_version = expect_version.replace("-SNAPSHOT", "")
+            if expect_version.endswith(".x"):
+                expect_version = expect_version[:-2]
+
+            # For snapshots we ensure that the version in the package
+            # at least *starts* with the version. This is to support
+            # build_version='7.x-SNAPSHOT'.
+            if not version.startswith(expect_version):
+                print(
+                    "Version of package (%s) didn't match the "
+                    "expected release version (%s)" % (version, build_version)
+                )
+                exit(1)
+
+        # A release that will be tagged, we want
+        # there to be no '+dev', etc.
+        elif expect_version != version:
+            print(
+                "Version of package (%s) didn't match the "
+                "expected release version (%s)" % (version, build_version)
+            )
+            exit(1)
 
     for suffix in ("", major_version):
         run("rm", "-rf", "build/", "*.egg-info", ".eggs")
@@ -178,7 +218,21 @@ def main():
             os.path.join(base_dir, "elasticsearch%s" % suffix),
         )
 
+        # Ensure that the version within 'elasticsearch/_version.py' is correct.
+        version_path = os.path.join(base_dir, f"elasticsearch{suffix}/_version.py")
+        with open(version_path) as f:
+            version_data = f.read()
+        version_data = re.sub(
+            r"__versionstr__ = \"[^\"]+\"",
+            '__versionstr__ = "%s"' % version,
+            version_data,
+        )
+        with open(version_path, "w") as f:
+            f.truncate()
+            f.write(version_data)
+
         # Rewrite setup.py with the new name.
+        setup_py_path = os.path.join(base_dir, "setup.py")
         with open(setup_py_path) as f:
             setup_py = f.read()
         with open(setup_py_path, "w") as f:
@@ -200,7 +254,9 @@ def main():
             run("rm", "-rf", "elasticsearch%s/" % suffix)
 
     # Test everything that got created
-    for dist in os.listdir(os.path.join(base_dir, "dist")):
+    dists = os.listdir(os.path.join(base_dir, "dist"))
+    assert len(dists) == 4
+    for dist in dists:
         test_dist(os.path.join(base_dir, "dist", dist))
 
     # After this run 'python -m twine upload dist/*'