Major test refactor.

- removed the git submodule, relying on the elasticsearch repo existing
  outside of this one
- modified the run_tests.py script to inspect currently running
  elasticsearch and try and checkout the exact SHA used to build the
  server
- removed the option of starting out own elasticsearch server
- fixed travis.yml to reflect the changed situation
- added a README file to the test suite explaining all the options

.travis.yml

@@ -21,18 +21,19 @@ matrix:
       env: TEST_ES_CONNECTION=ThriftConnection
 
 install:
-  - wget -O - http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/master/nightly/JDK6/elasticsearch-latest-SNAPSHOT.tar.gz | tar xz -C /tmp
+  - mkdir /tmp/elasticsearch
-  - /tmp/elasticsearch-1.0.0.RC1-SNAPSHOT/bin/plugin -install elasticsearch/elasticsearch-transport-memcached/1.6.0
+  - wget -O - http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/master/nightly/JDK6/elasticsearch-latest-SNAPSHOT.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
-  - /tmp/elasticsearch-1.0.0.RC1-SNAPSHOT/bin/plugin -install elasticsearch/elasticsearch-transport-thrift/1.6.0
+  - /tmp/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-transport-memcached/2.0.0.RC1
-  - git submodule init
+  - /tmp/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-transport-thrift/2.0.0.RC1
+  - git clone https://github.com/elasticsearch/elasticsearch.git ../elasticsearch
   - pip install coveralls
   - pip install .
 
 before_script:
-  - /tmp/elasticsearch-1.0.0.RC1-SNAPSHOT/bin/elasticsearch -d -D es.path.data=/tmp -D es.gateway.type=none -D es.index.store.type=memory -D es.discovery.zen.ping.multicast.enabled=false
+  - /tmp/elasticsearch/bin/elasticsearch -d -D es.path.data=/tmp -D es.gateway.type=none -D es.index.store.type=memory -D es.discovery.zen.ping.multicast.enabled=false
 
 script:
-  - TEST_ES_SERVER=localhost python setup.py test
+  - python setup.py test
 
 after_success:
   - coveralls

CONTRIBUTING.md

@@ -20,16 +20,11 @@ The process for contributing to any of the Elasticsearch repositories is similar
 2. Run the test suite to ensure your changes do not break existing code:
 
     ````
-    git submodules init
     python setup.py test
     ````
 
-   If your changes require special configuration of the client you can create a
+   See the README file in `test_elasticsearch` dirctory for more information on
-   `test_elasticsearch/local.py` file containing a `get_client` with the same
+   running the test suite.
-   signature as the client itself (`hosts, **kwargs` will do) and returns a client
-   instance. This function will be used for any tests running against ES and will
-   be passed all the parameters that would otherwise be used to construct the
-   client instance directly.
 
 3. Rebase your changes.
    Update your local repository with the most recent code from the main

test_elasticsearch/README.rst

@@ -0,0 +1,50 @@
+elasticsearch-py test suite
+===========================
+
+Warning - by default the tests will try and connect to `localhost:9200` and
+will destroy all contents of given cluster!
+
+Running the tests
+-----------------
+
+To simply run the tests just execute the `run_tests.py` script or invoke
+`python setup.py test`. The behavior is driven by environmental variables:
+
+ * `TEST_ES_SERVER` - can contain "hostname[:port]" of running es cluster
+
+ * `TEST_ES_CONNECTION` - name of the connection class to use from
+    `elasticsearch.connection` module. If you want to run completely with your
+    own see section on customizing tests.
+
+ * `TEST_ES_YAML_DIR` - path to the yaml test suite contained in the
+    elasticsearch repo. Defaults to `$TEST_ES_REPO/rest-api-spec/test`
+
+ * `TEST_ES_REPO` - path to the elasticsearch repo, by default it will look in
+    the same directory as `elasticsearch-py` is in. It will not be used if
+    `TEST_ES_YAML_DIR` is specified directly.
+ 
+ * `TEST_ES_NOFETCH` - controls if we should fetch new updates to elasticsearch
+   repo and reset it's version to the sha used to build the current es server.
+   Defaults to `False` which means we will fetch the elasticsearch repo and
+   `git checkout` the sha used to build the server.
+
+Alternatively, if you wish to control what you are doing you have several additional options:
+
+ * `run_tests.py` will pass any parameters specified to `nosetests`
+
+ * you can just run your favorite runner in the `test_elasticsearch` directory
+   (verified to work with nose and py.test) and bypass the fetch logic entirely.
+
+Customizing the tests
+---------------------
+
+You can create a `local.py` file in the `test_elasticsearch` directory which
+should contain a `get_client` function:
+
+    def get_client(hosts, ** kwargs):
+        ...
+
+If this file exists the function will be used instead of the built in one to
+construct the client used for any integration tests. You can use this to make
+sure your plugins and extensions work with `elasticsearch-py`.
+

test_elasticsearch/run_tests.py

@@ -1,11 +1,60 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import sys
-from os import path
+from os import environ
+from os.path import dirname, join, pardir, abspath, exists
+import subprocess
 
 import nose
 
+from test_elasticsearch.test_server import get_client
+from test_elasticsearch.test_cases import SkipTest
+
+def fetch_es_repo():
+    # user is manually setting YAML dir, don't tamper with it
+    if 'TEST_ES_YAML_DIR' in environ:
+        return
+
+    repo_path = environ.get(
+        'TEST_ES_REPO',
+        abspath(join(dirname(__file__), pardir, pardir, 'elasticsearch'))
+    )
+
+    # no repo
+    if not exists(repo_path) or not exists(join(repo_path, '.git')):
+        print('No elasticsearch repo found...')
+        # set YAML DIR to empty to skip yaml tests
+        environ['TEST_ES_YAML_DIR'] = ''
+        return
+
+    # set YAML test dir
+    environ['TEST_ES_YAML_DIR'] = join(repo_path, 'rest-api-spec', 'test')
+
+    # fetching of yaml tests disabled, we'll run with what's there
+    if environ.get('TEST_ES_NOFETCH', False):
+        return
+
+    # find out the sha of the running es
+    try:
+        es = get_client()
+        sha = es.info()['version']['build_hash']
+    except (SkipTest, KeyError):
+        print('No running elasticsearch >1.X server...')
+        return
+
+    # fetch new commits to be sure...
+    print('Fetching elasticsearch repo...')
+    subprocess.check_call('cd %s && git fetch https://github.com/elasticsearch/elasticsearch.git' % repo_path, shell=True)
+    # reset to the version fron info()
+    subprocess.check_call('cd %s && git checkout %s' % (repo_path, sha), shell=True)
+
 def run_all(argv=None):
     sys.exitfunc = lambda: sys.stderr.write('Shutting down....\n')
+
+    # fetch yaml tests
+    fetch_es_repo()
+
     # always insert coverage when running tests
     if argv is None:
         argv = [
@@ -21,7 +70,7 @@ def run_all(argv=None):
 
     nose.run_exit(
         argv=argv,
-        defaultTest=path.abspath(path.dirname(__file__))
+        defaultTest=abspath(dirname(__file__))
     )
 
 if __name__ == '__main__':

test_elasticsearch/test_server/__init__.py

@@ -1,104 +1,46 @@
 import time
-import subprocess
-import tempfile
 import os
 
-import requests
-
 from elasticsearch import Elasticsearch
 from elasticsearch.exceptions import ConnectionError, NotFoundError
 
 from ..test_cases import TestCase, SkipTest
 
-data_dir = None
+client = None
 
-CMD = """
+def get_client():
-    elasticsearch \
+    global client
-    -f \
+    if client is not None:
-    -D es.cluster.name=%(cluster_name)s \
+        return client
-    -D es.node.name=test_name \
-    -D es.http.port=%(port)s \
-    -D es.gateway.type=none \
-    -D es.index.store.type=memory \
-    -D es.discovery.zen.ping.multicast.enabled=false \
-    -D es.path.data=%(data_dir)s \
-    -D es.pidfile=%(pidfile)s \
-    >/dev/null 2>&1
-"""
 
-server = None
-pidfile = tempfile.mktemp()
-
-def get_client(**kwargs):
     # construct kwargs from the environment
     kw = {}
     if 'TEST_ES_CONNECTION' in os.environ:
         from elasticsearch import connection
         kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION'])
-    # update them with params
-    kw.update(kwargs)
 
     # try and locate manual override in the local environment
     try:
         from test_elasticsearch.local import get_client as local_get_client
-        return local_get_client([os.environ['TEST_ES_SERVER']], **kw)
+        client = local_get_client([os.environ.get('TEST_ES_SERVER', {})], **kw)
     except ImportError:
         # fallback to using vanilla client
-        return Elasticsearch([os.environ['TEST_ES_SERVER']], **kw)
+        client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw)
-
-
-def setup():
-    global server
-
-    # if use running ES instance, don't attempt to start our own
-    if 'TEST_ES_SERVER' not in os.environ:
-        # check installed
-        if subprocess.call('which elasticsearch >/dev/null 2>&1', shell=True) != 0:
-            raise SkipTest("No Elasticsearch server, skipping integration tests.")
-
-        args = {
-            'cluster_name': 'es_client_test',
-            'port': 9900,
-            'data_dir': tempfile.tempdir,
-            'pidfile': pidfile
-        }
-
-        # check running
-        try:
-            requests.get('http://localhost:%(port)s' % args)
-        except requests.ConnectionError:
-            pass
-        else:
-            raise SkipTest('Elasticsearch already running!')
-
-
-        cmd = CMD % args
-
-        server = subprocess.Popen(cmd, shell=True)
-        os.environ['TEST_ES_SERVER'] = 'localhost:%(port)s' % args
-
-    client = get_client()
 
     # wait for yellow status
     for _ in range(100):
         time.sleep(.1)
         try:
             client.cluster.health(wait_for_status='yellow')
-            break
+            return client
         except ConnectionError:
             continue
-
     else:
         # timeout
         raise SkipTest("Elasticsearch failed to start.")
 
-
+def setup():
-def teardown():
+    get_client()
-    if server is not None:
-        with open(pidfile) as pidf:
-            pid = pidf.read()
-            os.kill(int(pid), 15)
-        server.wait()
 
 ES_VERSION = None
 

test_elasticsearch/test_server/test_common.py

@@ -195,10 +195,10 @@ def construct_case(filename, name):
     return type(name, (YamlTestCase, ), attrs)
 
 YAML_DIR = environ.get(
-    'YAML_TEST_DIR',
+    'TEST_ES_YAML_DIR',
     join(
-        dirname(__file__), pardir, pardir,
+        dirname(__file__), pardir, pardir, pardir,
-        'rest-api-spec', 'rest-api-spec', 'test'
+        'elasticsearch', 'rest-api-spec', 'test'
     )
 )