In preparation for moving to Elastic's CI (#862)

* In preperation for moving to Elastic's CI

Add dockerfile to build image for testing python client in a docker container
add docker-compose to create environment for running tests against specific versions of ES
Add Makefile to make it easy to execute tests and cleanup
add wait-for-elasticsearch.sh script to wait for elasticsearch before executing the tests.

* add pull and comment about pushing requring auth

* spelling errors

* adding CI directory for jenkins to pick up

fixing typo too

* fix test-matrix and create working run-tests

* add comment about why skipping python 2.6

* adding dockerfiles to create base images for ci

.ci/Dockerfile-py

@@ -0,0 +1,2 @@
+ARG PYTHON_VERSION=3
+FROM python:${PYTHON_VERSION}

.ci/Dockerfile-py26

@@ -0,0 +1,10 @@
+FROM ubuntu:18.04
+RUN apt-get update && apt-get install -y curl build-essential zlib1g-dev libssl1.0-dev
+RUN ln -s /lib/i386-linux-gnu/libz.so.1 /lib/libz.so
+RUN curl -s https://www.python.org/ftp/python/2.6.9/Python-2.6.9.tgz | tar -xzf -
+WORKDIR Python-2.6.9
+RUN LDFLAGS="-L/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)" ./configure
+RUN make
+RUN make install
+RUN curl https://bootstrap.pypa.io/2.6/get-pip.py -o get-pip.py
+RUN python get-pip.py

.ci/run-tests

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#
+# Runs the client tests via Docker with the expectation that the required
+# environment variables have already been exported before running this script.
+#
+# The required environment variables include:
+#
+#   - $ELASTICSEARCH_VERSION
+#   - $PYTHON_VERSION
+#
+
+# TODO: implement the docker-based testing
+cd ..
+make run_tests

.ci/test-matrix.yml

@@ -0,0 +1,18 @@
+---
+ELASTICSEARCH_VERSION:
+  - 6.4.2
+  - 6.3.2
+  - 6.1.4
+  - 6.0.1
+
+
+
+PYTHON_VERSION:
+  - 3
+  - 2.7
+  - 2.6
+
+exclude:
+  # since a base image for python2.6 doens't exist i need to create one.
+  - PYTHON_VERSION: 2.6
+

Dockerfile

@@ -0,0 +1,12 @@
+ARG PYTHON_VERSION=3
+FROM python:${PYTHON_VERSION}
+RUN apt-get update && apt-get install -y git curl
+RUN pip install ipdb python-dateutil GitPython
+
+RUN git clone https://github.com/elastic/elasticsearch.git /code/elasticsearch
+
+WORKDIR /code/elasticsearch-py
+COPY . .
+RUN pip install .[develop]
+RUN python setup.py develop
+CMD ["/code/wait-for-elasticsearch.sh", "http://elasticsearch:9200", "--", "python", "setup.py", "test"]

Makefile

@@ -0,0 +1,18 @@
+clean:
+	docker-compose down --remove-orphans --volumes
+
+build:
+	PYTHON_VERSION=${PYTHON_VERSION} docker-compose build client
+
+pull:
+	ELASTIC_VERSION=${ELASTIC_VERSION} PYTHON_VERSION=${PYTHON_VERSION} docker-compose pull
+
+push:
+	# requires authentication.
+	PYTHON_VERSION=${PYTHON_VERSION} docker-compose push client
+
+run_tests:
+	ELASTIC_VERSION=${ELASTIC_VERSION} PYTHON_VERSION=${PYTHON_VERSION} docker-compose run client python setup.py test
+
+start_elasticsearch:
+	ELASTIC_VERSION=${ELASTIC_VERSION} docker-compose up -d elasticsearch

docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.3'
+services:
+  client:
+    image: docker.elastic.co/clients/elasticsearch-py:${PYTHON_VERSION:-3}
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+          PYTHON_VERSION: ${PYTHON_VERSION:-3}
+    environment:
+      - "TEST_ES_SERVER=http://elasticsearch:9200"
+    volumes:
+      - .:/code/elasticsearch-py
+      - esvol:/tmp
+    networks:
+      - esnet
+    depends_on:
+      - elasticsearch
+    command: ["true"]  # dummy command to override the command in the Dockerfile and do nothing.
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:${ELASTIC_VERSION:-6.2.4}
+    volumes:
+      - esvol:/tmp
+    networks:
+      - esnet
+    environment:
+      - path.repo=/tmp
+      - "repositories.url.allowed_urls=http://*"
+      - node.attr.testattr=test
+      - bootstrap.memory_lock=false
+      - "discovery.zen.ping.unicast.hosts=elasticsearch"
+      - "http.max_content_length=5mb"
+networks:
+  esnet:
+volumes:
+  esvol:

wait-for-elasticsearch.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+host="$1"
+shift
+cmd="$@"
+
+
+until $(curl --output /dev/null --silent --head --fail "$host"); do
+    printf '.'
+    sleep 1
+done
+
+# First wait for ES to start...
+response=$(curl $host)
+
+until [ "$response" = "200" ]; do
+    response=$(curl --write-out %{http_code} --silent --output /dev/null "$host")
+    >&2 echo "Elasticsearch is unavailable - sleeping"
+    sleep 1
+done
+
+
+# next wait for ES status to turn to Green
+health="$(curl -fsSL "$host/_cat/health?h=status")"
+health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
+
+until [ "$health" = 'green' ]; do
+    health="$(curl -fsSL "$host/_cat/health?h=status")"
+    health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ")
+    >&2 echo "Elasticsearch is unavailable - sleeping"
+    sleep 1
+done
+
+>&2 echo "Elasticsearch is up"
+exec $cmd