actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[7.x] [CI] Update integration test suite with new Docker distribution format

Browse Source
This commit is contained in:
Seth Michael Larson
2021-01-29 15:22:58 -06:00
committed by GitHub
parent da6629d1a4
commit 9744eae1b6
13 changed files with 285 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/functions/cleanup.sh
Executable
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
#
# Shared cleanup routines between different steps
#
# Please source .ci/functions/imports.sh as a whole not just this file
#
# Version 1.0.0
# - Initial version after refactor
function cleanup_volume {
if [[ "$(docker volume ls -q -f name=$1)" ]]; then
echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
(docker volume rm "$1") || true
fi
}
function container_running {
if [[ "$(docker ps -q -f name=$1)" ]]; then
return 0;
else return 1;
fi
}
function cleanup_node {
if container_running "$1"; then
echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
(docker container rm --force --volumes "$1") || true
fi
if [[ -n "$1" ]]; then
echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
cleanup_volume "$1-${suffix}-data"
fi
}
function cleanup_network {
if [[ "$(docker network ls -q -f name=$1)" ]]; then
echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
(docker network rm "$1") || true
fi
}
function cleanup_trap {
status=$?
set +x
if [[ "$DETACH" != "true" ]]; then
echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
cleanup_all_in_network "$1"
fi
# status is 0 or SIGINT
if [[ "$status" == "0" || "$status" == "130" ]]; then
echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
exit 0
else
echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
exit ${status}
fi
};
function cleanup_all_in_network {
if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
return 0
fi
containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
while read -r container; do
cleanup_node "$container"
done <<< "$containers"
cleanup_network $1
echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
};
.ci/functions/imports.sh
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
#
# Sets up all the common variables and imports relevant functions
#
# Version 1.0.1
# - Initial version after refactor
# - Validate STACK_VERSION asap
function require_stack_version() {
if [[ -z $STACK_VERSION ]]; then
echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
exit 1
fi
}
require_stack_version
if [[ -z $es_node_name ]]; then
# only set these once
set -euo pipefail
export TEST_SUITE=${TEST_SUITE-free}
export RUNSCRIPTS=${RUNSCRIPTS-}
export DETACH=${DETACH-false}
export CLEANUP=${CLEANUP-false}
export es_node_name=instance
export elastic_password=changeme
export elasticsearch_image=elasticsearch
export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
if [[ $TEST_SUITE != "platinum" ]]; then
export elasticsearch_url=http://${es_node_name}:9200
fi
export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
export suffix=rest-test
export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
export network_name=${moniker}${suffix}
export ssl_cert="${script_path}/certs/testnode.crt"
export ssl_key="${script_path}/certs/testnode.key"
export ssl_ca="${script_path}/certs/ca.crt"
fi
export script_path=$(dirname $(realpath -s $0))
source $script_path/functions/cleanup.sh
source $script_path/functions/wait-for-container.sh
trap "cleanup_trap ${network_name}" EXIT
if [[ "$CLEANUP" == "true" ]]; then
cleanup_all_in_network $network_name
exit 0
fi
echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
.ci/functions/wait-for-container.sh
Executable
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
#
# Exposes a routine scripts can call to wait for a container if that container set up a health command
#
# Please source .ci/functions/imports.sh as a whole not just this file
#
# Version 1.0.1
# - Initial version after refactor
# - Make sure wait_for_contiainer is silent
function wait_for_container {
set +x
until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
echo ""
docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
sleep 2;
done;
# Always show logs if the container is running, this is very useful both on CI as well as while developing
if container_running $1; then
docker logs $1
fi
if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
cleanup_all_in_network $2
echo
echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
return 1
else
echo
echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
return 0
fi
}
.ci/jobs/defaults.yml
+1 -1
View File
@@ -40,7 +40,7 @@
- axis:
type: yaml
filename: .ci/test-matrix.yml
name: ELASTICSEARCH_VERSION
name: STACK_VERSION
- axis:
type: yaml
filename: .ci/test-matrix.yml
.ci/run-elasticsearch.sh
+71 -159
View File
@@ -3,110 +3,35 @@
# Launch one or more Elasticsearch nodes via the Docker image,
# to form a cluster suitable for running the REST API tests.
#
# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'.
# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
# Export the NUMBER_OF_NODES variable to start more than 1 node
# Version 1.0.2
# Version 1.2.0
# - Initial version of the run-elasticsearch.sh script
# - Deleting the volume should not dependent on the container still running
# - Fixed `ES_JAVA_OPTS` config
# - Moved to STACK_VERSION and TEST_VERSION
# - Refactored into functions and imports
# - Support NUMBER_OF_NODES
# - Added 5 retries on docker pull for fixing transient network errors
if [[ -z "$ELASTICSEARCH_VERSION" ]]; then
echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
exit 1
fi
script_path=$(dirname $(realpath -s $0))
source $script_path/functions/imports.sh
set -euo pipefail
set -euxo pipefail
echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
cleanup_node $es_node_name
SCRIPT_PATH=$(dirname $(realpath -s $0))
moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-')
suffix=rest-test
NODE_NAME=${NODE_NAME-${moniker}node1}
MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}}
CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}}
HTTP_PORT=${HTTP_PORT-9200}
ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme}
SSL_CERT=${SSL_CERT-"${SCRIPT_PATH}/certs/testnode.crt"}
SSL_KEY=${SSL_KEY-"${SCRIPT_PATH}/certs/testnode.key"}
SSL_CA=${SSL_CA-"${SCRIPT_PATH}/certs/ca.crt"}
SSL_CA_PEM=${SSL_CA-"${SCRIPT_PATH}/certs/ca.pem"}
DETACH=${DETACH-false}
CLEANUP=${CLEANUP-false}
volume_name=${NODE_NAME}-${suffix}-data
network_default=${moniker}${suffix}
NETWORK_NAME=${NETWORK_NAME-"$network_default"}
set +x
function cleanup_volume {
if [[ "$(docker volume ls -q -f name=$1)" ]]; then
echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
(docker volume rm "$1") || true
fi
}
function container_running {
if [[ "$(docker ps -q -f name=$1)" ]]; then
return 0;
else return 1;
fi
}
function cleanup_node {
if container_running "$1"; then
echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
(docker container rm --force --volumes "$1") || true
fi
echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
cleanup_volume "$1-${suffix}-data"
}
function cleanup_network {
if [[ "$(docker network ls -q -f name=$1)" ]]; then
echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
(docker network rm "$1") || true
fi
}
function cleanup {
if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then
echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m"
cleanup_node "$NODE_NAME"
fi
if [[ "$DETACH" != "true" ]]; then
echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
cleanup_network "$NETWORK_NAME"
fi
};
trap "cleanup 0" EXIT
if [[ "$CLEANUP" == "true" ]]; then
trap - EXIT
if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then
echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m"
exit 0
fi
containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME})
while read -r container; do
cleanup_node "$container"
done <<< "$containers"
cleanup_network "$NETWORK_NAME"
echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
exit 0
fi
echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m"
cleanup 1
echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m"
docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME"
master_node_name=${es_node_name}
cluster_name=${moniker}${suffix}
declare -a volumes
environment=($(cat <<-END
--env node.name=$NODE_NAME
--env cluster.name=$CLUSTER_NAME
--env cluster.initial_master_nodes=$MASTER_NODE_NAME
--env discovery.seed_hosts=$MASTER_NODE_NAME
--env node.name=$es_node_name
--env cluster.name=$cluster_name
--env cluster.initial_master_nodes=$master_node_name
--env discovery.seed_hosts=$master_node_name
--env cluster.routing.allocation.disk.threshold_enabled=false
--env bootstrap.memory_lock=true
--env node.attr.testattr=test
@@ -114,15 +39,9 @@ environment=($(cat <<-END
--env repositories.url.allowed_urls=http://snapshot.test*
END
))
volumes=($(cat <<-END
--volume $volume_name:/usr/share/elasticsearch/data
END
))
if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
if [[ "$TEST_SUITE" == "platinum" ]]; then
environment+=($(cat <<-END
--env ELASTIC_PASSWORD=$ELASTIC_PASSWORD
--env ELASTIC_PASSWORD=$elastic_password
--env xpack.license.self_generated.type=trial
--env xpack.security.enabled=true
--env xpack.security.http.ssl.enabled=true
@@ -137,78 +56,71 @@ if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
END
))
volumes+=($(cat <<-END
--volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt
--volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key
--volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt
--volume $SSL_CA_PEM:/usr/share/elasticsearch/config/certs/ca.pem
--volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt
--volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key
--volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt
END
))
fi
url="http://$NODE_NAME"
if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then
url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME"
cert_validation_flags=""
if [[ "$TEST_SUITE" == "platinum" ]]; then
cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
fi
cert_validation_flags="--insecure"
if [[ "$NODE_NAME" == "instance" ]]; then
cert_validation_flags="--cacert /usr/share/elasticsearch/config/certs/ca.pem --resolve ${NODE_NAME}:443:127.0.0.1"
fi
echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m"
# Pull the container, retry on failures up to 5 times with
# short delays between each attempt. Fixes most transient network errors.
docker_pull_attempts=0
until [ "$docker_pull_attempts" -ge 5 ]
do
docker pull docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION" && break
docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
docker_pull_attempts=$((docker_pull_attempts+1))
echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
sleep 10
done
set -x
docker run \
--name "$NODE_NAME" \
--network "$NETWORK_NAME" \
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
"${environment[@]}" \
"${volumes[@]}" \
--publish "$HTTP_PORT":9200 \
--ulimit nofile=65536:65536 \
--ulimit memlock=-1:-1 \
--detach="$DETACH" \
--health-cmd="curl $cert_validation_flags --fail $url:9200/_cluster/health || exit 1" \
--health-interval=2s \
--health-retries=20 \
--health-timeout=2s \
--rm \
docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION";
set +x
NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
http_port=9200
for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
node_name=${es_node_name}$i
node_url=${external_elasticsearch_url/9200/${http_port}}$i
if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi
environment+=($(cat <<-END
--env node.name=$node_name
END
))
echo "$i: $http_port $node_url "
volume_name=${node_name}-${suffix}-data
volumes+=($(cat <<-END
--volume $volume_name:/usr/share/elasticsearch/data${i}
END
))
if [[ "$DETACH" == "true" ]]; then
until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do
echo ""
docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME}
echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m"
sleep 2;
done;
# make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
local_detach="true"
if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
set -x
docker run \
--name "$node_name" \
--network "$network_name" \
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
"${environment[@]}" \
"${volumes[@]}" \
--publish "$http_port":9200 \
--ulimit nofile=65536:65536 \
--ulimit memlock=-1:-1 \
--detach="$local_detach" \
--health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \
--health-interval=2s \
--health-retries=20 \
--health-timeout=2s \
--rm \
docker.elastic.co/elasticsearch/"$elasticsearch_container";
# Always show logs if the container is running, this is very useful both on CI as well as while developing
if container_running $NODE_NAME; then
docker logs $NODE_NAME
set +x
if wait_for_container "$es_node_name" "$network_name"; then
echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m"
fi
if ! container_running $NODE_NAME || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then
cleanup 1
echo
echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m"
echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
exit 1
else
echo
echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m"
echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m"
exit 0
fi
fi
done
.ci/run-repository.sh
+2 -2
View File
@@ -33,8 +33,8 @@ echo -e "\033[1m>>>>> Run [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>
mkdir -p junit
docker run \
--network=${NETWORK_NAME} \
--env "ELASTICSEARCH_HOST=${ELASTICSEARCH_URL}" \
--network=${network_name} \
--env "ELASTICSEARCH_URL=${elasticsearch_url}" \
--env "TEST_SUITE=${TEST_SUITE}" \
--env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \
--env "TEST_TYPE=server" \
.ci/run-tests
+25 -53
View File
@@ -1,58 +1,30 @@
#!/usr/bin/env bash
#
# Version 1.1
# - Moved to .ci folder and seperated out `run-repository.sh`
# - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
if [[ -z $ELASTICSEARCH_VERSION ]]; then
echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
exit 1
# Default environment variables
export STACK_VERSION="${STACK_VERSION:=8.0.0-SNAPSHOT}"
export TEST_SUITE="${TEST_SUITE:=free}"
export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
script_path=$(dirname $(realpath -s $0))
source $script_path/functions/imports.sh
set -euo pipefail
echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
DETACH=true bash .ci/run-elasticsearch.sh
if [[ -n "$RUNSCRIPTS" ]]; then
for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
CONTAINER_NAME=${RUNSCRIPT} \
DETACH=true \
bash .ci/run-${RUNSCRIPT}.sh
done
fi
set -euxo pipefail
PYTHON_VERSION=${PYTHON_VERSION-3.8}
PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS-Urllib3HttpConnection}
TEST_SUITE=${TEST_SUITE-oss}
NODE_NAME=instance
elasticsearch_image=elasticsearch
elasticsearch_url=https://elastic:changeme@${NODE_NAME}:9200
if [[ $TEST_SUITE != "xpack" ]]; then
elasticsearch_image=elasticsearch-${TEST_SUITE}
elasticsearch_url=http://${NODE_NAME}:9200
fi
function cleanup {
status=$?
set +x
ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
NODE_NAME=${NODE_NAME} \
NETWORK_NAME=elasticsearch \
CLEANUP=true \
bash ./.ci/run-elasticsearch.sh
# Report status and exit
if [[ "$status" == "0" ]]; then
echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
exit 0
else
echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
exit ${status}
fi
}
trap cleanup EXIT
echo -e "\033[1m>>>>> Start [$ELASTICSEARCH_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
NODE_NAME=${NODE_NAME} \
NETWORK_NAME=elasticsearch \
DETACH=true \
bash .ci/run-elasticsearch.sh
echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
NETWORK_NAME=elasticsearch \
NODE_NAME=${NODE_NAME} \
ELASTICSEARCH_URL=${elasticsearch_url} \
PYTHON_VERSION=${PYTHON_VERSION} \
PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS} \
TEST_SUITE=${TEST_SUITE} \
bash .ci/run-repository.sh
bash .ci/run-repository.sh
.ci/test-matrix.yml
+3 -3
View File
@@ -1,9 +1,9 @@
ELASTICSEARCH_VERSION:
STACK_VERSION:
- 7.x-SNAPSHOT
TEST_SUITE:
- oss
- xpack
- free
- platinum
PYTHON_VERSION:
- 2.7
CONTRIBUTING.md
+18
View File
@@ -10,6 +10,24 @@ We enjoy working with contributors to get their code accepted. There are many
approaches to fixing a problem and it is important to find the best approach
before writing too much code.
## Running Integration Tests
Integration tests are run against a live Elasticsearch instance in Docker.
Run the full integration test suite via `$ .ci/run-tests`.
There are several environment variables that control integration tests:
- `PYTHON_VERSION`: Version of Python to use, defaults to `3.9`
- `PYTHON_CONNECTION_CLASS`: Connection class to use, defaults to `Urllib3HttpConnection`
- `STACK_VERSION`: Version of Elasticsearch to use. These should be
the same as tags of `docker.elastic.co/elasticsearch/elasticsearch`
such as `8.0.0-SNAPSHOT`, `7.x-SNAPSHOT`, etc. Defaults to the
same `*-SNAPSHOT` version as the branch.
- `TEST_SUITE`: Determines how to configure Elasticsearch either by running
without any non-free features or by beginning a Platinum license. Possible options
are `free` and `platinum`. Defaults to `free` as there are fewer test cases.
## API Code Generation
All the API methods (any method in `elasticsearch.client` classes decorated
elasticsearch/helpers/test.py
+1 -1
View File
@@ -37,7 +37,7 @@ def get_test_client(nowait=False, **kwargs):
)
kw.update(kwargs)
client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw)
client = Elasticsearch(os.environ.get("ELASTICSEARCH_URL", {}), **kw)
# wait for yellow status
for _ in range(1 if nowait else 100):
test_elasticsearch/README.rst
-81
View File
@@ -1,81 +0,0 @@
elasticsearch-py test suite
===========================
Warning - by default the tests will try and connect to ``localhost:9200`` and
will destroy all contents of given cluster! The tests also rely on a checkout
of ``elasticsearch`` repository existing on the same level as the
``elasticsearch-py`` clone. Before running the tests we will, by default, pull
latest changes for that repo and perform ``git reset --hard`` to the exact
version that was used to build the server we are running against.
Running the tests
-----------------
To simply run the tests just execute ``python setup.py test``.
The behavior is driven by environmental variables:
* ``ELASTICSEARCH_HOST`` - can contain "hostname[:port]" of running es cluster
* ``PYTHON_CLASS_CONNECTION`` - name of the connection class to use from
``elasticsearch.connection`` module. If you want to run completely with your
own see section on customizing tests.
* ``TEST_ES_YAML_DIR`` - path to the yaml test suite contained in the
elasticsearch repo. Defaults to ``$TEST_ES_REPO/rest-api-spec/test``
* ``TEST_ES_REPO`` - path to the elasticsearch repo, by default it will look in
the same directory as ``elasticsearch-py`` is in. It will not be used if
``TEST_ES_YAML_DIR`` is specified directly.
* ``TEST_ES_NOFETCH`` - controls if we should fetch new updates to elasticsearch
repo and reset it's version to the sha used to build the current es server.
Defaults to ``False`` which means we will fetch the elasticsearch repo and
``git reset --hard`` the sha used to build the server.
Alternatively, if you wish to control what you are doing you have several additional options:
* ``run_tests.py`` will pass any parameters specified to ``pytest``
* to run a specific test, you can use ``python3 setup.py test -s <test_name>``, for example
``python3 setup.py test -s test_elasticsearch.test_helpers.TestParallelBulk.test_all_chunks_sent``
To install all test dependencies you can also run ``pip install -e .[develop]``.
Run Elasticsearch in a Container
--------------------------------
To run elasticsearch in a container set the ``ELASTICSEARCH_VERSION``
environment variable to 7.5, 7.5-SNAPSHOT. Then run ./.ci/run_elasticsearch.sh::
# Run OSS Elasticsearch v7.5
export ELASTICSEARCH_VERSION=elasticsearch-oss:7.5-SNAPSHOT
./.ci/run_elasticsearch.sh
# Run XPACK Elasticsearch v7.5
export ELASTICSEARCH_VERSION=elasticsearch:7.5-SNAPSHOT
./.ci/run_elasticsearch.sh
This will run a version for Elasticsearch in a Docker container suitable for
running the tests. To check that elasticsearch is running first wait for a
``healthy`` status in ``docker ps``:
.. code-block:: console
$ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
955e57564e53 7d2ad83f8446 "/docker-entrypoin..." 6 minutes ago Up 6 minutes (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp trusting_brattain
Then you can navigate to ``locahost:9200`` in your browser.
Customizing the tests
---------------------
You can create a `local.py` file in the `test_elasticsearch` directory which
should contain a `get_client` function.
If this file exists the function will be used instead of
`elasticsearch.helpers.test.get_test_client` to construct the client used for
any integration tests. You can use this to make sure your plugins and
extensions work with `elasticsearch-py`.
test_elasticsearch/test_server/conftest.py
+1 -1
View File
@@ -39,7 +39,7 @@ def sync_client():
}
client = elasticsearch.Elasticsearch(
[os.environ.get("ELASTICSEARCH_HOST", {})], **kw
[os.environ.get("ELASTICSEARCH_URL", {})], **kw
)
# wait for yellow status
test_elasticsearch/utils.py
+1 -1
View File
@@ -35,7 +35,7 @@ def wipe_cluster(client):
except ImportError:
pass
is_xpack = os.getenv("TEST_SUITE") == "xpack"
is_xpack = os.getenv("TEST_SUITE") == "platinum"
if is_xpack:
wipe_rollup_jobs(client)
wait_for_pending_tasks(client, filter="xpack/rollup/job")