diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 00000000..bcff333f --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,9 @@ +ARG PYTHON_VERSION=3.8 +FROM python:${PYTHON_VERSION} + +WORKDIR /code/elasticsearch-py + +COPY dev-requirements.txt . +RUN python -m pip install -r dev-requirements.txt + +COPY . . diff --git a/.ci/certs/ca.crt b/.ci/certs/ca.crt new file mode 100755 index 00000000..6402874d --- /dev/null +++ b/.ci/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUIwN+0zglsexRKwE1RGHvlCcmrdwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMjQwWhcNMjIwMjEyMDcyMjQwWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANILs0JO0e7x29zeVx21qalK +XKdX+AMlGJPH75wWO/Jq6YHtxt1wYIg762krOBXfG6JsFSOIwIv5VrzGGRGjSPt9 +OXQyXrDDiQvsBT3rpzLNdDs7KMl2tZswwv7w9ujgud0cYnS1MOpn81rfPc73DvMg +xuhplofDx6fn3++PjVRU2FNiIVWyEoaxRjCeGPMBubKZYaYbQA6vYM4Z+ByG727B +AyAER3t7xmvYti/EoO2hv2HQk5zgcj/Oq3AJKhnt8LH8fnfm3TnYNM1htvXqhN05 +vsvhvm2PHfnA5qLlSr/3W0aI/U/PqfsFDCgyRV097sMIaKkmavb0Ue7aQ7lgtp0C +AwEAAaNTMFEwHQYDVR0OBBYEFDRKlCMowWR1rwxE0d1lTEQe5O71MB8GA1UdIwQY +MBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAKbCJ95EBpeuvF70KEt6QU70k/SH1NRvM9YzKryV0D975Jvu +HOSm9HgSTULeAUFZIa4oYyf3QUfVoI+2T/aQrfXA3gfrJWsHURkyNmiHOFAbYHqi +xA6i249G2GTEjc1+le/M2N2CcDKAmurW6vSGK4upXQbPd6KmnhHREX74zkWjnOa+ ++tibbSSOCT4Tmja2DbBxAPuivU9IB1g/hIUmbYQqKffQrBJA0658tz6w63a/Q7xN +pCvvbSgiMZ6qcVIcJkBT2IooYie+ax45pQECHthgIUcQAzfmIfqlU0Qfl8rDgAmn +0c1o6HQjKGU2aVGgSRuaaiHaSZjbPIZVS51sOoI= +-----END CERTIFICATE----- diff --git a/.ci/certs/ca.pem b/.ci/certs/ca.pem new file mode 100644 index 00000000..6402874d --- /dev/null +++ b/.ci/certs/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUIwN+0zglsexRKwE1RGHvlCcmrdwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMjQwWhcNMjIwMjEyMDcyMjQwWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANILs0JO0e7x29zeVx21qalK +XKdX+AMlGJPH75wWO/Jq6YHtxt1wYIg762krOBXfG6JsFSOIwIv5VrzGGRGjSPt9 +OXQyXrDDiQvsBT3rpzLNdDs7KMl2tZswwv7w9ujgud0cYnS1MOpn81rfPc73DvMg +xuhplofDx6fn3++PjVRU2FNiIVWyEoaxRjCeGPMBubKZYaYbQA6vYM4Z+ByG727B +AyAER3t7xmvYti/EoO2hv2HQk5zgcj/Oq3AJKhnt8LH8fnfm3TnYNM1htvXqhN05 +vsvhvm2PHfnA5qLlSr/3W0aI/U/PqfsFDCgyRV097sMIaKkmavb0Ue7aQ7lgtp0C +AwEAAaNTMFEwHQYDVR0OBBYEFDRKlCMowWR1rwxE0d1lTEQe5O71MB8GA1UdIwQY +MBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAKbCJ95EBpeuvF70KEt6QU70k/SH1NRvM9YzKryV0D975Jvu +HOSm9HgSTULeAUFZIa4oYyf3QUfVoI+2T/aQrfXA3gfrJWsHURkyNmiHOFAbYHqi +xA6i249G2GTEjc1+le/M2N2CcDKAmurW6vSGK4upXQbPd6KmnhHREX74zkWjnOa+ ++tibbSSOCT4Tmja2DbBxAPuivU9IB1g/hIUmbYQqKffQrBJA0658tz6w63a/Q7xN +pCvvbSgiMZ6qcVIcJkBT2IooYie+ax45pQECHthgIUcQAzfmIfqlU0Qfl8rDgAmn +0c1o6HQjKGU2aVGgSRuaaiHaSZjbPIZVS51sOoI= +-----END CERTIFICATE----- diff --git a/.ci/certs/testnode.crt b/.ci/certs/testnode.crt new file mode 100755 index 00000000..ff3bcb37 --- /dev/null +++ b/.ci/certs/testnode.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIjCCAgqgAwIBAgIUI4QU6jA1dYSCbdIA6oAb2TBEluowDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMzEzWhcNMjIwMjEyMDcyMzEzWjATMREwDwYD +VQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJeT +yOy6EAScZxrULKjHePciiz38grivCrhFFV+dThaRCcl3DhDzb9Eny5q5iEw3WvLQ +Rqmf01jncNIhaocTt66VqveXaMubbE8O0LcG6e4kpFO+JtnVF8JTARTc+ux/1uD6 +hO1VG/HItM7WQrQxh4hfB2u1AX2YQtoqEtXXEC+UHWfl4QzuzXjBnKCkO/L9/6Tf +yNFQWXxKnIiTs8Xm9sEhhSCBJPlLTQu+MX4vR2Uwj5XZmflDUr+ZTenl9qYxL6b3 +SWhh/qEl4GAj1+tS7ZZOxE0237mUh3IIFYSWSaMm8K2m/BYHkLNWL5B1dMic0lsv +osSoYrQuCef4HQMCitsCAwEAAaNNMEswHQYDVR0OBBYEFFMg4l1GLW8lYbwASY+r +YeWYRzIiMB8GA1UdIwQYMBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQELBQADggEBAEQrgh1xALpumQTzsjxFRGque/vlKTgRs5Kh +xtgapr6wjIbdq7dagee+4yNOKzS5lGVXCgwrJlHESv9qY0uumT/33vK2uduJ7NAd +fR2ZzyBnhMX+mkYhmGrGYCTUMUIwOIQYa4Evis4W+LHmCIDG03l7gLHfdIBe9VMO +pDZum8f6ng0MM49s8/rXODNYKw8kFyUhnfChqMi/2yggb1uUIfKlJJIchkgYjE13 +zuC+fjo029Pq1jeMIdxugLf/7I/8NiW1Yj9aCXevUXG1qzHFEuKAinBXYOZO/vWS +LaEqOhwrzNynwgGpYAr7Rfgv4AflltYIIav4PZT03P7fbyAAf8s= +-----END CERTIFICATE----- diff --git a/.ci/certs/testnode.key b/.ci/certs/testnode.key new file mode 100755 index 00000000..c35b4bc8 --- /dev/null +++ b/.ci/certs/testnode.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAl5PI7LoQBJxnGtQsqMd49yKLPfyCuK8KuEUVX51OFpEJyXcO +EPNv0SfLmrmITDda8tBGqZ/TWOdw0iFqhxO3rpWq95doy5tsTw7Qtwbp7iSkU74m +2dUXwlMBFNz67H/W4PqE7VUb8ci0ztZCtDGHiF8Ha7UBfZhC2ioS1dcQL5QdZ+Xh +DO7NeMGcoKQ78v3/pN/I0VBZfEqciJOzxeb2wSGFIIEk+UtNC74xfi9HZTCPldmZ ++UNSv5lN6eX2pjEvpvdJaGH+oSXgYCPX61Ltlk7ETTbfuZSHcggVhJZJoybwrab8 +FgeQs1YvkHV0yJzSWy+ixKhitC4J5/gdAwKK2wIDAQABAoIBAQCRFTJna/xy/WUu +59FLR4qAOj8++JgCwACpue4oU7/vl6nffSYokWoAr2+RzG4qTX2vFi3cpA8+dGCn +sLZvTi8tWzKGxBTZdg2oakzaMzLr74SeZ052iCGyrZJGbvF6Ny7srr1XEXSq6+os +ZCb6pMHOhO7saBdiKMAsY8MdjTl/33AduuE6ztqv+L92xTr2g4QlbT1KvWlEgppU +k4Gy7zdETkPBTSH/17ZwyGJoJICIAhbL4IpmOM4dPIg8nFkVPPpy6p0z4uGjtgnK +nreZ2EKMzCafBaHn7A77gpi0OrQdl6pe0fsGqv/323YjCJPbwwl5TsoNq44DzwiX +3M7XiVJxAoGBAOCne56vdN4uZmCgLVGT2JSUNVPOu4bfjrxWH6cslzrPT2Zhp3lO +M4axZ3gmcervV252YEZXntXDHHCSfrECllRN1WFD63XmyQ/CkhuvZkkeRHfzL1TE +EdqHOTqs4sRETZ7+RITFC81DZQkWWOKeyXMjyPBqd7RnThQHijB1c8Y5AoGBAKy6 +CVKBx+zz5crVD0tz4UhOmz1wRNN0CL0l+FXRuFSgbzMIvwpfiqe25crgeLHe2M2/ +TogdWbjZ2nUZQTzoRsSkQ6cKHpj+G/gWurp/UcHHXFVwgLSPF7c3KHDtiYq7Vqw0 +bvmhM03LI6+ZIPRV7hLBr7WP7UmpAiREMF7tTnmzAoGBAIkx3w3WywFQxtblmyeB +qbd7F2IaE23XoxyjX+tBEQ4qQqwcoSE0v8TXHIBEwjceeX+NLVhn9ClJYVniLRq+ +oL3VVqVyzB4RleJZCc98e3PV1yyFx/b1Uo3pHOsXX9lKeTjKwV9v0rhFGzPEgP3M +yOvXA8TG0FnM6OLUg/D6GX0JAoGAMuHS4TVOGeV3ahr9mHKYiN5vKNgrzka+VEod +L9rJ/FQOrfADpyCiDen5I5ygsXU+VM3oanyK88NpcVlxOGoMft0M+OYoQVWKE7lO +ZKYhBX6fGqQ7pfUJPXXIOgwfmni5fZ0sm+j63g3bg10OsiumKGxaQJgXhL1+3gQg +Y7ZwibUCgYEAlZoFFvkMLjpOSaHk1z5ZZnt19X0QUIultBwkumSqMPm+Ks7+uDrx +thGUCoz4ecr/ci4bIUY7mB+zfAbqnBOMxreJqCRbAIuRypo1IlWkTp8DywoDOfMW +NfzjVmzJ7EJu44nGmVAi1jw4Pbseivvi1ujMCoPgaE8I1uSh144bwN8= +-----END RSA PRIVATE KEY----- diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml new file mode 100644 index 00000000..0190c51c --- /dev/null +++ b/.ci/jobs/defaults.yml @@ -0,0 +1,76 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: clients-ci + +##### JOB DEFAULTS + +- job: + project-type: matrix + logrotate: + daysToKeep: 30 + numToKeep: 100 + properties: + - github: + url: https://github.com/elastic/elasticsearch-py/ + - inject: + properties-content: HOME=$JENKINS_HOME + concurrent: true + node: flyweight + scm: + - git: + name: origin + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/elasticsearch-py.git + branches: + - ${branch_specifier} + url: git@github.com:elastic/elasticsearch-py.git + wipe-workspace: 'True' + triggers: + - github + - timed: '@daily' + axes: + - axis: + type: slave + name: label + values: + - linux + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: ELASTICSEARCH_VERSION + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: PYTHON_VERSION + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: PYTHON_CONNECTION_CLASS + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: TEST_SUITE + yaml-strategy: + exclude-key: exclude + filename: .ci/test-matrix.yml + wrappers: + - ansicolor + - timeout: + type: absolute + timeout: 120 + fail: true + - timestamps + - workspace-cleanup + builders: + - shell: |- + #!/usr/local/bin/runbld + .ci/run-tests + publishers: + - email: + recipients: infra-root+build@elastic.co + - junit: + results: "*-junit.xml" + allow-empty-results: true diff --git a/.ci/jobs/elastic+elasticsearch-py+7.x.yml b/.ci/jobs/elastic+elasticsearch-py+7.x.yml new file mode 100644 index 00000000..9bbb24e8 --- /dev/null +++ b/.ci/jobs/elastic+elasticsearch-py+7.x.yml @@ -0,0 +1,12 @@ +--- +- job: + name: elastic+elasticsearch-py+7.x + display-name: 'elastic / elasticsearch-py # 7.x' + description: Testing the elasticsearch-py 7.x branch. + junit_results: "*-junit.xml" + parameters: + - string: + name: branch_specifier + default: refs/heads/7.x + description: the Git branch specifier to build (<branchName>, <tagName>, + <commitId>, etc.) diff --git a/.ci/jobs/elastic+elasticsearch-py+master.yml b/.ci/jobs/elastic+elasticsearch-py+master.yml new file mode 100644 index 00000000..f83fe6de --- /dev/null +++ b/.ci/jobs/elastic+elasticsearch-py+master.yml @@ -0,0 +1,12 @@ +--- +- job: + name: elastic+elasticsearch-py+master + display-name: 'elastic / elasticsearch-py # master' + description: Testing the elasticsearch-py master branch. + junit_results: "*-junit.xml" + parameters: + - string: + name: branch_specifier + default: refs/heads/master + description: the Git branch specifier to build (<branchName>, <tagName>, + <commitId>, etc.) diff --git a/.ci/jobs/elastic+elasticsearch-py+pull-request.yml b/.ci/jobs/elastic+elasticsearch-py+pull-request.yml new file mode 100644 index 00000000..813fff24 --- /dev/null +++ b/.ci/jobs/elastic+elasticsearch-py+pull-request.yml @@ -0,0 +1,19 @@ +--- +- job: + name: elastic+elasticsearch-py+pull-request + display-name: 'elastic / elasticsearch-py # pull-request' + description: Testing of elasticsearch-py pull requests. + junit_results: "*-junit.xml" + scm: + - git: + branches: + - ${ghprbActualCommit} + refspec: +refs/pull/*:refs/remotes/origin/pr/* + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + github-hooks: true + status-context: clients-ci + cancel-builds-on-update: true diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh new file mode 100755 index 00000000..a2424159 --- /dev/null +++ b/.ci/run-elasticsearch.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. + +# Version 1.0.2 +# - Initial version of the run-elasticsearch.sh script +# - Deleting the volume should not dependent on the container still running +# - Fixed `ES_JAVA_OPTS` config + +if [[ -z "$ELASTICSEARCH_VERSION" ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi + +set -euxo pipefail + +SCRIPT_PATH=$(dirname $(realpath -s $0)) + +moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-') +suffix=rest-test + +NODE_NAME=${NODE_NAME-${moniker}node1} +MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}} +CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}} +HTTP_PORT=${HTTP_PORT-9200} + +ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme} +SSL_CERT=${SSL_CERT-"${SCRIPT_PATH}/certs/testnode.crt"} +SSL_KEY=${SSL_KEY-"${SCRIPT_PATH}/certs/testnode.key"} +SSL_CA=${SSL_CA-"${SCRIPT_PATH}/certs/ca.crt"} +SSL_CA_PEM=${SSL_CA-"${SCRIPT_PATH}/certs/ca.pem"} + +DETACH=${DETACH-false} +CLEANUP=${CLEANUP-false} + +volume_name=${NODE_NAME}-${suffix}-data +network_default=${moniker}${suffix} +NETWORK_NAME=${NETWORK_NAME-"$network_default"} + +set +x + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + fi + echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m" + cleanup_volume "$1-${suffix}-data" +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup { + if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m" + cleanup_node "$NODE_NAME" + fi + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_network "$NETWORK_NAME" + fi +}; +trap "cleanup 0" EXIT + +if [[ "$CLEANUP" == "true" ]]; then + trap - EXIT + if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then + echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m" + exit 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME}) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network "$NETWORK_NAME" + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m" +cleanup 1 + +echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m" +docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME" + +environment=($(cat <<-END + --env node.name=$NODE_NAME + --env cluster.name=$CLUSTER_NAME + --env cluster.initial_master_nodes=$MASTER_NODE_NAME + --env discovery.seed_hosts=$MASTER_NODE_NAME + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* +END +)) + +volumes=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data +END +)) + +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + environment+=($(cat <<-END + --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD + --env xpack.license.self_generated.type=trial + --env xpack.security.enabled=true + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key + --volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt + --volume $SSL_CA_PEM:/usr/share/elasticsearch/config/certs/ca.pem +END +)) +fi + +url="http://$NODE_NAME" +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME" +fi + +cert_validation_flags="--insecure" +if [[ "$NODE_NAME" == "instance" ]]; then + cert_validation_flags="--cacert /usr/share/elasticsearch/config/certs/ca.pem --resolve ${NODE_NAME}:443:127.0.0.1" +fi + +echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m" + +set -x +docker run \ + --name "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$HTTP_PORT":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$DETACH" \ + --health-cmd="curl $cert_validation_flags --fail $url:9200/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION"; +set +x + +if [[ "$DETACH" == "true" ]]; then + until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME} + echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m" + sleep 2; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running $NODE_NAME; then + docker logs $NODE_NAME + fi + + if ! container_running $NODE_NAME || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then + cleanup 1 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + exit 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m" + echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m" + exit 0 + fi +fi diff --git a/.ci/run-repository.sh b/.ci/run-repository.sh new file mode 100755 index 00000000..871f4691 --- /dev/null +++ b/.ci/run-repository.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Called by entry point `run-test` use this script to add your repository specific test commands +# Once called Elasticsearch is up and running and the following parameters are available to this script + +# ELASTICSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease) +# ELASTICSEARCH_CONTAINER -- the docker moniker as a reference to know which docker image distribution is used +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable +# NETWORK_NAME -- The docker network name +# NODE_NAME -- The docker container name also used as Elasticsearch node name + +# When run in CI the test-matrix is used to define additional variables +# TEST_SUITE -- either `oss` or `xpack`, defaults to `oss` in `run-tests` + +set -e + +echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m" +echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLASS}\033[0m" + +echo -e "\033[1m>>>>> Build [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build \ + --file .ci/Dockerfile \ + --tag elastic/elasticsearch-py \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ + . + +echo -e "\033[1m>>>>> Run [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker run \ + --network=${NETWORK_NAME} \ + --env "ELASTICSEARCH_HOST=${ELASTICSEARCH_URL}" \ + --env "TEST_SUITE=${TEST_SUITE}" \ + --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \ + --name elasticsearch-py \ + --rm \ + elastic/elasticsearch-py \ + python setup.py test diff --git a/.ci/run-tests b/.ci/run-tests new file mode 100755 index 00000000..ff84f3a1 --- /dev/null +++ b/.ci/run-tests @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +if [[ -z $ELASTICSEARCH_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi +set -euxo pipefail + + +PYTHON_VERSION=${PYTHON_VERSION-3.8} +PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS-Urllib3HttpConnection} +TEST_SUITE=${TEST_SUITE-oss} +NODE_NAME=instance + +elasticsearch_image=elasticsearch +elasticsearch_url=https://elastic:changeme@${NODE_NAME}:9200 +if [[ $TEST_SUITE != "xpack" ]]; then + elasticsearch_image=elasticsearch-${TEST_SUITE} + elasticsearch_url=http://${NODE_NAME}:9200 +fi + +function cleanup { + status=$? + set +x + ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=elasticsearch \ + CLEANUP=true \ + bash ./.ci/run-elasticsearch.sh + # Report status and exit + if [[ "$status" == "0" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +} +trap cleanup EXIT + +echo -e "\033[1m>>>>> Start [$ELASTICSEARCH_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=elasticsearch \ + DETACH=true \ + bash .ci/run-elasticsearch.sh + +echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NETWORK_NAME=elasticsearch \ + NODE_NAME=${NODE_NAME} \ + ELASTICSEARCH_URL=${elasticsearch_url} \ + PYTHON_VERSION=${PYTHON_VERSION} \ + PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS} \ + TEST_SUITE=${TEST_SUITE} \ + bash .ci/run-repository.sh diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml new file mode 100755 index 00000000..5a2fd52a --- /dev/null +++ b/.ci/test-matrix.yml @@ -0,0 +1,20 @@ +ELASTICSEARCH_VERSION: + - 7.5-SNAPSHOT + +TEST_SUITE: + - oss + - xpack + +PYTHON_VERSION: + - 2.7 + - 3.4 + - 3.5 + - 3.6 + - 3.7 + - 3.8 + +PYTHON_CONNECTION_CLASS: + - Urllib3HttpConnection + - RequestsHttpConnection + +exclude: ~ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b094a512 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +docs +example +venv +.git +.tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f59d92c3..00000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -language: python - -services: - - docker - -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - -env: - # different connection classes to test - - TEST_ES_CONNECTION=Urllib3HttpConnection - - TEST_ES_CONNECTION=RequestsHttpConnection - -before_install: - - docker run --rm --detach --publish 9200:9200 --env "discovery.type=single-node" -e path.repo=/tmp -e "repositories.url.allowed_urls=http://*" -e node.attr.testattr=test -e node.name=test --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:7.5.1 - -install: - - git clone https://github.com/elastic/elasticsearch.git ../elasticsearch - -script: - - python setup.py test diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 00000000..792f9019 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,14 @@ +requests>=2, <3 +nose +coverage +mock +nosexcover +sphinx<1.7 +sphinx_rtd_theme +jinja2 + +# PyYAML 5.3 dropped support for Python 3.4 while +# not amending that requirement to the package. :( +pyyaml<5.3 + +black; python_version>="3.6" diff --git a/elasticsearch/helpers/test.py b/elasticsearch/helpers/test.py index cdc7a981..4d120c47 100644 --- a/elasticsearch/helpers/test.py +++ b/elasticsearch/helpers/test.py @@ -14,13 +14,13 @@ from elasticsearch.exceptions import ConnectionError def get_test_client(nowait=False, **kwargs): # construct kwargs from the environment kw = {"timeout": 30} - if "TEST_ES_CONNECTION" in os.environ: - from elasticsearch import connection - kw["connection_class"] = getattr(connection, os.environ["TEST_ES_CONNECTION"]) + if "PYTHON_CONNECTION_CLASS" in os.environ: + from elasticsearch import connection + kw["connection_class"] = getattr(connection, os.environ["PYTHON_CONNECTION_CLASS"]) kw.update(kwargs) - client = Elasticsearch([os.environ.get("TEST_ES_SERVER", {})], **kw) + client = Elasticsearch([os.environ.get("ELASTICSEARCH_HOST", {})], **kw) # wait for yellow status for _ in range(1 if nowait else 100): diff --git a/test_elasticsearch/README.rst b/test_elasticsearch/README.rst index 75071a05..5b4d32ac 100644 --- a/test_elasticsearch/README.rst +++ b/test_elasticsearch/README.rst @@ -11,12 +11,12 @@ version that was used to build the server we are running against. Running the tests ----------------- -To simply run the tests just execute the ``run_tests.py`` script or invoke -``python setup.py test``. The behavior is driven by environmental variables: +To simply run the tests just execute ``python setup.py test``. +The behavior is driven by environmental variables: - * ``TEST_ES_SERVER`` - can contain "hostname[:port]" of running es cluster + * ``ELASTICSEARCH_HOST`` - can contain "hostname[:port]" of running es cluster - * ``TEST_ES_CONNECTION`` - name of the connection class to use from + * ``PYTHON_CLASS_CONNECTION`` - name of the connection class to use from ``elasticsearch.connection`` module. If you want to run completely with your own see section on customizing tests. @@ -34,8 +34,6 @@ To simply run the tests just execute the ``run_tests.py`` script or invoke Alternatively, if you wish to control what you are doing you have several additional options: - * ``run_tests.py`` will pass any parameters specified to ``nosetests`` - * you can just run your favorite runner in the ``test_elasticsearch`` directory (verified to work with nose and py.test) and bypass the fetch logic entirely. @@ -47,13 +45,16 @@ To install all test dependencies you can also run ``pip install -e .[develop]``. Run Elasticsearch in a Container -------------------------------- -To run elasticsearch in a container, optionally set the ``ES_VERSION`` -environment evariable to either 5.4, 5.3 or 2.4. ``ES_VERSION`` is defaulted to -``latest``. Then run ./start_elasticsearch.sh:: +To run elasticsearch in a container set the ``ELASTICSEARCH_VERSION`` +environment variable to 7.5, 7.5-SNAPSHOT. Then run ./.ci/run_elasticsearch.sh:: - export ES_VERSION=5.4 - ./start_elasticsearch.sh + # Run OSS Elasticsearch v7.5 + export ELASTICSEARCH_VERSION=elasticsearch-oss:7.5-SNAPSHOT + ./.ci/run_elasticsearch.sh + # Run XPACK Elasticsearch v7.5 + export ELASTICSEARCH_VERSION=elasticsearch:7.5-SNAPSHOT + ./.ci/run_elasticsearch.sh This will run a version for Elasticsearch in a Docker container suitable for running the tests. To check that elasticsearch is running first wait for a diff --git a/test_elasticsearch/run_tests.py b/test_elasticsearch/run_tests.py index 9a2c530e..b6a1a25d 100755 --- a/test_elasticsearch/run_tests.py +++ b/test_elasticsearch/run_tests.py @@ -21,10 +21,10 @@ def fetch_es_repo(): # no repo if not exists(repo_path) or not exists(join(repo_path, ".git")): - print("No elasticsearch repo found...") - # set YAML DIR to empty to skip yaml tests - environ["TEST_ES_YAML_DIR"] = "" - return + subprocess.check_call( + "git clone https://github.com/elastic/elasticsearch %s" % repo_path, + shell=True, + ) # set YAML test dir environ["TEST_ES_YAML_DIR"] = join( @@ -53,7 +53,7 @@ def fetch_es_repo(): % repo_path, shell=True, ) - # reset to the version fron info() + # reset to the version from info() subprocess.check_call("cd %s && git fetch" % repo_path, shell=True) subprocess.check_call("cd %s && git reset --hard %s" % (repo_path, sha), shell=True) diff --git a/test_elasticsearch/start_elasticsearch.sh b/test_elasticsearch/start_elasticsearch.sh deleted file mode 100755 index ae3a0cb5..00000000 --- a/test_elasticsearch/start_elasticsearch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Start elasticsearch in a docker container - -ES_VERSION=${ES_VERSION:-"7.5.0"} -ES_TEST_SERVER=${ES_TEST_SERVER:-"http://localhost:9200"} - -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -exec docker run -d \ - -e path.repo=/tmp \ - -e "repositories.url.allowed_urls=http://*" \ - -e node.attr.testattr=test \ - -e node.name=test \ - -e ES_HOST=$ES_TEST_SERVER \ - -e cluster.initial_master_nodes=test \ - -v $SOURCE_DIR/../elasticsearch:/code/elasticsearch \ - -v /tmp:/tmp \ - -p "9200:9200" \ -docker.elastic.co/elasticsearch/elasticsearch-oss:$ES_VERSION diff --git a/test_elasticsearch/test_server/test_common.py b/test_elasticsearch/test_server/test_common.py index 19910293..ae7ceed0 100644 --- a/test_elasticsearch/test_server/test_common.py +++ b/test_elasticsearch/test_server/test_common.py @@ -9,7 +9,7 @@ from os.path import exists, join, dirname, pardir import yaml from shutil import rmtree -from elasticsearch import TransportError +from elasticsearch import TransportError, RequestError from elasticsearch.compat import string_types from elasticsearch.helpers.test import _get_version @@ -108,10 +108,20 @@ class YamlTestCase(ElasticsearchTestCase): def _feature_enabled(self, name): global XPACK_FEATURES if XPACK_FEATURES is None: - xinfo = self.client.xpack.info() - XPACK_FEATURES = set( - f for f in xinfo["features"] if xinfo["features"][f]["enabled"] - ) + try: + xinfo = self.client.xpack.info() + XPACK_FEATURES = set( + f for f in xinfo["features"] if xinfo["features"][f]["enabled"] + ) + except RequestError as e: + # We receive a 'RequestError' here when using 'oss' because the request + # for xpack.info() looks like this: 'GET /_xpack' which errors on the + # oss container due to having a similar route to 'GET /'. + if "invalid_index_name_exception" not in e.error: + raise + + XPACK_FEATURES = set() + return name in XPACK_FEATURES def _resolve(self, value): @@ -236,9 +246,7 @@ class YamlTestCase(ElasticsearchTestCase): elif feature == "benchmark": if self._get_benchmark_nodes(): continue - raise SkipTest( - skip.get("reason", "Feature %s is not supported" % feature) - ) + raise SkipTest("Feature %s is not supported" % feature) if "version" in skip: version, reason = skip["version"], skip["reason"] diff --git a/utils/Dockerfile b/utils/Dockerfile deleted file mode 100644 index 6b84ac16..00000000 --- a/utils/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -ARG PYTHON_VERSION=3 -FROM python:${PYTHON_VERSION} -RUN apt-get update && apt-get install -y git curl -RUN pip install ipdb python-dateutil GitPython - -RUN git clone https://github.com/elastic/elasticsearch.git /code/elasticsearch - -WORKDIR /code/elasticsearch-py -COPY .. . -RUN pip install .[develop] -RUN python setup.py develop -CMD ["/code/wait-for-elasticsearch.sh", "http://elasticsearch:9200", "--", "python", "setup.py", "test"] diff --git a/utils/docker-compose.yml b/utils/docker-compose.yml deleted file mode 100644 index 33fc4573..00000000 --- a/utils/docker-compose.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: '3.2' -services: - client: - image: docker.elastic.co/clients/elasticsearch-py:${PYTHON_VERSION:-3} - build: - context: . - dockerfile: ./Dockerfile - args: - PYTHON_VERSION: ${PYTHON_VERSION:-3} - environment: - - "TEST_ES_SERVER=http://elasticsearch:9200" - volumes: - - ..:/code/elasticsearch-py - - esvol:/tmp - networks: - - esnet - depends_on: - - elasticsearch - command: ["true"] # dummy command to override the command in the Dockerfile and do nothing. - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:${ELASTICSEARCH_VERSION:-6.2.4} - volumes: - - esvol:/tmp - networks: - - esnet - environment: - - path.repo=/tmp - - "repositories.url.allowed_urls=http://*" - - node.attr.testattr=test - - bootstrap.memory_lock=false - - "node.name=test" - - "cluster.initial_master_nodes=test" - - "discovery.zen.ping.unicast.hosts=elasticsearch" - - "http.max_content_length=5mb" -networks: - esnet: -volumes: - esvol: diff --git a/utils/wait-for-elasticsearch.sh b/utils/wait-for-elasticsearch.sh deleted file mode 100644 index 9a6f2201..00000000 --- a/utils/wait-for-elasticsearch.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - -host="$1" -shift -cmd="$@" - - -until $(curl --output /dev/null --silent --head --fail "$host"); do - printf '.' - sleep 1 -done - -# First wait for ES to start... -response=$(curl $host) - -until [ "$response" = "200" ]; do - response=$(curl --write-out %{http_code} --silent --output /dev/null "$host") - >&2 echo "Elasticsearch is unavailable - sleeping" - sleep 1 -done - - -# next wait for ES status to turn to Green -health="$(curl -fsSL "$host/_cat/health?h=status")" -health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") - -until [ "$health" = 'green' ]; do - health="$(curl -fsSL "$host/_cat/health?h=status")" - health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") - >&2 echo "Elasticsearch is unavailable - sleeping" - sleep 1 -done - ->&2 echo "Elasticsearch is up" -exec $cmd