diff --git a/.ci/Dockerfile.server b/.ci/Dockerfile.server deleted file mode 100644 index e9cdcd0f..00000000 --- a/.ci/Dockerfile.server +++ /dev/null @@ -1,3 +0,0 @@ -FROM amazon/opendistro-for-elasticsearch:1.13.2 -RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security -COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch.yml /usr/share/elasticsearch/config/ diff --git a/.ci/functions/imports.sh b/.ci/functions/imports.sh index e66de8e3..e4daa47e 100755 --- a/.ci/functions/imports.sh +++ b/.ci/functions/imports.sh @@ -22,11 +22,16 @@ if [[ -z $es_node_name ]]; then export RUNSCRIPTS=${RUNSCRIPTS-} export DETACH=${DETACH-false} export CLEANUP=${CLEANUP-false} + export ELASTICSEARCH_URL_EXTENSION=${ELASTICSEARCH_URL_EXTENSION-http} export es_node_name=instance export elastic_password=changeme - export elasticsearch_image=opendistro-for-elasticsearch - export elasticsearch_url=http://elastic:${elastic_password}@${es_node_name}:9200 + export elasticsearch_image=opensearchproject/opensearch + if [[ "$CLUSTER" == "opendistro" ]]; then + export elasticsearch_image=amazon/opendistro-for-elasticsearch + fi + + export elasticsearch_url=$ELASTICSEARCH_URL_EXTENSION://${es_node_name}:9200 export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost} export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}" diff --git a/.ci/opendistro/Dockerfile b/.ci/opendistro/Dockerfile new file mode 100644 index 00000000..611022c6 --- /dev/null +++ b/.ci/opendistro/Dockerfile @@ -0,0 +1,4 @@ +FROM amazon/opendistro-for-elasticsearch:1.13.2 +ARG SECURE_INTEGRATION +RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi +COPY --chown=elasticsearch:elasticsearch .ci/opendistro/elasticsearch.yml /usr/share/elasticsearch/config/ diff --git a/.ci/elasticsearch.yml b/.ci/opendistro/elasticsearch.yml similarity index 100% rename from .ci/elasticsearch.yml rename to .ci/opendistro/elasticsearch.yml diff --git a/.ci/opensearch/Dockerfile b/.ci/opensearch/Dockerfile new file mode 100755 index 00000000..3012ba0a --- /dev/null +++ b/.ci/opensearch/Dockerfile @@ -0,0 +1,12 @@ +ARG OPENSEARCH_VERSION +FROM opensearchproject/opensearch:latest + +ARG opensearch_path=/usr/share/opensearch +ARG opensearch_yml=$opensearch_path/config/opensearch.yml + +ARG SECURE_INTEGRATION +RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi + +COPY --chown=opensearch:opensearch .ci/opensearch/opensearch-run.sh $opensearch_path/ + +CMD /usr/share/opensearch/opensearch-run.sh diff --git a/.ci/opensearch/docker-compose.yml b/.ci/opensearch/docker-compose.yml new file mode 100755 index 00000000..3626f0e0 --- /dev/null +++ b/.ci/opensearch/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + + opensearch: + build: + context: . + dockerfile: Dockerfile + args: + - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} + command: /usr/share/opensearch/opensearch-run.sh + ports: + - "9200:9200" + user: opensearch diff --git a/.ci/opensearch/opensearch-run.sh b/.ci/opensearch/opensearch-run.sh new file mode 100755 index 00000000..2f6d0d4b --- /dev/null +++ b/.ci/opensearch/opensearch-run.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -ex + +/usr/share/opensearch/bin/opensearch -Ediscovery.type=single-node diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh index e8409e5d..d16e8ce4 100755 --- a/.ci/run-elasticsearch.sh +++ b/.ci/run-elasticsearch.sh @@ -48,7 +48,7 @@ END docker_pull_attempts=0 until [ "$docker_pull_attempts" -ge 5 ] do - docker pull amazon/"$elasticsearch_container" && break + docker pull "$elasticsearch_container" && break docker_pull_attempts=$((docker_pull_attempts+1)) echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..." sleep 10 @@ -75,11 +75,12 @@ END local_detach="true" if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi - - echo -e "\033[34;1mINFO: building odfe container\033[0m" + echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m" + echo 'cluster is' $CLUSTER docker build \ - --file=.ci/Dockerfile.server \ - --tag=odfe \ + --file=.ci/$CLUSTER/Dockerfile \ + --build-arg SECURE_INTEGRATION=false \ + --tag=$CLUSTER \ . echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m" @@ -99,7 +100,8 @@ END --health-retries=20 \ --health-timeout=2s \ --rm \ - odfe; + -d \ + $CLUSTER; set +x if wait_for_container "$es_node_name" "$network_name"; then diff --git a/.ci/run-repository.sh b/.ci/run-repository.sh index cc71aac9..f43b2123 100755 --- a/.ci/run-repository.sh +++ b/.ci/run-repository.sh @@ -14,7 +14,7 @@ set -e -echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m URL ${elasticsearch_url}\033[0m" echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m" echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m" echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" diff --git a/.ci/run-tests b/.ci/run-tests index 713e4146..822859a1 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -5,10 +5,12 @@ # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products # Default environment variables -export STACK_VERSION="${STACK_VERSION:=1.13.2}" +export STACK_VERSION="${STACK_VERSION:=latest}" export TEST_SUITE="${TEST_SUITE:=oss}" export PYTHON_VERSION="${PYTHON_VERSION:=3.9}" export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}" +export ELASTICSEARCH_URL_EXTENSION="${ELASTICSEARCH_URL_EXTENSION:=http}" +export CLUSTER="${1:=opensearch}" script_path=$(dirname $(realpath -s $0)) source $script_path/functions/imports.sh diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ca26e95f..f5e8f619 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,9 +20,10 @@ jobs: fail-fast: false matrix: stack_version: ['7.10.2'] + cluster: ["opendistro", "opensearch"] steps: - name: Checkout uses: actions/checkout@v2 - - run: "./.ci/run-tests" - name: Integ ${{ matrix.stack_version }} + - run: "./.ci/run-tests ${{ matrix.cluster }} " + name: Integ ${{ matrix.stack_version }} ${{ matrix.cluster }} diff --git a/test_elasticsearch/run_tests.py b/test_elasticsearch/run_tests.py index d8ad448f..a9f5a216 100755 --- a/test_elasticsearch/run_tests.py +++ b/test_elasticsearch/run_tests.py @@ -40,13 +40,13 @@ def fetch_es_repo(): repo_path = environ.get( "TEST_ES_REPO", - abspath(join(dirname(__file__), pardir, pardir, "elasticsearch")), + abspath(join(dirname(__file__), pardir, pardir, "opensearch")), ) # no repo if not exists(repo_path) or not exists(join(repo_path, ".git")): subprocess.check_call( - "git clone https://github.com/elastic/elasticsearch %s" % repo_path, + "git clone https://github.com/opensearch-project/opensearch %s" % repo_path, shell=True, ) @@ -73,7 +73,8 @@ def fetch_es_repo(): # fetch new commits to be sure... print("Fetching elasticsearch repo...") subprocess.check_call( - "cd %s && git fetch https://github.com/elastic/elasticsearch.git" % repo_path, + "cd %s && git fetch https://github.com/opensearch-project/opensearch.git" + % repo_path, shell=True, ) # reset to the version from info()