Support both OpenDistro and OpenSearch in CI

Note that it supports only without security plugin for now

Signed-off-by: Rushi Agrawal <[email protected]>

.ci/Dockerfile.server

@@ -1,3 +0,0 @@
-FROM amazon/opendistro-for-elasticsearch:1.13.2
-RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security
-COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch.yml /usr/share/elasticsearch/config/

.ci/functions/imports.sh

@@ -22,11 +22,16 @@ if [[ -z $es_node_name ]]; then
   export RUNSCRIPTS=${RUNSCRIPTS-}
   export DETACH=${DETACH-false}
   export CLEANUP=${CLEANUP-false}
+  export ELASTICSEARCH_URL_EXTENSION=${ELASTICSEARCH_URL_EXTENSION-http}
 
   export es_node_name=instance
   export elastic_password=changeme
-  export elasticsearch_image=opendistro-for-elasticsearch
+  export elasticsearch_image=opensearchproject/opensearch
-  export elasticsearch_url=http://elastic:${elastic_password}@${es_node_name}:9200
+  if [[ "$CLUSTER" == "opendistro" ]]; then
+    export elasticsearch_image=amazon/opendistro-for-elasticsearch
+  fi
+
+  export elasticsearch_url=$ELASTICSEARCH_URL_EXTENSION://${es_node_name}:9200
   export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
   export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
 

.ci/opendistro/Dockerfile

@@ -0,0 +1,4 @@
+FROM amazon/opendistro-for-elasticsearch:1.13.2
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi
+COPY --chown=elasticsearch:elasticsearch .ci/opendistro/elasticsearch.yml /usr/share/elasticsearch/config/

.ci/opensearch/Dockerfile

@@ -0,0 +1,12 @@
+ARG OPENSEARCH_VERSION
+FROM opensearchproject/opensearch:latest
+
+ARG opensearch_path=/usr/share/opensearch
+ARG opensearch_yml=$opensearch_path/config/opensearch.yml
+
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi
+
+COPY --chown=opensearch:opensearch .ci/opensearch/opensearch-run.sh $opensearch_path/
+
+CMD /usr/share/opensearch/opensearch-run.sh

.ci/opensearch/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3'
+
+services:
+
+  opensearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+    command: /usr/share/opensearch/opensearch-run.sh
+    ports:
+      - "9200:9200"
+    user: opensearch

.ci/opensearch/opensearch-run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -ex
+
+/usr/share/opensearch/bin/opensearch -Ediscovery.type=single-node

.ci/run-elasticsearch.sh

@@ -48,7 +48,7 @@ END
 docker_pull_attempts=0
 until [ "$docker_pull_attempts" -ge 5 ]
 do
-   docker pull amazon/"$elasticsearch_container" && break
+   docker pull "$elasticsearch_container" && break
    docker_pull_attempts=$((docker_pull_attempts+1))
    echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
    sleep 10
@@ -75,11 +75,12 @@ END
   local_detach="true"
   if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
 
-
+  echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m"
-  echo -e "\033[34;1mINFO: building odfe container\033[0m"
+  echo 'cluster is' $CLUSTER
   docker build \
-    --file=.ci/Dockerfile.server \
+    --file=.ci/$CLUSTER/Dockerfile \
-    --tag=odfe \
+    --build-arg SECURE_INTEGRATION=false \
+    --tag=$CLUSTER \
     .
 
   echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
@@ -99,7 +100,8 @@ END
     --health-retries=20 \
     --health-timeout=2s \
     --rm \
-    odfe;
+    -d \
+    $CLUSTER;
 
   set +x
   if wait_for_container "$es_node_name" "$network_name"; then

.ci/run-repository.sh

@@ -14,7 +14,7 @@
 
 set -e
 
-echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m URL ${elasticsearch_url}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"

.ci/run-tests

@@ -5,10 +5,12 @@
 # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
 
 # Default environment variables
-export STACK_VERSION="${STACK_VERSION:=1.13.2}"
+export STACK_VERSION="${STACK_VERSION:=latest}"
 export TEST_SUITE="${TEST_SUITE:=oss}"
 export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
+export ELASTICSEARCH_URL_EXTENSION="${ELASTICSEARCH_URL_EXTENSION:=http}"
+export CLUSTER="${1:=opensearch}"
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh

.github/workflows/integration.yml

@@ -20,9 +20,10 @@ jobs:
       fail-fast: false
       matrix:
         stack_version: ['7.10.2']
+        cluster: ["opendistro", "opensearch"]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - run: "./.ci/run-tests"
+      - run: "./.ci/run-tests ${{ matrix.cluster }} "
-        name: Integ ${{ matrix.stack_version }}
+        name: Integ ${{ matrix.stack_version }} ${{ matrix.cluster }}

test_elasticsearch/run_tests.py

@@ -40,13 +40,13 @@ def fetch_es_repo():
 
     repo_path = environ.get(
         "TEST_ES_REPO",
-        abspath(join(dirname(__file__), pardir, pardir, "elasticsearch")),
+        abspath(join(dirname(__file__), pardir, pardir, "opensearch")),
     )
 
     # no repo
     if not exists(repo_path) or not exists(join(repo_path, ".git")):
         subprocess.check_call(
-            "git clone https://github.com/elastic/elasticsearch %s" % repo_path,
+            "git clone https://github.com/opensearch-project/opensearch %s" % repo_path,
             shell=True,
         )
 
@@ -73,7 +73,8 @@ def fetch_es_repo():
     # fetch new commits to be sure...
     print("Fetching elasticsearch repo...")
     subprocess.check_call(
-        "cd %s && git fetch https://github.com/elastic/elasticsearch.git" % repo_path,
+        "cd %s && git fetch https://github.com/opensearch-project/opensearch.git"
+        % repo_path,
         shell=True,
     )
     # reset to the version from info()