Support both OpenDistro and OpenSearch in CI

Note that it supports only without security plugin for now Signed-off-by: Rushi Agrawal <rushi.agr@gmail.com>
2021-08-18 01:32:55 +05:30
parent 75eb02aca8
commit 3dc1881a9d
12 changed files with 61 additions and 18 deletions
@@ -1,3 +0,0 @@
-FROM amazon/opendistro-for-elasticsearch:1.13.2
-RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security
-COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch.yml /usr/share/elasticsearch/config/
@@ -22,11 +22,16 @@ if [[ -z $es_node_name ]]; then
  export RUNSCRIPTS=${RUNSCRIPTS-}
  export DETACH=${DETACH-false}
  export CLEANUP=${CLEANUP-false}
+  export ELASTICSEARCH_URL_EXTENSION=${ELASTICSEARCH_URL_EXTENSION-http}

  export es_node_name=instance
  export elastic_password=changeme
-  export elasticsearch_image=opendistro-for-elasticsearch
-  export elasticsearch_url=http://elastic:${elastic_password}@${es_node_name}:9200
+  export elasticsearch_image=opensearchproject/opensearch
+  if [[ "$CLUSTER" == "opendistro" ]]; then
+    export elasticsearch_image=amazon/opendistro-for-elasticsearch
+  fi
+
+  export elasticsearch_url=$ELASTICSEARCH_URL_EXTENSION://${es_node_name}:9200
  export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
  export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"

@@ -0,0 +1,4 @@
+FROM amazon/opendistro-for-elasticsearch:1.13.2
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi
+COPY --chown=elasticsearch:elasticsearch .ci/opendistro/elasticsearch.yml /usr/share/elasticsearch/config/
@@ -0,0 +1,12 @@
+ARG OPENSEARCH_VERSION
+FROM opensearchproject/opensearch:latest
+
+ARG opensearch_path=/usr/share/opensearch
+ARG opensearch_yml=$opensearch_path/config/opensearch.yml
+
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi
+
+COPY --chown=opensearch:opensearch .ci/opensearch/opensearch-run.sh $opensearch_path/
+
+CMD /usr/share/opensearch/opensearch-run.sh
@@ -0,0 +1,15 @@
+version: '3'
+
+services:
+
+  opensearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+    command: /usr/share/opensearch/opensearch-run.sh
+    ports:
+      - "9200:9200"
+    user: opensearch
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -ex
+
+/usr/share/opensearch/bin/opensearch -Ediscovery.type=single-node
@@ -48,7 +48,7 @@ END
 docker_pull_attempts=0
 until [ "$docker_pull_attempts" -ge 5 ]
 do
-   docker pull amazon/"$elasticsearch_container" && break
+   docker pull "$elasticsearch_container" && break
   docker_pull_attempts=$((docker_pull_attempts+1))
   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
   sleep 10
@@ -75,11 +75,12 @@ END
  local_detach="true"
  if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi

-
-  echo -e "\033[34;1mINFO: building odfe container\033[0m"
+  echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m"
+  echo 'cluster is' $CLUSTER
  docker build \
-    --file=.ci/Dockerfile.server \
-    --tag=odfe \
+    --file=.ci/$CLUSTER/Dockerfile \
+    --build-arg SECURE_INTEGRATION=false \
+    --tag=$CLUSTER \
    .

  echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
@@ -99,7 +100,8 @@ END
    --health-retries=20 \
    --health-timeout=2s \
    --rm \
-    odfe;
+    -d \
+    $CLUSTER;

  set +x
  if wait_for_container "$es_node_name" "$network_name"; then
@@ -14,7 +14,7 @@

 set -e

-echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m URL ${elasticsearch_url}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
@@ -5,10 +5,12 @@
 # - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products

 # Default environment variables
-export STACK_VERSION="${STACK_VERSION:=1.13.2}"
+export STACK_VERSION="${STACK_VERSION:=latest}"
 export TEST_SUITE="${TEST_SUITE:=oss}"
 export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
+export ELASTICSEARCH_URL_EXTENSION="${ELASTICSEARCH_URL_EXTENSION:=http}"
+export CLUSTER="${1:=opensearch}"

 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
@@ -20,9 +20,10 @@ jobs:
      fail-fast: false
      matrix:
        stack_version: ['7.10.2']
+        cluster: ["opendistro", "opensearch"]

    steps:
      - name: Checkout
        uses: actions/checkout@v2
-      - run: "./.ci/run-tests"
-        name: Integ ${{ matrix.stack_version }}
+      - run: "./.ci/run-tests ${{ matrix.cluster }} "
+        name: Integ ${{ matrix.stack_version }} ${{ matrix.cluster }}
@@ -40,13 +40,13 @@ def fetch_es_repo():

    repo_path = environ.get(
        "TEST_ES_REPO",
-        abspath(join(dirname(__file__), pardir, pardir, "elasticsearch")),
+        abspath(join(dirname(__file__), pardir, pardir, "opensearch")),
    )

    # no repo
    if not exists(repo_path) or not exists(join(repo_path, ".git")):
        subprocess.check_call(
-            "git clone https://github.com/elastic/elasticsearch %s" % repo_path,
+            "git clone https://github.com/opensearch-project/opensearch %s" % repo_path,
            shell=True,
        )

@@ -73,7 +73,8 @@ def fetch_es_repo():
    # fetch new commits to be sure...
    print("Fetching elasticsearch repo...")
    subprocess.check_call(
-        "cd %s && git fetch https://github.com/elastic/elasticsearch.git" % repo_path,
+        "cd %s && git fetch https://github.com/opensearch-project/opensearch.git"
+        % repo_path,
        shell=True,
    )
    # reset to the version from info()