Support integ test with security plugin enabled

Removed a docker-compose.yml file which wasn't being used.

Signed-off-by: Rushi Agrawal <rushi.agr@gmail.com>

.ci/opensearch/docker-compose.yml

@@ -1,15 +0,0 @@
-version: '3'
-
-services:
-
-  opensearch:
-    build:
-      context: .
-      dockerfile: Dockerfile
-      args:
-        - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
-        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
-    command: /usr/share/opensearch/opensearch-run.sh
-    ports:
-      - "9200:9200"
-    user: opensearch

.ci/run-opensearch.sh

@@ -65,15 +65,19 @@ END
   if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
 
   echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m"
-  echo 'cluster is' $CLUSTER
   docker build \
     --file=.ci/$CLUSTER/Dockerfile \
-    --build-arg SECURE_INTEGRATION=false \
-    --tag=$CLUSTER \
+    --build-arg SECURE_INTEGRATION=$SECURE_INTEGRATION \
+    --tag=$CLUSTER-secure-$SECURE_INTEGRATION \
     .
 
   echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
   set -x
+  healthcmd="curl -vvv -s --fail http://localhost:9200/_cluster/health || exit 1"
+  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    healthcmd="curl -vvv -s --insecure -u admin:admin --fail https://localhost:9200/_cluster/health || exit 1"
+  fi
+
   docker run \
     --name "$node_name" \
     --network "$network_name" \
@@ -84,13 +88,13 @@ END
     --ulimit nofile=65536:65536 \
     --ulimit memlock=-1:-1 \
     --detach="$local_detach" \
-    --health-cmd="curl -vvv -s --fail $opensearch_url/_cluster/health || exit 1" \
+    --health-cmd="$(echo $healthcmd)" \
     --health-interval=2s \
     --health-retries=20 \
     --health-timeout=2s \
     --rm \
     -d \
-    $CLUSTER;
+    $CLUSTER-secure-$SECURE_INTEGRATION;
 
   set +x
   if wait_for_container "$opensearch_node_name" "$network_name"; then

.ci/run-tests

@@ -8,13 +8,19 @@
 export TEST_SUITE="${TEST_SUITE:=oss}"
 export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
-export OPENSEARCH_URL_EXTENSION="${OPENSEARCH_URL_EXTENSION:=http}"
-export CLUSTER="${1:=opensearch}"
+export CLUSTER="${1:-opensearch}"
+export SECURE_INTEGRATION="${2:-false}"
+if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    export OPENSEARCH_URL_EXTENSION="https"
+else
+    export OPENSEARCH_URL_EXTENSION="http"
+fi
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 
+echo -e "\033[1m>>>>> Start server container >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 DETACH=true bash .ci/run-opensearch.sh
 
 if [[ -n "$RUNSCRIPTS" ]]; then

.github/workflows/integration.yml

@@ -3,16 +3,17 @@ name: Integration tests
 on: [push, pull_request]
 
 jobs:
-  assemble:
-    name: Assemble
+  integration:
+    name: Integ
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         cluster: ["opendistro", "opensearch"]
+        secured: ["true", "false"]
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - run: "./.ci/run-tests ${{ matrix.cluster }} "
-        name: Integ ${{ matrix.cluster }}
+      - run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }}"
+        name: Integ ${{ matrix.stack_version }} ${{ matrix.cluster }} secured=${{ matrix.secured }}

opensearch/helpers/test.py

@@ -80,7 +80,7 @@ class OpenSearchTestCase(TestCase):
     def teardown_method(self, _):
         # Hidden indices expanded in wildcards in OpenSearch 7.7
         expand_wildcards = ["open", "closed"]
-        if self.opensearch_version() >= (7, 7):
+        if self.opensearch_version() >= (1, 0):
             expand_wildcards.append("hidden")
 
         self.client.indices.delete(

test_opensearch/run_tests.py

@@ -103,6 +103,10 @@ def run_all(argv=None):
             "-vv",
         ]
 
+        secured = False
+        if environ.get("OPENSEARCH_URL", "").startswith("https://"):
+            secured = True
+
         ignores = []
         # Python 3.6+ is required for async
         if sys.version_info < (3, 6):
@@ -113,18 +117,34 @@ def run_all(argv=None):
             ignores.extend(
                 [
                     "test_opensearch/test_server/",
+                    "test_opensearch/test_server_secured/",
                     "test_opensearch/test_async/test_server/",
                 ]
             )
-        if ignores:
-            argv.extend(["--ignore=%s" % ignore for ignore in ignores])
 
-        # Jenkins, only run server tests
+        # Jenkins/Github actions, only run server tests
         if environ.get("TEST_TYPE") == "server":
             test_dir = abspath(dirname(__file__))
-            argv.append(join(test_dir, "test_server"))
-            if sys.version_info >= (3, 6):
-                argv.append(join(test_dir, "test_async/test_server"))
+            if secured:
+                argv.append(join(test_dir, "test_server_secured"))
+                ignores.extend(
+                    [
+                        "test_opensearch/test_server/",
+                        "test_opensearch/test_async/test_server/",
+                    ]
+                )
+            else:
+                argv.append(join(test_dir, "test_server"))
+                if sys.version_info >= (3, 6):
+                    argv.append(join(test_dir, "test_async/test_server"))
+                ignores.extend(
+                    [
+                        "test_opensearch/test_server_secured/",
+                    ]
+                )
+
+        if ignores:
+            argv.extend(["--ignore=%s" % ignore for ignore in ignores])
 
         # Not in CI, run all tests specified.
         else:

test_opensearch/test_server_secured/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.

test_opensearch/test_server_secured/test_clients.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+from unittest import TestCase
+
+from opensearch import OpenSearch
+from opensearch.helpers.test import OPENSEARCH_URL
+
+
+class TestSecurity(TestCase):
+    def test_security(self):
+        client = OpenSearch(
+            OPENSEARCH_URL,
+            http_auth=("admin", "admin"),
+            verify_certs=False,
+        )
+
+        info = client.info()
+        self.assertNotEqual(info["version"]["number"], "")
+        self.assertNotEqual(info["tagline"], "")
+        self.assertTrue(
+            "build_flavor" in info["version"] or "distribution" in info["version"]
+        )