diff --git a/.ci/opensearch/docker-compose.yml b/.ci/opensearch/docker-compose.yml deleted file mode 100755 index 3626f0e0..00000000 --- a/.ci/opensearch/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' - -services: - - opensearch: - build: - context: . - dockerfile: Dockerfile - args: - - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} - command: /usr/share/opensearch/opensearch-run.sh - ports: - - "9200:9200" - user: opensearch diff --git a/.ci/run-opensearch.sh b/.ci/run-opensearch.sh index 8395bbaa..2e0cf0d7 100755 --- a/.ci/run-opensearch.sh +++ b/.ci/run-opensearch.sh @@ -65,15 +65,19 @@ END if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m" - echo 'cluster is' $CLUSTER docker build \ --file=.ci/$CLUSTER/Dockerfile \ - --build-arg SECURE_INTEGRATION=false \ - --tag=$CLUSTER \ + --build-arg SECURE_INTEGRATION=$SECURE_INTEGRATION \ + --tag=$CLUSTER-secure-$SECURE_INTEGRATION \ . echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m" set -x + healthcmd="curl -vvv -s --fail http://localhost:9200/_cluster/health || exit 1" + if [[ "$SECURE_INTEGRATION" == "true" ]]; then + healthcmd="curl -vvv -s --insecure -u admin:admin --fail https://localhost:9200/_cluster/health || exit 1" + fi + docker run \ --name "$node_name" \ --network "$network_name" \ @@ -84,13 +88,13 @@ END --ulimit nofile=65536:65536 \ --ulimit memlock=-1:-1 \ --detach="$local_detach" \ - --health-cmd="curl -vvv -s --fail $opensearch_url/_cluster/health || exit 1" \ + --health-cmd="$(echo $healthcmd)" \ --health-interval=2s \ --health-retries=20 \ --health-timeout=2s \ --rm \ -d \ - $CLUSTER; + $CLUSTER-secure-$SECURE_INTEGRATION; set +x if wait_for_container "$opensearch_node_name" "$network_name"; then diff --git a/.ci/run-tests b/.ci/run-tests index eb49297a..d7bd3bb4 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -8,13 +8,19 @@ export TEST_SUITE="${TEST_SUITE:=oss}" export PYTHON_VERSION="${PYTHON_VERSION:=3.9}" export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}" -export OPENSEARCH_URL_EXTENSION="${OPENSEARCH_URL_EXTENSION:=http}" -export CLUSTER="${1:=opensearch}" +export CLUSTER="${1:-opensearch}" +export SECURE_INTEGRATION="${2:-false}" +if [[ "$SECURE_INTEGRATION" == "true" ]]; then + export OPENSEARCH_URL_EXTENSION="https" +else + export OPENSEARCH_URL_EXTENSION="http" +fi script_path=$(dirname $(realpath -s $0)) source $script_path/functions/imports.sh set -euo pipefail +echo -e "\033[1m>>>>> Start server container >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" DETACH=true bash .ci/run-opensearch.sh if [[ -n "$RUNSCRIPTS" ]]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bdcf7269..6ccddadb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -3,16 +3,17 @@ name: Integration tests on: [push, pull_request] jobs: - assemble: - name: Assemble + integration: + name: Integ runs-on: ubuntu-latest strategy: fail-fast: false matrix: cluster: ["opendistro", "opensearch"] + secured: ["true", "false"] steps: - name: Checkout uses: actions/checkout@v2 - - run: "./.ci/run-tests ${{ matrix.cluster }} " - name: Integ ${{ matrix.cluster }} + - run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }}" + name: Integ ${{ matrix.stack_version }} ${{ matrix.cluster }} secured=${{ matrix.secured }} diff --git a/opensearch/helpers/test.py b/opensearch/helpers/test.py index dd49242c..9e75bf0b 100644 --- a/opensearch/helpers/test.py +++ b/opensearch/helpers/test.py @@ -80,7 +80,7 @@ class OpenSearchTestCase(TestCase): def teardown_method(self, _): # Hidden indices expanded in wildcards in OpenSearch 7.7 expand_wildcards = ["open", "closed"] - if self.opensearch_version() >= (7, 7): + if self.opensearch_version() >= (1, 0): expand_wildcards.append("hidden") self.client.indices.delete( diff --git a/test_opensearch/run_tests.py b/test_opensearch/run_tests.py index 6798baee..18144f47 100755 --- a/test_opensearch/run_tests.py +++ b/test_opensearch/run_tests.py @@ -103,6 +103,10 @@ def run_all(argv=None): "-vv", ] + secured = False + if environ.get("OPENSEARCH_URL", "").startswith("https://"): + secured = True + ignores = [] # Python 3.6+ is required for async if sys.version_info < (3, 6): @@ -113,18 +117,34 @@ def run_all(argv=None): ignores.extend( [ "test_opensearch/test_server/", + "test_opensearch/test_server_secured/", "test_opensearch/test_async/test_server/", ] ) - if ignores: - argv.extend(["--ignore=%s" % ignore for ignore in ignores]) - # Jenkins, only run server tests + # Jenkins/Github actions, only run server tests if environ.get("TEST_TYPE") == "server": test_dir = abspath(dirname(__file__)) - argv.append(join(test_dir, "test_server")) - if sys.version_info >= (3, 6): - argv.append(join(test_dir, "test_async/test_server")) + if secured: + argv.append(join(test_dir, "test_server_secured")) + ignores.extend( + [ + "test_opensearch/test_server/", + "test_opensearch/test_async/test_server/", + ] + ) + else: + argv.append(join(test_dir, "test_server")) + if sys.version_info >= (3, 6): + argv.append(join(test_dir, "test_async/test_server")) + ignores.extend( + [ + "test_opensearch/test_server_secured/", + ] + ) + + if ignores: + argv.extend(["--ignore=%s" % ignore for ignore in ignores]) # Not in CI, run all tests specified. else: diff --git a/test_opensearch/test_server_secured/__init__.py b/test_opensearch/test_server_secured/__init__.py new file mode 100644 index 00000000..6c0097cd --- /dev/null +++ b/test_opensearch/test_server_secured/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. diff --git a/test_opensearch/test_server_secured/test_clients.py b/test_opensearch/test_server_secured/test_clients.py new file mode 100644 index 00000000..38b551de --- /dev/null +++ b/test_opensearch/test_server_secured/test_clients.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + +from unittest import TestCase + +from opensearch import OpenSearch +from opensearch.helpers.test import OPENSEARCH_URL + + +class TestSecurity(TestCase): + def test_security(self): + client = OpenSearch( + OPENSEARCH_URL, + http_auth=("admin", "admin"), + verify_certs=False, + ) + + info = client.info() + self.assertNotEqual(info["version"]["number"], "") + self.assertNotEqual(info["tagline"], "") + self.assertTrue( + "build_flavor" in info["version"] or "distribution" in info["version"] + )