actions/opensearch-pyd
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Added samples, benchmarks and docs for nox format. (#556)

Browse Source 
* Added samples for nox format.

Signed-off-by: dblock <dblock@amazon.com>

* Added space after #!/usr/bin/env python.

Signed-off-by: dblock <dblock@amazon.com>

* Added benchmarks and docs.

Signed-off-by: dblock <dblock@amazon.com>

---------

Signed-off-by: dblock <dblock@amazon.com>
This commit is contained in:
Daniel (dB.) Doubrovkine
2023-10-26 19:55:50 -07:00
committed by GitHub
parent 0da60b2623
commit bcfef113c4
28 changed files with 745 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -13,6 +13,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added `remote store` client APIs ([#552](https://github.com/opensearch-project/opensearch-py/pull/552))
- Added `nox -rs generate` ([#554](https://github.com/opensearch-project/opensearch-py/pull/554))
- Added a utf-8 header to all .py files ([#557](https://github.com/opensearch-project/opensearch-py/pull/557))
- Added `samples`, `benchmarks` and `docs` to `nox -rs format` ([#556](https://github.com/opensearch-project/opensearch-py/pull/556))
### Changed
- Generate `tasks` client from API specs ([#508](https://github.com/opensearch-project/opensearch-py/pull/508))
- Generate `ingest` client from API specs ([#513](https://github.com/opensearch-project/opensearch-py/pull/513))
benchmarks/bench_async.py
+4 -1
View File
@@ -6,6 +6,9 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import asyncio
import uuid
@@ -25,7 +28,7 @@ async def index_records(client, item_count):
client.index(
index=index_name,
body={
"title": f"Moneyball",
"title": "Moneyball",
"director": "Bennett Miller",
"year": "2011",
},
benchmarks/bench_info_sync.py
+5 -1
View File
@@ -6,6 +6,10 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import logging
import sys
@@ -36,7 +40,7 @@ def get_info(client, request_count):
tt = 0
for n in range(request_count):
start = time.time() * 1000
rc = client.info()
client.info()
total_time = time.time() * 1000 - start
tt += total_time
return tt
benchmarks/bench_sync.py
+5 -3
View File
@@ -6,8 +6,13 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import json
import logging
import sys
import time
import uuid
@@ -21,9 +26,6 @@ auth = ("admin", "admin")
index_name = "test-index-sync"
item_count = 1000
import logging
import sys
root = logging.getLogger()
# root.setLevel(logging.DEBUG)
# logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG)
benchmarks/bench_sync_async.py
+4
View File
@@ -6,6 +6,10 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import bench_async
import bench_sync
benchmarks/thread_with_return_value.py
+4 -2
View File
@@ -1,11 +1,13 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
from threading import Thread
docs/source/conf.py
+9
View File
@@ -1,3 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
noxfile.py
+3
View File
@@ -34,6 +34,9 @@ SOURCE_FILES = (
"opensearchpy/",
"test_opensearchpy/",
"utils/",
"samples/",
"benchmarks/",
"docs/",
)
samples/advanced_index_actions/advanced_index_actions_sample.py
+57 -35
View File
@@ -1,6 +1,17 @@
from opensearchpy import OpenSearch
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import time
from opensearchpy import OpenSearch
# For cleaner output, comment in the two lines below to disable warnings and informational messages
# import urllib3
@@ -10,73 +21,84 @@ import time
def test_opensearch_examples():
# Set up
client = OpenSearch(
hosts=['https://localhost:9200'],
hosts=["https://localhost:9200"],
use_ssl=True,
verify_certs=False,
http_auth=('admin', 'admin')
http_auth=("admin", "admin"),
)
client.indices.create(index='movies')
client.indices.create(index="movies")
print("'movies' index created!")
# Test Clear Index Cache
client.indices.clear_cache(index='movies')
client.indices.clear_cache(index="movies")
print("Cache for 'movies' index cleared!")
client.indices.clear_cache(index='movies', query=True)
client.indices.clear_cache(index="movies", query=True)
print("Query cache for 'movies' index cleared!")
client.indices.clear_cache(index='movies', fielddata=True, request=True)
client.indices.clear_cache(index="movies", fielddata=True, request=True)
print("Field data and request cache for 'movies' index cleared!")
# Test Flush Index
client.indices.flush(index='movies')
client.indices.flush(index="movies")
print("'movies' index flushed!")
# Test Refresh Index
client.indices.refresh(index='movies')
client.indices.refresh(index="movies")
print("'movies' index refreshed!")
# Test Close or Open Index
client.indices.close(index='movies')
client.indices.close(index="movies")
print("'movies' index closed!")
time.sleep(2) # add sleep to ensure the index has time to close
client.indices.open(index='movies')
client.indices.open(index="movies")
print("'movies' index opened!")
# Test Force Merge Index
client.indices.forcemerge(index='movies')
client.indices.forcemerge(index="movies")
print("'movies' index force merged!")
# Test Clone
client.indices.put_settings(index='movies', body={'index': {'blocks': {'write': True}}})
client.indices.put_settings(
index="movies", body={"index": {"blocks": {"write": True}}}
)
print("Write operations blocked for 'movies' index!")
time.sleep(2)
client.indices.clone(index='movies', target='movies_clone')
client.indices.clone(index="movies", target="movies_clone")
print("'movies' index cloned to 'movies_clone'!")
client.indices.put_settings(index='movies', body={'index': {'blocks': {'write': False}}})
client.indices.put_settings(
index="movies", body={"index": {"blocks": {"write": False}}}
)
print("Write operations enabled for 'movies' index!")
# Test Split
# Test Split
client.indices.create(
index='books',
body={'settings': {
'index': {'number_of_shards': 5, 'number_of_routing_shards': 30, 'blocks': {'write': True}}}}
index="books",
body={
"settings": {
"index": {
"number_of_shards": 5,
"number_of_routing_shards": 30,
"blocks": {"write": True},
}
}
},
)
print("'books' index created!")
time.sleep(2) # add sleep to ensure the index has time to become read-only
client.indices.split(
index='books',
target='bigger_books',
body={'settings': {'index': {'number_of_shards': 10 }}}
index="books",
target="bigger_books",
body={"settings": {"index": {"number_of_shards": 10}}},
)
print("'books' index split into 'bigger_books'!")
client.indices.put_settings(index='books', body={'index': {'blocks': {'write': False}}})
client.indices.put_settings(
index="books", body={"index": {"blocks": {"write": False}}}
)
print("Write operations enabled for 'books' index!")
# Cleanup
client.indices.delete(index=['movies', 'books', 'movies_clone', 'bigger_books'])
client.indices.delete(index=["movies", "books", "movies_clone", "bigger_books"])
print("All indices deleted!")
if __name__ == "__main__":
test_opensearch_examples()
test_opensearch_examples()
samples/aws/search-requests.py
+31 -32
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
@@ -9,62 +11,59 @@
# GitHub history for details.
import logging
from os import environ
from time import sleep
from urllib.parse import urlparse
from boto3 import Session
from opensearchpy import RequestsAWSV4SignerAuth, OpenSearch, RequestsHttpConnection
from opensearchpy import OpenSearch, RequestsAWSV4SignerAuth, RequestsHttpConnection
# verbose logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
# cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com
url = urlparse(environ['ENDPOINT'])
region = environ.get('AWS_REGION', 'us-east-1')
service = environ.get('SERVICE', 'es')
url = urlparse(environ["ENDPOINT"])
region = environ.get("AWS_REGION", "us-east-1")
service = environ.get("SERVICE", "es")
credentials = Session().get_credentials()
auth = RequestsAWSV4SignerAuth(credentials, region, service)
client = OpenSearch(
hosts=[{
'host': url.netloc,
'port': url.port or 443
}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
timeout=30
hosts=[{"host": url.netloc, "port": url.port or 443}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
timeout=30,
)
# TODO: remove when OpenSearch Serverless adds support for /
if service == 'es':
info = client.info()
print(f"{info['version']['distribution']}: {info['version']['number']}")
if service == "es":
info = client.info()
print(f"{info['version']['distribution']}: {info['version']['number']}")
# create an index
index = 'movies'
index = "movies"
client.indices.create(index=index)
try:
# index data
document = {'director': 'Bennett Miller', 'title': 'Moneyball', 'year': 2011}
client.index(index=index, body=document, id='1')
# index data
document = {"director": "Bennett Miller", "title": "Moneyball", "year": 2011}
client.index(index=index, body=document, id="1")
# wait for the document to index
sleep(1)
# wait for the document to index
sleep(1)
# search for the document
results = client.search(body={'query': {'match': {'director': 'miller'}}})
for hit in results['hits']['hits']:
print(hit['_source'])
# search for the document
results = client.search(body={"query": {"match": {"director": "miller"}}})
for hit in results["hits"]["hits"]:
print(hit["_source"])
# delete the document
client.delete(index=index, id='1')
# delete the document
client.delete(index=index, id="1")
finally:
# delete the index
client.indices.delete(index=index)
# delete the index
client.indices.delete(index=index)
samples/aws/search-urllib3.py
+31 -32
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
@@ -9,62 +11,59 @@
# GitHub history for details.
import logging
from os import environ
from time import sleep
from urllib.parse import urlparse
from boto3 import Session
from opensearchpy import Urllib3AWSV4SignerAuth, OpenSearch, Urllib3HttpConnection
from opensearchpy import OpenSearch, Urllib3AWSV4SignerAuth, Urllib3HttpConnection
# verbose logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
# cluster endpoint, for example: my-test-domain.us-east-1.es.amazonaws.com
url = urlparse(environ['ENDPOINT'])
region = environ.get('AWS_REGION', 'us-east-1')
service = environ.get('SERVICE', 'es')
url = urlparse(environ["ENDPOINT"])
region = environ.get("AWS_REGION", "us-east-1")
service = environ.get("SERVICE", "es")
credentials = Session().get_credentials()
auth = Urllib3AWSV4SignerAuth(credentials, region, service)
client = OpenSearch(
hosts=[{
'host': url.netloc,
'port': url.port or 443
}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=Urllib3HttpConnection,
timeout=30
hosts=[{"host": url.netloc, "port": url.port or 443}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=Urllib3HttpConnection,
timeout=30,
)
# TODO: remove when OpenSearch Serverless adds support for /
if service == 'es':
info = client.info()
print(f"{info['version']['distribution']}: {info['version']['number']}")
if service == "es":
info = client.info()
print(f"{info['version']['distribution']}: {info['version']['number']}")
# create an index
index = 'movies'
index = "movies"
client.indices.create(index=index)
try:
# index data
document = {'director': 'Bennett Miller', 'title': 'Moneyball', 'year': 2011}
client.index(index=index, body=document, id='1')
# index data
document = {"director": "Bennett Miller", "title": "Moneyball", "year": 2011}
client.index(index=index, body=document, id="1")
# wait for the document to index
sleep(1)
# wait for the document to index
sleep(1)
# search for the document
results = client.search(body={'query': {'match': {'director': 'miller'}}})
for hit in results['hits']['hits']:
print(hit['_source'])
# search for the document
results = client.search(body={"query": {"match": {"director": "miller"}}})
for hit in results["hits"]["hits"]:
print(hit["_source"])
# delete the document
client.delete(index=index, id='1')
# delete the document
client.delete(index=index, id="1")
finally:
# delete the index
client.indices.delete(index=index)
# delete the index
client.indices.delete(index=index)
samples/bulk/bulk-array.py
+20 -23
View File
@@ -6,55 +6,53 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import json
from opensearchpy import OpenSearch
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
index_name = "my-index"
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"mappings":{
"mappings": {
"properties": {
"value": {
"type": "float"
},
"value": {"type": "float"},
}
}
}
},
)
# index data
data = []
for i in range(100):
data.append({ "index": {"_index": index_name, "_id": i }})
data.append({ "value": i })
data.append({"index": {"_index": index_name, "_id": i}})
data.append({"value": i})
rc = client.bulk(data)
if rc["errors"]:
print(f"There were errors:")
print("There were errors:")
for item in rc["items"]:
print(f"{item['index']['status']}: {item['index']['error']['type']}")
else:
@@ -62,4 +60,3 @@ else:
# delete index
client.indices.delete(index=index_name)
samples/bulk/bulk-helpers.py
+18 -21
View File
@@ -6,54 +6,51 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import json
from opensearchpy import OpenSearch, helpers
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
index_name = "my-index"
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"mappings":{
"mappings": {
"properties": {
"value": {
"type": "float"
},
"value": {"type": "float"},
}
}
}
},
)
# index data
data = []
for i in range(100):
data.append({ "_index": index_name, "_id": i, "value": i })
data.append({"_index": index_name, "_id": i, "value": i})
rc = helpers.bulk(client, data)
print(f"Bulk-inserted {rc[0]} items.")
# delete index
client.indices.delete(index=index_name)
samples/bulk/bulk-ld.py
+22 -24
View File
@@ -6,55 +6,54 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import json
import os
from opensearchpy import OpenSearch
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
index_name = "my-index"
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"mappings":{
"mappings": {
"properties": {
"value": {
"type": "float"
},
"value": {"type": "float"},
}
}
}
},
)
# index data
data = ''
data = ""
for i in range(100):
data += json.dumps({ "index": {"_index": index_name, "_id": i }}) + "\n"
data += json.dumps({ "value": i }) + "\n"
data += json.dumps({"index": {"_index": index_name, "_id": i}}) + "\n"
data += json.dumps({"value": i}) + "\n"
rc = client.bulk(data)
if rc["errors"]:
print(f"There were errors:")
print("There were errors:")
for item in rc["items"]:
print(f"{item['index']['status']}: {item['index']['error']['type']}")
else:
@@ -62,4 +61,3 @@ else:
# delete index
client.indices.delete(index=index_name)
samples/hello/hello-async.py
+59 -71
View File
@@ -6,103 +6,91 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import asyncio
from opensearchpy import AsyncOpenSearch
async def main():
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = AsyncOpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
try:
info = await client.info()
print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!")
# create an index
index_name = 'test-index'
index_body = {
'settings': {
'index': {
'number_of_shards': 4
}
}
}
if not await client.indices.exists(index=index_name):
await client.indices.create(
index_name,
body=index_body
info = await client.info()
print(
f"Welcome to {info['version']['distribution']} {info['version']['number']}!"
)
# add some documents to the index, asynchronously
await asyncio.gather(*[
client.index(
index = index_name,
body = {
'title': f"Moneyball {i}",
'director': 'Bennett Miller',
'year': '2011'
},
id = i
) for i in range(10)
])
# create an index
# refresh the index
await client.indices.refresh(index=index_name)
index_name = "test-index"
# search for a document
q = 'miller'
index_body = {"settings": {"index": {"number_of_shards": 4}}}
query = {
'size': 5,
'query': {
'multi_match': {
'query': q,
'fields': ['title^2', 'director']
}
if not await client.indices.exists(index=index_name):
await client.indices.create(index_name, body=index_body)
# add some documents to the index, asynchronously
await asyncio.gather(
*[
client.index(
index=index_name,
body={
"title": f"Moneyball {i}",
"director": "Bennett Miller",
"year": "2011",
},
id=i,
)
for i in range(10)
]
)
# refresh the index
await client.indices.refresh(index=index_name)
# search for a document
q = "miller"
query = {
"size": 5,
"query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}},
}
}
results = await client.search(
body = query,
index = index_name
)
results = await client.search(body=query, index=index_name)
for hit in results["hits"]["hits"]:
print(hit)
# delete the documents
await asyncio.gather(*[
client.delete(
index = index_name,
id = i
) for i in range(10)
])
for hit in results["hits"]["hits"]:
print(hit)
# delete the index
await client.indices.delete(
index = index_name
)
# delete the documents
await asyncio.gather(
*[client.delete(index=index_name, id=i) for i in range(10)]
)
# delete the index
await client.indices.delete(index=index_name)
finally:
await client.close()
finally:
await client.close()
if __name__ == "__main__":
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main())
loop.close()
samples/hello/hello.py
+23 -50
View File
@@ -6,21 +6,25 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
from opensearchpy import OpenSearch
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
info = client.info()
@@ -28,76 +32,45 @@ print(f"Welcome to {info['version']['distribution']} {info['version']['number']}
# create an index
index_name = 'test-index'
index_name = "test-index"
index_body = {
'settings': {
'index': {
'number_of_shards': 4
}
}
}
index_body = {"settings": {"index": {"number_of_shards": 4}}}
response = client.indices.create(
index_name,
body=index_body
)
response = client.indices.create(index_name, body=index_body)
print(response)
# add a document to the index
document = {
'title': 'Moneyball',
'director': 'Bennett Miller',
'year': '2011'
}
document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"}
id = '1'
id = "1"
response = client.index(
index = index_name,
body = document,
id = id,
refresh = True
)
response = client.index(index=index_name, body=document, id=id, refresh=True)
print(response)
# search for a document
q = 'miller'
q = "miller"
query = {
'size': 5,
'query': {
'multi_match': {
'query': q,
'fields': ['title^2', 'director']
}
}
"size": 5,
"query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}},
}
response = client.search(
body = query,
index = index_name
)
response = client.search(body=query, index=index_name)
print(response)
# delete the document
response = client.delete(
index = index_name,
id = id
)
response = client.delete(index=index_name, id=id)
print(response)
# delete the index
response = client.indices.delete(
index = index_name
)
response = client.indices.delete(index=index_name)
print(response)
samples/index_template/index_template_sample.py
+88 -102
View File
@@ -1,143 +1,129 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
from opensearchpy import OpenSearch
# Create a client instance
client = OpenSearch(
hosts=['https://localhost:9200'],
use_ssl=True,
verify_certs=False,
http_auth=('admin', 'admin')
hosts=["https://localhost:9200"],
use_ssl=True,
verify_certs=False,
http_auth=("admin", "admin"),
)
# You can create an index template to define default settings and mappings for indices of certain patterns. The following example creates an index template named `books` with default settings and mappings for indices of the `books-*` pattern:
client.indices.put_index_template(
name='books',
body={
'index_patterns': ['books-*'],
'priority': 1,
'template': {
'settings': {
'index': {
'number_of_shards': 3,
'number_of_replicas': 0
}
name="books",
body={
"index_patterns": ["books-*"],
"priority": 1,
"template": {
"settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}},
"mappings": {
"properties": {
"title": {"type": "text"},
"author": {"type": "text"},
"published_on": {"type": "date"},
"pages": {"type": "integer"},
}
},
},
},
'mappings': {
'properties': {
'title': { 'type': 'text' },
'author': { 'type': 'text' },
'published_on': { 'type': 'date' },
'pages': { 'type': 'integer' }
}
}
}
}
)
# Now, when you create an index that matches the `books-*` pattern, OpenSearch will automatically apply the template's settings and mappings to the index. Let's create an index named books-nonfiction and verify that its settings and mappings match those of the template:
client.indices.create(index='books-nonfiction')
print(client.indices.get(index='books-nonfiction'))
client.indices.create(index="books-nonfiction")
print(client.indices.get(index="books-nonfiction"))
# If multiple index templates match the index's name, OpenSearch will apply the template with the highest `priority`. The following example creates two index templates named `books-*` and `books-fiction-*` with different settings:
client.indices.put_index_template(
name='books',
body={
'index_patterns': ['books-*'],
'priority': 1,
'template': {
'settings': {
'index': {
'number_of_shards': 3,
'number_of_replicas': 0
}
}
}
}
name="books",
body={
"index_patterns": ["books-*"],
"priority": 1,
"template": {
"settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}}
},
},
)
client.indices.put_index_template(
name='books-fiction',
body={
'index_patterns': ['books-fiction-*'],
'priority': 2,
'template': {
'settings': {
'index': {
'number_of_shards': 1,
'number_of_replicas': 1
}
}
}
}
name="books-fiction",
body={
"index_patterns": ["books-fiction-*"],
"priority": 2,
"template": {
"settings": {"index": {"number_of_shards": 1, "number_of_replicas": 1}}
},
},
)
# # Test multiple index templates
client.indices.create(index='books-fiction-romance')
print(client.indices.get(index='books-fiction-romance'))
client.indices.create(index="books-fiction-romance")
print(client.indices.get(index="books-fiction-romance"))
# Composable index templates are a new type of index template that allow you to define multiple component templates and compose them into a final template. The following example creates a component template named `books_mappings` with default mappings for indices of the `books-*` and `books-fiction-*` patterns:
client.cluster.put_component_template(
name='books_mappings',
body={
'template': {
'mappings': {
'properties': {
'title': { 'type': 'text' },
'author': { 'type': 'text' },
'published_on': { 'type': 'date' },
'pages': { 'type': 'integer' }
}
}
}
}
name="books_mappings",
body={
"template": {
"mappings": {
"properties": {
"title": {"type": "text"},
"author": {"type": "text"},
"published_on": {"type": "date"},
"pages": {"type": "integer"},
}
}
}
},
)
client.indices.put_index_template(
name='books',
body={
'index_patterns': ['books-*'],
'composed_of': ['books_mappings'],
'priority': 4,
'template': {
'settings': {
'index': {
'number_of_shards': 3,
'number_of_replicas': 0
}
}
}
}
name="books",
body={
"index_patterns": ["books-*"],
"composed_of": ["books_mappings"],
"priority": 4,
"template": {
"settings": {"index": {"number_of_shards": 3, "number_of_replicas": 0}}
},
},
)
client.indices.put_index_template(
name='books-fiction',
body={
'index_patterns': ['books-fiction-*'],
'composed_of': ['books_mappings'],
'priority': 5,
'template': {
'settings': {
'index': {
'number_of_shards': 1,
'number_of_replicas': 1
}
}
}
}
name="books-fiction",
body={
"index_patterns": ["books-fiction-*"],
"composed_of": ["books_mappings"],
"priority": 5,
"template": {
"settings": {"index": {"number_of_shards": 1, "number_of_replicas": 1}}
},
},
)
# Test composable index templates
client.indices.create(index='books-fiction-horror')
print(client.indices.get(index='books-fiction-horror'))
client.indices.create(index="books-fiction-horror")
print(client.indices.get(index="books-fiction-horror"))
# Get an index template
print(client.indices.get_index_template(name='books'))
print(client.indices.get_index_template(name="books"))
# Delete an index template
client.indices.delete_index_template(name='books')
client.indices.delete_index_template(name="books")
# Cleanup
client.indices.delete(index='books-*')
client.indices.delete_index_template(name='books-fiction')
client.cluster.delete_component_template(name='books_mappings')
client.indices.delete(index="books-*")
client.indices.delete_index_template(name="books-fiction")
client.cluster.delete_component_template(name="books_mappings")
samples/json/hello-async.py
+58 -53
View File
@@ -6,86 +6,91 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import asyncio
from opensearchpy import AsyncOpenSearch
async def main():
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = AsyncOpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
try:
info = await client.transport.perform_request('GET', '/')
print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!")
info = await client.transport.perform_request("GET", "/")
print(
f"Welcome to {info['version']['distribution']} {info['version']['number']}!"
)
# create an index
# create an index
index_name = 'movies'
index_name = "movies"
index_body = {
'settings': {
'index': {
'number_of_shards': 4
}
index_body = {"settings": {"index": {"number_of_shards": 4}}}
print(
await client.transport.perform_request(
"PUT", f"/{index_name}", body=index_body
)
)
# add a document to the index
document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"}
id = "1"
print(
await client.transport.perform_request(
"PUT", f"/{index_name}/_doc/{id}?refresh=true", body=document
)
)
# search for a document
q = "miller"
query = {
"size": 5,
"query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}},
}
}
print(await client.transport.perform_request("PUT", f"/{index_name}", body=index_body))
print(
await client.transport.perform_request(
"POST", f"/{index_name}/_search", body=query
)
)
# add a document to the index
# delete the document
document = {
'title': 'Moneyball',
'director': 'Bennett Miller',
'year': '2011'
}
print(
await client.transport.perform_request("DELETE", f"/{index_name}/_doc/{id}")
)
id = '1'
# delete the index
print(await client.transport.perform_request("PUT", f"/{index_name}/_doc/{id}?refresh=true", body = document))
print(await client.transport.perform_request("DELETE", f"/{index_name}"))
# search for a document
finally:
await client.close()
q = 'miller'
query = {
'size': 5,
'query': {
'multi_match': {
'query': q,
'fields': ['title^2', 'director']
}
}
}
print(await client.transport.perform_request("POST", f"/{index_name}/_search", body = query))
# delete the document
print(await client.transport.perform_request("DELETE", f"/{index_name}/_doc/{id}"))
# delete the index
print(await client.transport.perform_request("DELETE", f"/{index_name}"))
finally:
await client.close()
if __name__ == "__main__":
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main())
loop.close()
samples/json/hello.py
+25 -32
View File
@@ -6,67 +6,60 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
from opensearchpy import OpenSearch
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
info = client.transport.perform_request('GET', '/')
info = client.transport.perform_request("GET", "/")
print(f"Welcome to {info['version']['distribution']} {info['version']['number']}!")
# create an index
index_name = 'movies'
index_name = "movies"
index_body = {
'settings': {
'index': {
'number_of_shards': 4
}
}
}
index_body = {"settings": {"index": {"number_of_shards": 4}}}
print(client.transport.perform_request("PUT", f"/{index_name}", body=index_body))
# add a document to the index
document = {
'title': 'Moneyball',
'director': 'Bennett Miller',
'year': '2011'
}
document = {"title": "Moneyball", "director": "Bennett Miller", "year": "2011"}
id = '1'
id = "1"
print(client.transport.perform_request("PUT", f"/{index_name}/_doc/{id}?refresh=true", body = document))
print(
client.transport.perform_request(
"PUT", f"/{index_name}/_doc/{id}?refresh=true", body=document
)
)
# search for a document
q = 'miller'
q = "miller"
query = {
'size': 5,
'query': {
'multi_match': {
'query': q,
'fields': ['title^2', 'director']
}
}
"size": 5,
"query": {"multi_match": {"query": q, "fields": ["title^2", "director"]}},
}
print(client.transport.perform_request("POST", f"/{index_name}/_search", body = query))
print(client.transport.perform_request("POST", f"/{index_name}/_search", body=query))
# delete the document
samples/knn/knn-async-basics.py
+34 -34
View File
@@ -6,29 +6,31 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import asyncio
import os
import random
from opensearchpy import AsyncOpenSearch, AsyncHttpConnection, helpers
from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, helpers
async def main():
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = AsyncOpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
connection_class=AsyncHttpConnection,
ssl_show_warn = False
ssl_show_warn=False,
)
# check whether an index exists
@@ -36,34 +38,32 @@ async def main():
dimensions = 5
if not await client.indices.exists(index_name):
await client.indices.create(index_name,
await client.indices.create(
index_name,
body={
"settings":{
"index.knn": True
},
"mappings":{
"settings": {"index.knn": True},
"mappings": {
"properties": {
"values": {
"type": "knn_vector",
"dimension": dimensions
},
"values": {"type": "knn_vector", "dimension": dimensions},
}
}
}
},
},
)
# index data
vectors = []
for i in range(10):
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append({
"_index": index_name,
"_id": i,
"values": vec,
})
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append(
{
"_index": index_name,
"_id": i,
"values": vec,
}
)
# bulk index
await helpers.async_bulk(client, vectors)
@@ -72,8 +72,8 @@ async def main():
# search
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
print(f"Searching for {vec} ...")
search_query = {"query": {"knn": {"values": {"vector": vec, "k": 3}}}}
@@ -86,9 +86,9 @@ async def main():
await client.close()
if __name__ == "__main__":
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main())
loop.close()
samples/knn/knn-basics.py
+31 -33
View File
@@ -6,6 +6,10 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import random
@@ -14,19 +18,16 @@ from opensearchpy import OpenSearch, helpers
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
@@ -34,34 +35,32 @@ index_name = "my-index"
dimensions = 5
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"settings":{
"index.knn": True
},
"mappings":{
"settings": {"index.knn": True},
"mappings": {
"properties": {
"values": {
"type": "knn_vector",
"dimension": dimensions
},
"values": {"type": "knn_vector", "dimension": dimensions},
}
}
}
},
},
)
# index data
vectors = []
for i in range(10):
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append({
"_index": index_name,
"_id": i,
"values": vec,
})
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append(
{
"_index": index_name,
"_id": i,
"values": vec,
}
)
# bulk index
helpers.bulk(client, vectors)
@@ -70,8 +69,8 @@ client.indices.refresh(index=index_name)
# search
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
print(f"Searching for {vec} ...")
search_query = {"query": {"knn": {"values": {"vector": vec, "k": 3}}}}
@@ -81,4 +80,3 @@ for hit in results["hits"]["hits"]:
# delete index
client.indices.delete(index=index_name)
samples/knn/knn-boolean-filter.py
+34 -52
View File
@@ -6,6 +6,10 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import random
@@ -14,19 +18,16 @@ from opensearchpy import OpenSearch, helpers
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
@@ -34,38 +35,34 @@ index_name = "my-index"
dimensions = 5
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"settings":{
"index.knn": True
},
"mappings":{
"settings": {"index.knn": True},
"mappings": {
"properties": {
"values": {
"type": "knn_vector",
"dimension": dimensions
},
"values": {"type": "knn_vector", "dimension": dimensions},
}
}
}
},
},
)
# index data
vectors = []
genres = ['fiction', 'drama', 'romance']
genres = ["fiction", "drama", "romance"]
for i in range(3000):
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append({
"_index": index_name,
"_id": i,
"values": vec,
"metadata": {
"genre": random.choice(genres)
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
vectors.append(
{
"_index": index_name,
"_id": i,
"values": vec,
"metadata": {"genre": random.choice(genres)},
}
})
)
# bulk index
helpers.bulk(client, vectors)
@@ -75,30 +72,15 @@ client.indices.refresh(index=index_name)
# search
genre = random.choice(genres)
vec = []
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
for j in range(dimensions):
vec.append(round(random.uniform(0, 1), 2))
print(f"Searching for {vec} with the '{genre}' genre ...")
search_query = {
"query": {
"bool": {
"filter": {
"bool": {
"must": [{
"term": {
"metadata.genre": genre
}
}]
}
},
"must": {
"knn": {
"values": {
"vector": vec,
"k": 5
}
}
}
"filter": {"bool": {"must": [{"term": {"metadata.genre": genre}}]}},
"must": {"knn": {"values": {"vector": vec, "k": 5}}},
}
}
}
samples/knn/knn-efficient-filter.py
+115 -56
View File
@@ -6,75 +6,145 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
import os
import random
from opensearchpy import OpenSearch, helpers
# connect to an instance of OpenSearch
host = os.getenv('HOST', default='localhost')
port = int(os.getenv('PORT', 9200))
auth = (
os.getenv('USERNAME', 'admin'),
os.getenv('PASSWORD', 'admin')
)
host = os.getenv("HOST", default="localhost")
port = int(os.getenv("PORT", 9200))
auth = (os.getenv("USERNAME", "admin"), os.getenv("PASSWORD", "admin"))
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# check whether an index exists
index_name = "hotels-index"
if not client.indices.exists(index_name):
client.indices.create(index_name,
client.indices.create(
index_name,
body={
"settings":{
"settings": {
"index.knn": True,
"knn.algo_param.ef_search": 100,
"number_of_shards": 1,
"number_of_replicas": 0
"number_of_replicas": 0,
},
"mappings":{
"mappings": {
"properties": {
"location": {
"type": "knn_vector",
"type": "knn_vector",
"dimension": 2,
"method": {
"name": "hnsw",
"space_type": "l2",
"engine": "lucene",
"parameters": {
"ef_construction": 100,
"m": 16
}
}
"parameters": {"ef_construction": 100, "m": 16},
},
},
}
}
}
},
},
)
# index data
vectors = [
{ "_index": "hotels-index", "_id": "1", "location": [5.2, 4.4], "parking" : "true", "rating" : 5 },
{ "_index": "hotels-index", "_id": "2", "location": [5.2, 3.9], "parking" : "false", "rating" : 4 },
{ "_index": "hotels-index", "_id": "3", "location": [4.9, 3.4], "parking" : "true", "rating" : 9 },
{ "_index": "hotels-index", "_id": "4", "location": [4.2, 4.6], "parking" : "false", "rating" : 6},
{ "_index": "hotels-index", "_id": "5", "location": [3.3, 4.5], "parking" : "true", "rating" : 8 },
{ "_index": "hotels-index", "_id": "6", "location": [6.4, 3.4], "parking" : "true", "rating" : 9 },
{ "_index": "hotels-index", "_id": "7", "location": [4.2, 6.2], "parking" : "true", "rating" : 5 },
{ "_index": "hotels-index", "_id": "8", "location": [2.4, 4.0], "parking" : "true", "rating" : 8 },
{ "_index": "hotels-index", "_id": "9", "location": [1.4, 3.2], "parking" : "false", "rating" : 5 },
{ "_index": "hotels-index", "_id": "10", "location": [7.0, 9.9], "parking" : "true", "rating" : 9 },
{ "_index": "hotels-index", "_id": "11", "location": [3.0, 2.3], "parking" : "false", "rating" : 6 },
{ "_index": "hotels-index", "_id": "12", "location": [5.0, 1.0], "parking" : "true", "rating" : 3 },
{
"_index": "hotels-index",
"_id": "1",
"location": [5.2, 4.4],
"parking": "true",
"rating": 5,
},
{
"_index": "hotels-index",
"_id": "2",
"location": [5.2, 3.9],
"parking": "false",
"rating": 4,
},
{
"_index": "hotels-index",
"_id": "3",
"location": [4.9, 3.4],
"parking": "true",
"rating": 9,
},
{
"_index": "hotels-index",
"_id": "4",
"location": [4.2, 4.6],
"parking": "false",
"rating": 6,
},
{
"_index": "hotels-index",
"_id": "5",
"location": [3.3, 4.5],
"parking": "true",
"rating": 8,
},
{
"_index": "hotels-index",
"_id": "6",
"location": [6.4, 3.4],
"parking": "true",
"rating": 9,
},
{
"_index": "hotels-index",
"_id": "7",
"location": [4.2, 6.2],
"parking": "true",
"rating": 5,
},
{
"_index": "hotels-index",
"_id": "8",
"location": [2.4, 4.0],
"parking": "true",
"rating": 8,
},
{
"_index": "hotels-index",
"_id": "9",
"location": [1.4, 3.2],
"parking": "false",
"rating": 5,
},
{
"_index": "hotels-index",
"_id": "10",
"location": [7.0, 9.9],
"parking": "true",
"rating": 9,
},
{
"_index": "hotels-index",
"_id": "11",
"location": [3.0, 2.3],
"parking": "false",
"rating": 6,
},
{
"_index": "hotels-index",
"_id": "12",
"location": [5.0, 1.0],
"parking": "true",
"rating": 3,
},
]
helpers.bulk(client, vectors)
@@ -87,30 +157,19 @@ search_query = {
"query": {
"knn": {
"location": {
"vector": [5, 4],
"k": 3,
"filter": {
"bool": {
"must": [
{
"range": {
"rating": {
"gte": 8,
"lte": 10
}
}
},
{
"term": {
"parking": "true"
}
}
"vector": [5, 4],
"k": 3,
"filter": {
"bool": {
"must": [
{"range": {"rating": {"gte": 8, "lte": 10}}},
{"term": {"parking": "true"}},
]
}
}
},
}
}
}
},
}
results = client.search(index=index_name, body=search_query)
samples/security/roles.py
+20 -17
View File
@@ -6,6 +6,9 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
# A basic OpenSearch sample that create and manage roles.
@@ -14,16 +17,16 @@ from opensearchpy import OpenSearch
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# Create a Role
@@ -31,16 +34,16 @@ client = OpenSearch(
role_name = "test-role"
role_content = {
"cluster_permissions": ["cluster_monitor"],
"index_permissions": [
{
"index_patterns": ["index", "test-*"],
"allowed_actions": [
"data_access",
"indices_monitor",
],
}
],
"cluster_permissions": ["cluster_monitor"],
"index_permissions": [
{
"index_patterns": ["index", "test-*"],
"allowed_actions": [
"data_access",
"indices_monitor",
],
}
],
}
response = client.security.create_role(role_name, body=role_content)
samples/security/users.py
+10 -7
View File
@@ -6,6 +6,9 @@
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
# A basic OpenSearch sample that create and manage users.
@@ -14,16 +17,16 @@ from opensearchpy import OpenSearch
# connect to OpenSearch
host = 'localhost'
host = "localhost"
port = 9200
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
auth = ("admin", "admin") # For testing only. Don't store credentials in code.
client = OpenSearch(
hosts = [{'host': host, 'port': port}],
http_auth = auth,
use_ssl = True,
verify_certs = False,
ssl_show_warn = False
hosts=[{"host": host, "port": port}],
http_auth=auth,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False,
)
# Create a User
test_opensearchpy/run_tests.py
+5 -1
View File
@@ -1,10 +1,14 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
utils/generate-api.py
+5 -1
View File
@@ -1,10 +1,14 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
utils/license-headers.py
+24 -43
View File
@@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
@@ -6,24 +7,6 @@
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
#
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Script which verifies that all source files have a license header.
Has two modes: 'fix' and 'check'. 'fix' fixes problems, 'check' will
@@ -33,20 +16,20 @@ error out if 'fix' would have changed the file.
import os
import re
import sys
from itertools import chain
from typing import Iterator, List
lines_to_keep = ["# -*- coding: utf-8 -*-\n", "#!/usr/bin/env python\n"]
license_header_lines = [
"# SPDX-License-Identifier: Apache-2.0\n",
"#\n",
"# The OpenSearch Contributors require contributions made to\n",
"# this file be licensed under the Apache-2.0 license or a\n",
"# compatible open source license.\n",
"#\n",
"# Modifications Copyright OpenSearch Contributors. See\n",
"# GitHub history for details.\n",
]
lines_to_keep = ["# -*- coding: utf-8 -*-", "#!/usr/bin/env python"]
license_header = """
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
#
# Modifications Copyright OpenSearch Contributors. See
# GitHub history for details.
""".strip()
def find_files_to_fix(sources: List[str]) -> Iterator[str]:
@@ -67,20 +50,18 @@ def find_files_to_fix(sources: List[str]) -> Iterator[str]:
def does_file_need_fix(filepath: str) -> bool:
if not re.search(r"\.pyi?$", filepath):
return False
existing_header = ""
with open(filepath, mode="r") as f:
first_license_line = None
for line in f:
if line == license_header_lines[0]:
first_license_line = line
line = line.strip()
if len(line) == 0 or line in lines_to_keep:
pass
elif line[0] == "#":
existing_header += line
existing_header += "\n"
else:
break
elif line not in lines_to_keep:
return True
for header_line, line in zip(
license_header_lines, chain((first_license_line,), f)
):
if line != header_line:
return True
return False
return not existing_header.startswith(license_header)
def add_header_to_file(filepath: str) -> None:
@@ -88,9 +69,9 @@ def add_header_to_file(filepath: str) -> None:
lines = list(f)
i = 0
for i, line in enumerate(lines):
if line not in lines_to_keep:
if len(line) > 0 and line not in lines_to_keep:
break
lines = lines[:i] + license_header_lines + lines[i:]
lines = lines[:i] + [license_header] + lines[i:]
with open(filepath, mode="w") as f:
f.truncate()
f.write("".join(lines))