Redis connection refactor (#104)

For ultimate clarity and safety of our library, the decision was made to
separate the sync and async search index modules for now until we can
better utilize the redis-py client to serve both needs simultaneously.

This PR splits `SearchIndex` and `AsyncSearchIndex` into two classes. It
also refactors the way RedisVL handles Redis connections (much more
lightweight) and makes some major improvements to the overall
documentation on these topics.

Author: tylerhutcherson

Makefile

@@ -77,17 +77,17 @@ servedocs:
 # help: test                           - Run all tests
 .PHONY: test
 test:
-	@python -m pytest
+	@python -m pytest --log-level=CRITICAL
 
 # help: test-verbose                   - Run all tests verbosely
 .PHONY: test-verbose
 test-verbose:
-	@python -m pytest -vv -s
+	@python -m pytest -vv -s --log-level=CRITICAL
 
 # help: test-cov                       - Run all tests with coverage
 .PHONY: test-cov
 test-cov:
-	@python -m pytest -vv --cov=./redisvl
+	@python -m pytest -vv --cov=./redisvl --log-level=CRITICAL
 
 # help: cov                            - generate html coverage report
 .PHONY: cov

conftest.py

@@ -2,25 +2,20 @@
 import pytest
 import asyncio
 
-from redisvl.utils.connection import RedisConnection
+from redisvl.redis.connection import RedisConnectionFactory
 
-REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
-
-aredis = RedisConnection.get_async_redis_connection(REDIS_URL)
-redis = RedisConnection.get_redis_connection(REDIS_URL)
 
 @pytest.fixture()
 def redis_url():
-    return REDIS_URL
+    return os.getenv("REDIS_URL", "redis://localhost:6379")
 
 @pytest.fixture
-def async_client():
-    return aredis
+def async_client(redis_url):
+    return RedisConnectionFactory.get_async_redis_connection(redis_url)
 
 @pytest.fixture
-def client():
-    return redis
-
+def client(redis_url):
+    return RedisConnectionFactory.get_redis_connection(redis_url)
 
 @pytest.fixture
 def openai_key():
@@ -38,6 +33,68 @@ def gcp_location():
 def gcp_project_id():
     return os.getenv("GCP_PROJECT_ID")
 
+
+@pytest.fixture
+def sample_data():
+    return [
+    {
+        "user": "john",
+        "age": 18,
+        "job": "engineer",
+        "credit_score": "high",
+        "location": "-122.4194,37.7749",
+        "user_embedding": [0.1, 0.1, 0.5]
+    },
+    {
+        "user": "mary",
+        "age": 14,
+        "job": "doctor",
+        "credit_score": "low",
+        "location": "-122.4194,37.7749",
+        "user_embedding": [0.1, 0.1, 0.5]
+    },
+    {
+        "user": "nancy",
+        "age": 94,
+        "job": "doctor",
+        "credit_score": "high",
+        "location": "-122.4194,37.7749",
+        "user_embedding": [0.7, 0.1, 0.5]
+    },
+    {
+        "user": "tyler",
+        "age": 100,
+        "job": "engineer",
+        "credit_score": "high",
+        "location": "-110.0839,37.3861",
+        "user_embedding": [0.1, 0.4, 0.5]
+    },
+    {
+        "user": "tim",
+        "age": 12,
+        "job": "dermatologist",
+        "credit_score": "high",
+        "location": "-110.0839,37.3861",
+        "user_embedding": [0.4, 0.4, 0.5]
+    },
+    {
+        "user": "taimur",
+        "age": 15,
+        "job": "CEO",
+        "credit_score": "low",
+        "location": "-110.0839,37.3861",
+        "user_embedding": [0.6, 0.1, 0.5]
+    },
+    {
+        "user": "joe",
+        "age": 35,
+        "job": "dentist",
+        "credit_score": "medium",
+        "location": "-110.0839,37.3861",
+        "user_embedding": [0.9, 0.9, 0.1]
+    },
+]
+
 @pytest.fixture(scope="session")
 def event_loop():
     try:
@@ -48,7 +105,7 @@ def event_loop():
     loop.close()
 
 @pytest.fixture
-def clear_db():
+def clear_db(redis):
     redis.flushall()
     yield
     redis.flushall()

docs/_static/gallery.yaml

@@ -1,6 +1,6 @@
 
 - title: Arxiv Paper Search
-  website: https://docsearch.redisventures.com
+  website: https://docsearch.redisvl.com
   img-bottom: ../_static/gallery-images/arxiv-search.png
 - title: Real-Time Embeddings with Redis and Bytewax
   website: https://github.com/awmatheson/real-time-embeddings

docs/_static/js/sidebar.js

@@ -2,19 +2,18 @@ const toc = [
     { header: "Overview", toc: [
         { title: "RedisVL", path: "/index.html" },
         { title: "Install", path: "/overview/installation.html" },
-        { title: "CLI", path: "/user_guide/cli.html" },
+        { title: "CLI", path: "/overview/cli.html" },
     ]},
     { header: "User Guides", toc: [
         { title: "Getting Started", path: "/user_guide/getting_started_01.html" },
         { title: "Query and Filter", path: "/user_guide/hybrid_queries_02.html" },
         { title: "JSON vs Hash Storage", path: "/user_guide/hash_vs_json_05.html" },
         { title: "Vectorizers", path: "/user_guide/vectorizers_04.html" },
         { title: "Semantic Caching", path: "/user_guide/llmcache_03.html" },
-
     ]},
     { header: "API", toc: [
-        { title: "Schema", path: "/api/indexschema.html"},
-        { title: "Index", path: "/api/searchindex.html" },
+        { title: "Schema", path: "/api/schema.html"},
+        { title: "Search Index", path: "/api/searchindex.html" },
         { title: "Query", path: "/api/query.html" },
         { title: "Filter", path: "/api/filter.html" },
         { title: "Vectorizers", path: "/api/vectorizer.html" },

docs/api/index.md

@@ -11,7 +11,7 @@ myst:
 :caption: RedisVL
 :maxdepth: 2
 
-indexschema
+schema
 searchindex
 query
 filter

docs/api/indexschema.rst docs/api/schema.rstdocs/api/indexschema.rst renamed to docs/api/schema.rst

@@ -1,6 +1,17 @@
-***********
-SearchIndex
-***********
+********************
+Search Index Classes
+********************
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 1
+
+   * - Class
+     - Description
+   * - `SearchIndex <#searchindex_api>`_
+     - Primary class to write, read, and search across data structures in Redis.
+   * - `AsyncSearchIndex <#asyncsearchindex_api>`_
+     - Async version of the SearchIndex to write, read, and search across data structures in Redis.
 
 SearchIndex
 ===========
@@ -9,36 +20,19 @@ SearchIndex
 
 .. currentmodule:: redisvl.index
 
-.. autosummary::
-
-    SearchIndex.from_yaml
-    SearchIndex.from_dict
-    SearchIndex.client
-    SearchIndex.name
-    SearchIndex.prefix
-    SearchIndex.key_separator
-    SearchIndex.storage_type
-    SearchIndex.connect
-    SearchIndex.disconnect
-    SearchIndex.set_client
-    SearchIndex.create
-    SearchIndex.acreate
-    SearchIndex.exists
-    SearchIndex.aexists
-    SearchIndex.load
-    SearchIndex.aload
-    SearchIndex.search
-    SearchIndex.asearch
-    SearchIndex.query
-    SearchIndex.aquery
-    SearchIndex.query_batch
-    SearchIndex.aquery_batch
-    SearchIndex.delete
-    SearchIndex.adelete
-    SearchIndex.info
-    SearchIndex.ainfo
-
 .. autoclass:: SearchIndex
    :show-inheritance:
    :inherited-members:
    :members:
+
+AsyncSearchIndex
+================
+
+.. _asyncsearchindex_api:
+
+.. currentmodule:: redisvl.index
+
+.. autoclass:: AsyncSearchIndex
+   :show-inheritance:
+   :inherited-members:
+   :members: