Skip to content

Commit 727cd76

Browse files
tylerhutchersontylerhutcherson
authored
Improve redisvl module structure (#106)
In order to position RedisVL for growth and adhere to some best practices, this PR moves around a few submodules and packages things in a better way overall: ``` redisvl ├── __init__.py ├── cli │ ├── __init__.py │ ├── index.py │ ├── main.py │ ├── runner.py │ ├── stats.py │ ├── utils.py │ └── version.py ├── extensions │ ├── __init__.py │ └── llmcache │ ├── __init__.py │ ├── base.py │ └── semantic.py ├── index │ ├── __init__.py │ ├── index.py │ └── storage.py ├── query │ ├── __init__.py │ ├── filter.py │ └── query.py ├── redis │ ├── __init__.py │ ├── connection.py │ ├── constants.py │ └── utils.py ├── schema │ ├── __init__.py │ ├── fields.py │ └── schema.py ├── utils │ ├── __init__.py │ ├── log.py │ ├── token_escaper.py │ └── vectorize │ ├── __init__.py │ ├── base.py │ └── text │ ├── __init__.py │ ├── cohere.py │ ├── huggingface.py │ ├── openai.py │ └── vertexai.py └── version.py ``` - Move vectorizers under utils (since its a useful tool) - Create an extensions module and add llmcache. Others will be added there too.
1 parent 4f120b0 commit 727cd76

26 files changed

+77
-71
lines changed

‎docs/examples/openai_qna.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@
574574
"import os\n",
575575
"import getpass\n",
576576
"\n",
577-
"from redisvl.vectorize.text import OpenAITextVectorizer\n",
577+
"from redisvl.utils.vectorize import OpenAITextVectorizer\n",
578578
"\n",
579579
"api_key = os.getenv(\"OPENAI_API_KEY\") or getpass.getpass(\"Enter your OpenAI API key: \")\n",
580580
"oaip = OpenAITextVectorizer(EMBEDDINGS_MODEL, api_config={\"api_key\": api_key})\n",
@@ -859,7 +859,7 @@
859859
"metadata": {},
860860
"outputs": [],
861861
"source": [
862-
"from redisvl.llmcache import SemanticCache\n",
862+
"from redisvl.extensions.llmcache import SemanticCache\n",
863863
"\n",
864864
"cache = SemanticCache(redis_url=\"redis://localhost:6379\", distance_threshold=0.2)"
865865
]

‎docs/user_guide/getting_started_01.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@
379379
"cell_type": "markdown",
380380
"metadata": {},
381381
"source": [
382-
">By default, `load` will create a unique Redis \"key\" as a combination of the index key `prefix` and a UUID. You can also customize the key by providing direct keys or pointing to a specified `key_field` on load."
382+
">By default, `load` will create a unique Redis \"key\" as a combination of the index key `prefix` and a UUID. You can also customize the key by providing direct keys or pointing to a specified `id_field` on load."
383383
]
384384
},
385385
{

‎docs/user_guide/llmcache_03.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@
7979
"metadata": {},
8080
"outputs": [],
8181
"source": [
82-
"from redisvl.llmcache import SemanticCache\n",
82+
"from redisvl.extensions.llmcache import SemanticCache\n",
8383
"\n",
8484
"llmcache = SemanticCache(\n",
8585
" name=\"llmcache\", # underlying search index name\n",

‎docs/user_guide/vectorizers_04.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@
107107
}
108108
],
109109
"source": [
110-
"from redisvl.vectorize.text import OpenAITextVectorizer\n",
110+
"from redisvl.utils.vectorize import OpenAITextVectorizer\n",
111111
"\n",
112112
"# create a vectorizer\n",
113113
"oai = OpenAITextVectorizer(\n",
@@ -224,7 +224,7 @@
224224
],
225225
"source": [
226226
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
227-
"from redisvl.vectorize.text import HFTextVectorizer\n",
227+
"from redisvl.utils.vectorize import HFTextVectorizer\n",
228228
"\n",
229229
"\n",
230230
"# create a vectorizer\n",
@@ -297,7 +297,7 @@
297297
}
298298
],
299299
"source": [
300-
"from redisvl.vectorize.text import VertexAITextVectorizer\n",
300+
"from redisvl.utils.vectorize import VertexAITextVectorizer\n",
301301
"\n",
302302
"\n",
303303
"# create a vectorizer\n",
@@ -363,7 +363,7 @@
363363
}
364364
],
365365
"source": [
366-
"from redisvl.vectorize.text import CohereTextVectorizer\n",
366+
"from redisvl.utils.vectorize import CohereTextVectorizer\n",
367367
"\n",
368368
"# create a vectorizer\n",
369369
"co = CohereTextVectorizer(\n",

redisvl/vectorize/__init__.py renamed to redisvl/extensions/__init__.py

File renamed without changes.

‎redisvl/extensions/llmcache/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from redisvl.extensions.llmcache.semantic import SemanticCache
2+
3+
__all__ = ["SemanticCache"]

redisvl/llmcache/base.py renamed to redisvl/extensions/llmcache/base.py

File renamed without changes.

redisvl/llmcache/semantic.py renamed to redisvl/extensions/llmcache/semantic.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@
22

33
from redis import Redis
44

5+
from redisvl.extensions.llmcache.base import BaseLLMCache
56
from redisvl.index import SearchIndex
6-
from redisvl.llmcache.base import BaseLLMCache
77
from redisvl.query import RangeQuery
88
from redisvl.redis.utils import array_to_buffer
99
from redisvl.schema.schema import IndexSchema
10-
from redisvl.vectorize.base import BaseVectorizer
11-
from redisvl.vectorize.text import HFTextVectorizer
10+
from redisvl.utils.vectorize import BaseVectorizer, HFTextVectorizer
1211

1312

1413
class SemanticCache(BaseLLMCache):
@@ -331,5 +330,5 @@ def store(
331330
payload[self.metadata_field_name] = self.serialize(metadata)
332331

333332
# Load LLMCache entry with TTL
334-
keys = self._index.load(data=[payload], ttl=self._ttl, key_field=id_field)
333+
keys = self._index.load(data=[payload], ttl=self._ttl, id_field=id_field)
335334
return keys[0]

‎redisvl/index/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from redisvl.index.index import AsyncSearchIndex, SearchIndex
2+
3+
__all__ = ["SearchIndex", "AsyncSearchIndex"]

redisvl/index.py renamed to redisvl/index/index.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@
2222
import redis.asyncio as aredis
2323
from redis.commands.search.indexDefinition import IndexDefinition
2424

25+
from redisvl.index.storage import HashStorage, JsonStorage
2526
from redisvl.query.query import BaseQuery, CountQuery, FilterQuery
2627
from redisvl.redis.connection import RedisConnectionFactory
2728
from redisvl.redis.utils import convert_bytes
2829
from redisvl.schema import IndexSchema, StorageType
29-
from redisvl.storage import HashStorage, JsonStorage
3030
from redisvl.utils.log import get_logger
3131

3232
logger = get_logger(__name__)
@@ -454,7 +454,7 @@ def delete(self, drop: bool = True):
454454
def load(
455455
self,
456456
data: Iterable[Any],
457-
key_field: Optional[str] = None,
457+
id_field: Optional[str] = None,
458458
keys: Optional[Iterable[str]] = None,
459459
ttl: Optional[int] = None,
460460
preprocess: Optional[Callable] = None,
@@ -465,7 +465,8 @@ def load(
465465
466466
Args:
467467
data (Iterable[Any]): An iterable of objects to store.
468-
key_field (Optional[str], optional): Field used as the key for each
468+
id_field (Optional[str], optional): Specified field used as the id
469+
portion of the redis key (after the prefix) for each
469470
object. Defaults to None.
470471
keys (Optional[Iterable[str]], optional): Optional iterable of keys.
471472
Must match the length of objects if provided. Defaults to None.
@@ -492,7 +493,7 @@ def load(
492493
return self._storage.write(
493494
self._redis_client, # type: ignore
494495
objects=data,
495-
key_field=key_field,
496+
id_field=id_field,
496497
keys=keys,
497498
ttl=ttl,
498499
preprocess=preprocess,
@@ -792,7 +793,7 @@ async def delete(self, drop: bool = True):
792793
async def load(
793794
self,
794795
data: Iterable[Any],
795-
key_field: Optional[str] = None,
796+
id_field: Optional[str] = None,
796797
keys: Optional[Iterable[str]] = None,
797798
ttl: Optional[int] = None,
798799
preprocess: Optional[Callable] = None,
@@ -803,7 +804,8 @@ async def load(
803804
804805
Args:
805806
data (Iterable[Any]): An iterable of objects to store.
806-
key_field (Optional[str], optional): Field used as the key for each
807+
id_field (Optional[str], optional): Specified field used as the id
808+
portion of the redis key (after the prefix) for each
807809
object. Defaults to None.
808810
keys (Optional[Iterable[str]], optional): Optional iterable of keys.
809811
Must match the length of objects if provided. Defaults to None.
@@ -831,7 +833,7 @@ async def load(
831833
return await self._storage.awrite(
832834
self._redis_client, # type: ignore
833835
objects=data,
834-
key_field=key_field,
836+
id_field=id_field,
835837
keys=keys,
836838
ttl=ttl,
837839
preprocess=preprocess,

0 commit comments

Comments
 (0)