Skip to content

Commit 6edc352

Browse files
author
Sam Partee
authored
Refactor filtering approach (#45)
## Description This PR breaks the current filter interface in favor of a more expressive syntax that allows arbitrary combination and filtering.
1 parent 0e7dde0 commit 6edc352

File tree

10 files changed

+857
-575
lines changed

10 files changed

+857
-575
lines changed

‎docs/examples/openai_qna.ipynb‎

Lines changed: 3 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -40,39 +40,9 @@
4040
},
4141
{
4242
"cell_type": "code",
43-
"execution_count": 1,
43+
"execution_count": null,
4444
"metadata": {},
45-
"outputs": [
46-
{
47-
"name": "stdout",
48-
"output_type": "stream",
49-
"text": [
50-
"Requirement already satisfied: pandas in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (1.5.0)\n",
51-
"Requirement already satisfied: wget in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (3.2)\n",
52-
"Requirement already satisfied: tenacity in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (8.2.2)\n",
53-
"Requirement already satisfied: tiktoken in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (0.4.0)\n",
54-
"Requirement already satisfied: openai in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (0.27.2)\n",
55-
"Requirement already satisfied: python-dateutil>=2.8.1 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from pandas) (2.8.2)\n",
56-
"Requirement already satisfied: numpy>=1.20.3 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from pandas) (1.24.2)\n",
57-
"Requirement already satisfied: pytz>=2020.1 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from pandas) (2022.4)\n",
58-
"Requirement already satisfied: regex>=2022.1.18 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from tiktoken) (2022.9.13)\n",
59-
"Requirement already satisfied: requests>=2.26.0 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from tiktoken) (2.28.1)\n",
60-
"Requirement already satisfied: aiohttp in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from openai) (3.8.3)\n",
61-
"Requirement already satisfied: tqdm in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from openai) (4.63.0)\n",
62-
"Requirement already satisfied: six>=1.5 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",
63-
"Requirement already satisfied: charset-normalizer<3,>=2 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (2.0.4)\n",
64-
"Requirement already satisfied: idna<4,>=2.5 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (3.3)\n",
65-
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (1.26.8)\n",
66-
"Requirement already satisfied: certifi>=2017.4.17 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from requests>=2.26.0->tiktoken) (2022.9.24)\n",
67-
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (4.0.2)\n",
68-
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (1.3.1)\n",
69-
"Requirement already satisfied: attrs>=17.3.0 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (22.1.0)\n",
70-
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (1.3.3)\n",
71-
"Requirement already satisfied: yarl<2.0,>=1.0 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (1.8.2)\n",
72-
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/tyler.hutcherson/miniconda3/lib/python3.9/site-packages (from aiohttp->openai) (6.0.4)\n"
73-
]
74-
}
75-
],
45+
"outputs": [],
7646
"source": [
7747
"# first we need to install a few things\n",
7848
"\n",
@@ -879,7 +849,7 @@
879849
"name": "python",
880850
"nbconvert_exporter": "python",
881851
"pygments_lexer": "ipython3",
882-
"version": "3.9.12"
852+
"version": "3.8.13"
883853
},
884854
"orig_nbformat": 4
885855
},

‎docs/user_guide/hybrid_queries_02.ipynb‎

Lines changed: 213 additions & 92 deletions
Large diffs are not rendered by default.

‎docs/user_guide/llmcache_03.ipynb‎

Lines changed: 102 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,20 @@
2020
},
2121
{
2222
"cell_type": "code",
23-
"execution_count": 27,
23+
"execution_count": 16,
2424
"metadata": {},
2525
"outputs": [],
2626
"source": [
27-
"import openai\n",
2827
"import os\n",
28+
"import openai\n",
2929
"import getpass\n",
30+
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"False\"\n",
3031
"\n",
31-
"api_key = os.getenv(\"OPENAI_API_KEY\") or getpass.getpass(\"Enter your OpenAI API key: \")\n",
32+
"api_key = os.getenv(\"OPENAI_API_KEY\", None)\n",
33+
"if not api_key:\n",
34+
" api_key = getpass.getpass(\"Enter your OpenAI API key: \")\n",
35+
"\n",
36+
"openai.api_key = api_key\n",
3237
"\n",
3338
"def ask_openai(question):\n",
3439
" response = openai.Completion.create(\n",
@@ -41,14 +46,14 @@
4146
},
4247
{
4348
"cell_type": "code",
44-
"execution_count": 28,
49+
"execution_count": 17,
4550
"metadata": {},
4651
"outputs": [
4752
{
4853
"name": "stdout",
4954
"output_type": "stream",
5055
"text": [
51-
"The capital of France is Paris.\n"
56+
"Paris\n"
5257
]
5358
}
5459
],
@@ -68,7 +73,7 @@
6873
},
6974
{
7075
"cell_type": "code",
71-
"execution_count": 46,
76+
"execution_count": 18,
7277
"metadata": {},
7378
"outputs": [],
7479
"source": [
@@ -81,7 +86,38 @@
8186
},
8287
{
8388
"cell_type": "code",
84-
"execution_count": 47,
89+
"execution_count": 19,
90+
"metadata": {},
91+
"outputs": [
92+
{
93+
"name": "stdout",
94+
"output_type": "stream",
95+
"text": [
96+
"\n",
97+
"\n",
98+
"Index Information:\n",
99+
"╭──────────────┬────────────────┬──────────────┬─────────────────┬────────────╮\n",
100+
"│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n",
101+
"├──────────────┼────────────────┼──────────────┼─────────────────┼────────────┤\n",
102+
"│ cache │ HASH │ ['llmcache'] │ [] │ 0 │\n",
103+
"╰──────────────┴────────────────┴──────────────┴─────────────────┴────────────╯\n",
104+
"Index Fields:\n",
105+
"╭───────────────┬───────────────┬────────╮\n",
106+
"│ Name │ Attribute │ Type │\n",
107+
"├───────────────┼───────────────┼────────┤\n",
108+
"│ prompt_vector │ prompt_vector │ VECTOR │\n",
109+
"╰───────────────┴───────────────┴────────╯\n"
110+
]
111+
}
112+
],
113+
"source": [
114+
"# look at the index specification created for the semantic cache lookup\n",
115+
"!rvl index info -i cache"
116+
]
117+
},
118+
{
119+
"cell_type": "code",
120+
"execution_count": 20,
85121
"metadata": {},
86122
"outputs": [
87123
{
@@ -90,7 +126,7 @@
90126
"[]"
91127
]
92128
},
93-
"execution_count": 47,
129+
"execution_count": 20,
94130
"metadata": {},
95131
"output_type": "execute_result"
96132
}
@@ -102,7 +138,7 @@
102138
},
103139
{
104140
"cell_type": "code",
105-
"execution_count": 31,
141+
"execution_count": 21,
106142
"metadata": {},
107143
"outputs": [],
108144
"source": [
@@ -112,7 +148,7 @@
112148
},
113149
{
114150
"cell_type": "code",
115-
"execution_count": 32,
151+
"execution_count": 22,
116152
"metadata": {},
117153
"outputs": [
118154
{
@@ -121,7 +157,7 @@
121157
"['Paris']"
122158
]
123159
},
124-
"execution_count": 32,
160+
"execution_count": 22,
125161
"metadata": {},
126162
"output_type": "execute_result"
127163
}
@@ -133,7 +169,7 @@
133169
},
134170
{
135171
"cell_type": "code",
136-
"execution_count": 33,
172+
"execution_count": 23,
137173
"metadata": {},
138174
"outputs": [
139175
{
@@ -142,7 +178,7 @@
142178
"[]"
143179
]
144180
},
145-
"execution_count": 33,
181+
"execution_count": 23,
146182
"metadata": {},
147183
"output_type": "execute_result"
148184
}
@@ -154,7 +190,7 @@
154190
},
155191
{
156192
"cell_type": "code",
157-
"execution_count": 34,
193+
"execution_count": 24,
158194
"metadata": {},
159195
"outputs": [
160196
{
@@ -163,7 +199,7 @@
163199
"['Paris']"
164200
]
165201
},
166-
"execution_count": 34,
202+
"execution_count": 24,
167203
"metadata": {},
168204
"output_type": "execute_result"
169205
}
@@ -176,7 +212,7 @@
176212
},
177213
{
178214
"cell_type": "code",
179-
"execution_count": 35,
215+
"execution_count": 25,
180216
"metadata": {},
181217
"outputs": [
182218
{
@@ -185,7 +221,7 @@
185221
"[]"
186222
]
187223
},
188-
"execution_count": 35,
224+
"execution_count": 25,
189225
"metadata": {},
190226
"output_type": "execute_result"
191227
}
@@ -197,11 +233,11 @@
197233
},
198234
{
199235
"cell_type": "code",
200-
"execution_count": 45,
236+
"execution_count": 26,
201237
"metadata": {},
202238
"outputs": [],
203239
"source": [
204-
"cache.index.delete()"
240+
"cache.clear()"
205241
]
206242
},
207243
{
@@ -215,7 +251,7 @@
215251
},
216252
{
217253
"cell_type": "code",
218-
"execution_count": 48,
254+
"execution_count": 27,
219255
"metadata": {},
220256
"outputs": [],
221257
"source": [
@@ -231,14 +267,14 @@
231267
},
232268
{
233269
"cell_type": "code",
234-
"execution_count": 49,
270+
"execution_count": 28,
235271
"metadata": {},
236272
"outputs": [
237273
{
238274
"name": "stdout",
239275
"output_type": "stream",
240276
"text": [
241-
"Time taken without cache 0.6192829608917236\n"
277+
"Time taken without cache 0.8732700347900391\n"
242278
]
243279
}
244280
],
@@ -252,15 +288,15 @@
252288
},
253289
{
254290
"cell_type": "code",
255-
"execution_count": 50,
291+
"execution_count": 29,
256292
"metadata": {},
257293
"outputs": [
258294
{
259295
"name": "stdout",
260296
"output_type": "stream",
261297
"text": [
262-
"Time Taken with cache: 0.05961775779724121\n",
263-
"Percentage of time saved: 90.37%\n"
298+
"Time Taken with cache: 0.04746699333190918\n",
299+
"Percentage of time saved: 94.56%\n"
264300
]
265301
}
266302
],
@@ -274,20 +310,55 @@
274310
},
275311
{
276312
"cell_type": "code",
277-
"execution_count": 51,
313+
"execution_count": 30,
278314
"metadata": {},
279-
"outputs": [],
315+
"outputs": [
316+
{
317+
"name": "stdout",
318+
"output_type": "stream",
319+
"text": [
320+
"\n",
321+
"Statistics:\n",
322+
"╭─────────────────────────────┬─────────────╮\n",
323+
"│ Stat Key │ Value │\n",
324+
"├─────────────────────────────┼─────────────┤\n",
325+
"│ num_docs │ 1 │\n",
326+
"│ num_terms │ 0 │\n",
327+
"│ max_doc_id │ 2 │\n",
328+
"│ num_records │ 2 │\n",
329+
"│ percent_indexed │ 1 │\n",
330+
"│ hash_indexing_failures │ 0 │\n",
331+
"│ number_of_uses │ 11 │\n",
332+
"│ bytes_per_record_avg │ 0 │\n",
333+
"│ doc_table_size_mb │ 0.000134468 │\n",
334+
"│ inverted_sz_mb │ 0 │\n",
335+
"│ key_table_size_mb │ 2.76566e-05 │\n",
336+
"│ offset_bits_per_record_avg │ nan │\n",
337+
"│ offset_vectors_sz_mb │ 0 │\n",
338+
"│ offsets_per_term_avg │ 0 │\n",
339+
"│ records_per_doc_avg │ 2 │\n",
340+
"│ sortable_values_size_mb │ 0 │\n",
341+
"│ total_indexing_time │ 0.211 │\n",
342+
"│ total_inverted_index_blocks │ 11 │\n",
343+
"│ vector_index_sz_mb │ 3.00814 │\n",
344+
"╰─────────────────────────────┴─────────────╯\n"
345+
]
346+
}
347+
],
280348
"source": [
281-
"# remove the index and all cached items\n",
282-
"cache.index.delete()"
349+
"# check the stats of the index\n",
350+
"!rvl stats -i cache"
283351
]
284352
},
285353
{
286354
"cell_type": "code",
287-
"execution_count": null,
355+
"execution_count": 31,
288356
"metadata": {},
289357
"outputs": [],
290-
"source": []
358+
"source": [
359+
"# remove the index and all cached items\n",
360+
"cache.index.delete()"
361+
]
291362
}
292363
],
293364
"metadata": {

‎redisvl/index.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
if TYPE_CHECKING:
66
from redis.commands.search.field import Field
77
from redis.commands.search.result import Result
8+
from redisvl.query.query import BaseQuery
89

910
import redis
1011
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
1112

12-
from redisvl.query import BaseQuery
1313
from redisvl.schema import SchemaModel, read_schema
1414
from redisvl.utils.connection import (
1515
check_connected,
@@ -67,7 +67,8 @@ def search(self, *args, **kwargs) -> List["Result"]:
6767
)
6868
return results
6969

70-
def query(self, query: BaseQuery) -> List[Dict[str, Any]]:
70+
@check_connected("_redis_conn")
71+
def query(self, query: "BaseQuery") -> List[Dict[str, Any]]:
7172
"""Run a query on this index.
7273
7374
This is similar to the search method, but takes a BaseQuery
@@ -534,7 +535,7 @@ async def search(self, *args, **kwargs) -> List["Result"]:
534535
results: List["Result"] = await self._redis_conn.ft(self._name).search(*args, **kwargs) # type: ignore
535536
return results
536537

537-
async def query(self, query: BaseQuery) -> List[Dict[str, Any]]:
538+
async def query(self, query: "BaseQuery") -> List[Dict[str, Any]]:
538539
"""Run a query on this index.
539540
540541
This is similar to the search method, but takes a BaseQuery

0 commit comments

Comments
 (0)