Python: Introducing vector search to the redis collections

python/pyproject.toml

@@ -101,7 +101,8 @@ qdrant = [
 ]
 redis = [
     "redis[hiredis] ~= 5.0",
-    "types-redis ~= 4.6.0.20240425"
+    "types-redis ~= 4.6.0.20240425",
+    "redisvl >= 0.3.6",
 ]
 usearch = [
     "usearch ~= 2.9",
@@ -206,3 +207,4 @@ name = "semantic_kernel"
 requires = ["flit-core >= 3.9,<4.0"]
 build-backend = "flit_core.buildapi"
 
+

python/samples/concepts/memory/new_memory.py

@@ -10,32 +10,34 @@
 import numpy as np
 
 from semantic_kernel import Kernel
-from semantic_kernel.connectors.ai.open_ai import OpenAIEmbeddingPromptExecutionSettings, OpenAITextEmbedding
-from semantic_kernel.connectors.ai.open_ai.services.azure_text_embedding import AzureTextEmbedding
-from semantic_kernel.connectors.memory.azure_ai_search import AzureAISearchCollection
-from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_collection import (
-    AzureCosmosDBNoSQLCollection,
+from semantic_kernel.connectors.ai.open_ai import (
+    AzureTextEmbedding,
+    OpenAIEmbeddingPromptExecutionSettings,
+    OpenAITextEmbedding,
 )
+from semantic_kernel.connectors.memory.azure_ai_search import AzureAISearchCollection
+from semantic_kernel.connectors.memory.azure_cosmos_db import AzureCosmosDBNoSQLCollection
 from semantic_kernel.connectors.memory.in_memory import InMemoryVectorCollection
-from semantic_kernel.connectors.memory.postgres.postgres_collection import PostgresCollection
+from semantic_kernel.connectors.memory.postgres import PostgresCollection
 from semantic_kernel.connectors.memory.qdrant import QdrantCollection
 from semantic_kernel.connectors.memory.redis import RedisHashsetCollection, RedisJsonCollection
-from semantic_kernel.connectors.memory.weaviate.weaviate_collection import WeaviateCollection
+from semantic_kernel.connectors.memory.weaviate import WeaviateCollection
 from semantic_kernel.data import (
+    DistanceFunction,
+    IndexKind,
+    VectorizableTextSearchMixin,
+    VectorizedSearchMixin,
+    VectorSearchFilter,
+    VectorSearchOptions,
+    VectorSearchResult,
     VectorStoreRecordCollection,
     VectorStoreRecordDataField,
     VectorStoreRecordKeyField,
     VectorStoreRecordUtils,
     VectorStoreRecordVectorField,
+    VectorTextSearchMixin,
     vectorstoremodel,
 )
-from semantic_kernel.data.const import DistanceFunction, IndexKind
-from semantic_kernel.data.vector_search.vector_search_filter import VectorSearchFilter
-from semantic_kernel.data.vector_search.vector_search_options import VectorSearchOptions
-from semantic_kernel.data.vector_search.vector_search_result import VectorSearchResult
-from semantic_kernel.data.vector_search.vector_text_search import VectorTextSearchMixin
-from semantic_kernel.data.vector_search.vectorizable_text_search import VectorizableTextSearchMixin
-from semantic_kernel.data.vector_search.vectorized_search import VectorizedSearchMixin
 
 
 def get_data_model_array(index_kind: IndexKind, distance_function: DistanceFunction) -> type:
@@ -64,8 +66,8 @@ class DataModelArray:
                 is_full_text_searchable=True,
             ),
         ] = "content1"
-        title: Annotated[str, VectorStoreRecordDataField(property_type="str", is_full_text_searchable=True)] = "title1"
-        tag: Annotated[str, VectorStoreRecordDataField(property_type="str", is_filterable=True)] = "tag1"
+        title: Annotated[str, VectorStoreRecordDataField(property_type="str", is_full_text_searchable=True)] = "title"
+        tag: Annotated[str, VectorStoreRecordDataField(property_type="str", is_filterable=True)] = "tag"
 
     return DataModelArray
 
@@ -94,8 +96,8 @@ class DataModelList:
                 is_full_text_searchable=True,
             ),
         ] = "content1"
-        title: Annotated[str, VectorStoreRecordDataField(property_type="str", is_full_text_searchable=True)] = "title1"
-        tag: Annotated[str, VectorStoreRecordDataField(property_type="str", is_filterable=True)] = "tag1"
+        title: Annotated[str, VectorStoreRecordDataField(property_type="str", is_full_text_searchable=True)] = "title"
+        tag: Annotated[str, VectorStoreRecordDataField(property_type="str", is_filterable=True)] = "tag"
 
     return DataModelList
 
@@ -139,7 +141,7 @@ class DataModelList:
         collection_name=collection_name,
         prefix_collection_name_to_key_names=True,
     ),
-    "redis_hashset": lambda: RedisHashsetCollection[DataModel](
+    "redis_hash": lambda: RedisHashsetCollection[DataModel](
         data_model_type=DataModel,
         collection_name=collection_name,
         prefix_collection_name_to_key_names=True,
@@ -171,9 +173,6 @@ def print_record(result: VectorSearchResult | None = None, record: DataModel | N
     print(f"    Content: {record.content}")
     if record.vector is not None:
         print(f"    Vector (first five): {record.vector[:5]}")
-    if result:
-        print(f"  Score: {result.score:.4f}")
-    print()
 
 
 async def main(collection: str, use_azure_openai: bool, embedding_model: str):
@@ -207,6 +206,7 @@ async def main(collection: str, use_azure_openai: bool, embedding_model: str):
         records = await VectorStoreRecordUtils(kernel).add_vector_to_records(
             [record1, record2], data_model_type=DataModel
         )
+
         keys = await record_collection.upsert_batch(records)
         print(f"    Upserted {keys=}")
         print("Getting records!")

python/semantic_kernel/connectors/memory/azure_ai_search/const.py

@@ -18,9 +18,10 @@
 }
 
 DISTANCE_FUNCTION_MAP = {
-    DistanceFunction.COSINE_SIMILARITY: VectorSearchAlgorithmMetric.COSINE,
+    DistanceFunction.COSINE_DISTANCE: VectorSearchAlgorithmMetric.COSINE,
     DistanceFunction.DOT_PROD: VectorSearchAlgorithmMetric.DOT_PRODUCT,
     DistanceFunction.EUCLIDEAN_DISTANCE: VectorSearchAlgorithmMetric.EUCLIDEAN,
+    DistanceFunction.HAMMING: VectorSearchAlgorithmMetric.HAMMING,
     "default": VectorSearchAlgorithmMetric.COSINE,
 }
 

python/semantic_kernel/connectors/memory/azure_ai_search/utils.py

@@ -145,12 +145,21 @@ def data_model_definition_to_azure_ai_search_index(
                     algorithm_configuration_name=algo_name,
                 )
             )
-            algo_class, algo_params = INDEX_ALGORITHM_MAP[field.index_kind or "default"]
+            try:
+                algo_class, algo_params = INDEX_ALGORITHM_MAP[field.index_kind or "default"]
+            except KeyError as e:
+                raise ServiceInitializationError(f"Error: {field.index_kind} not found in INDEX_ALGORITHM_MAP.") from e
+            try:
+                distance_metric = DISTANCE_FUNCTION_MAP[field.distance_function or "default"]
+            except KeyError as e:
+                raise ServiceInitializationError(
+                    f"Error: {field.distance_function} not found in DISTANCE_FUNCTION_MAP."
+                ) from e
             search_algos.append(
                 algo_class(
                     name=algo_name,
                     parameters=algo_params(
-                        metric=DISTANCE_FUNCTION_MAP[field.distance_function or "default"],
+                        metric=distance_metric,
                     ),
                 )
             )

python/semantic_kernel/connectors/memory/azure_cosmos_db/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_collection import (
+    AzureCosmosDBNoSQLCollection,
+)
+from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_composite_key import (
+    AzureCosmosDBNoSQLCompositeKey,
+)
+from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_settings import AzureCosmosDBNoSQLSettings
+from semantic_kernel.connectors.memory.azure_cosmos_db.azure_cosmos_db_no_sql_store import AzureCosmosDBNoSQLStore
+
+__all__ = [
+    "AzureCosmosDBNoSQLCollection",
+    "AzureCosmosDBNoSQLCompositeKey",
+    "AzureCosmosDBNoSQLSettings",
+    "AzureCosmosDBNoSQLStore",
+]

python/semantic_kernel/connectors/memory/redis/const.py

@@ -4,6 +4,7 @@
 from enum import Enum
 
 from redis.commands.search.indexDefinition import IndexType
+from redisvl.schema import StorageType
 
 from semantic_kernel.data.const import DistanceFunction
 
@@ -18,6 +19,11 @@ class RedisCollectionTypes(str, Enum):
     RedisCollectionTypes.HASHSET: IndexType.HASH,
 }
 
+STORAGE_TYPE_MAP = {
+    RedisCollectionTypes.JSON: StorageType.JSON,
+    RedisCollectionTypes.HASHSET: StorageType.HASH,
+}
+
 DISTANCE_FUNCTION_MAP = {
     DistanceFunction.COSINE_SIMILARITY: "COSINE",
     DistanceFunction.DOT_PROD: "IP",