diff --git a/README.md b/README.md index bbe8101913..83700bc55b 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Supports a wide range of data types including strings, numerics, vectors, and mo Infinity, also available as a Python module, eliminates the need for a separate back-end server and all the complex communication settings. Using `pip install` and `import infinity`, you can quickly build a local AI application in Python, leveraging the world's fastest and the most powerful RAG database: ```bash - pip install infinity-sdk==0.3.0.dev8 + pip install infinity-sdk==0.3.0 ``` ```python diff --git a/benchmark/remote_infinity/remote_query_benchmark.cpp b/benchmark/remote_infinity/remote_query_benchmark.cpp index 8f6aef8751..dfcdb0617d 100644 --- a/benchmark/remote_infinity/remote_query_benchmark.cpp +++ b/benchmark/remote_infinity/remote_query_benchmark.cpp @@ -51,7 +51,7 @@ struct InfinityClient { transport->open(); CommonResponse response; ConnectRequest request; - request.__set_client_version(17); // 0.3.0.dev8 + request.__set_client_version(17); // 0.3.0 client->Connect(response, request); session_id = response.session_id; } diff --git a/client/cpp/infinity_client.cpp b/client/cpp/infinity_client.cpp index a1f08883bd..98b6a55a40 100644 --- a/client/cpp/infinity_client.cpp +++ b/client/cpp/infinity_client.cpp @@ -25,7 +25,7 @@ Client Client::Connect(const std::string &ip_address, uint16_t port) { transport->open(); CommonResponse response; ConnectRequest request; - request.__set_client_version(17); // 0.3.0.dev8 + request.__set_client_version(17); // 0.3.0 client->Connect(response, request); return {socket, transport, protocol, std::move(client), response.session_id}; } diff --git a/conf/infinity_conf.toml b/conf/infinity_conf.toml index 4807f0efa3..da65c8708b 100644 --- a/conf/infinity_conf.toml +++ b/conf/infinity_conf.toml @@ -20,7 +20,7 @@ log_file_rotate_count = 10 log_level = "info" [storage] -persistence_dir = "/var/infinity/persistence" +#persistence_dir = "/var/infinity/persistence" data_dir = "/var/infinity/data" # periodically activates garbage collection: # 0 means real-time, diff --git a/docs/getstarted/deploy_infinity_server.mdx b/docs/getstarted/deploy_infinity_server.mdx index 63b871bca7..c0289315f1 100644 --- a/docs/getstarted/deploy_infinity_server.mdx +++ b/docs/getstarted/deploy_infinity_server.mdx @@ -30,7 +30,7 @@ This approach allows you to embed Infinity as a module in a Python application. ### Install Infinity as a module ``` -pip install infinity-sdk==0.3.0.dev8 +pip install infinity-sdk==0.3.0 ``` ### Create an Infinity object @@ -98,7 +98,7 @@ If you are on Windows 10+, you must enable WSL or WSL2 to deploy Infinity using ### Install Infinity client ``` -pip install infinity-sdk==0.3.0.dev8 +pip install infinity-sdk==0.3.0 ``` ### Connect to Infinity Server @@ -140,7 +140,7 @@ This section provides instructions on deploying Infinity using binary package on Fedora/RHEL/CentOS/OpenSUSE ```bash -sudo rpm -i infinity-0.3.0.dev8-x86_64.rpm +sudo rpm -i infinity-0.3.0-x86_64.rpm ``` ```bash @@ -151,7 +151,7 @@ sudo systemctl start infinity ```bash -sudo dpkg -i infinity-0.3.0.dev8-x86_64.deb +sudo dpkg -i infinity-0.3.0-x86_64.deb ``` ```bash @@ -172,7 +172,7 @@ sudo systemctl start infinity ### Install Infinity client ``` -pip install infinity-sdk==0.3.0.dev8 +pip install infinity-sdk==0.3.0 ``` ### Connect to Infinity Server diff --git a/docs/getstarted/quickstart.md b/docs/getstarted/quickstart.md index d5b82c1ec8..1a6e43e1e6 100644 --- a/docs/getstarted/quickstart.md +++ b/docs/getstarted/quickstart.md @@ -17,7 +17,7 @@ Infinity, also available as a Python module, eliminates the need for a separate 1. Install Infinity as a Python module: ```bash - pip install infinity-sdk==0.3.0.dev8 + pip install infinity-sdk==0.3.0 ``` 2. Use Infinity to conduct a KNN search: ```python diff --git a/python/infinity/remote_thrift/client.py b/python/infinity/remote_thrift/client.py index 1693d280ed..846c51a8f2 100644 --- a/python/infinity/remote_thrift/client.py +++ b/python/infinity/remote_thrift/client.py @@ -63,8 +63,8 @@ def reconnect(self): # version: 0.3.0.dev5, client_version: 14 # version: 0.3.0.dev6, client_version: 15 # version: 0.3.0.dev7, client_version: 16 - # version: 0.3.0.dev8, client_version: 17 - res = self.client.Connect(ConnectRequest(client_version=17)) # 0.3.0.dev8 + # version: 0.3.0.dev8 and 0.3.0, client_version: 17 + res = self.client.Connect(ConnectRequest(client_version=17)) # 0.3.0 if res.error_code != 0: raise InfinityException(res.error_code, res.error_msg) self.session_id = res.session_id diff --git a/python/test_pysdk/test_basic.py b/python/test_pysdk/test_basic.py index 24e1b2f259..c682a83477 100644 --- a/python/test_pysdk/test_basic.py +++ b/python/test_pysdk/test_basic.py @@ -133,8 +133,13 @@ def test_basic(self, check_data ,suffix): res = table_obj.create_index("my_index", index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK diff --git a/python/test_pysdk/test_index.py b/python/test_pysdk/test_index.py index 5de90062d3..95a5cdf49f 100644 --- a/python/test_pysdk/test_index.py +++ b/python/test_pysdk/test_index.py @@ -1,5 +1,6 @@ import sys import os + current_dir = os.path.dirname(os.path.abspath(__file__)) parent_dir = os.path.dirname(current_dir) if parent_dir not in sys.path: @@ -45,11 +46,13 @@ def setup_class(request, local_infinity, http): @pytest.mark.usefixtures("setup_class") @pytest.mark.usefixtures("suffix") class TestInfinity: + + @pytest.mark.skip(reason="Disable IVF index") def test_create_index_IVFFlat(self, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index_ivfflat"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index_ivfflat" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_index_ivfflat"+suffix, { + table_obj = db_obj.create_table("test_index_ivfflat" + suffix, { "c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -62,16 +65,16 @@ def test_create_index_IVFFlat(self, suffix): res = table_obj.drop_index("my_index", ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index_ivfflat"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index_ivfflat" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_create_index_HNSW(self, suffix): # CREATE INDEX idx1 ON test_hnsw (col1) USING Hnsw WITH (M = 16, ef_construction = 50, ef = 50, metric = l2); db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index_hnsw"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index_hnsw" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK table_obj = db_obj.create_table( - "test_index_hnsw"+suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) + "test_index_hnsw" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None res = table_obj.create_index("my_index", @@ -87,17 +90,18 @@ def test_create_index_HNSW(self, suffix): res = table_obj.drop_index("my_index") assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index_hnsw"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index_hnsw" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("block_size", [8, 128]) @pytest.mark.parametrize("compress_type", ["compress", "raww"]) def test_create_index_BMP(self, block_size, compress_type, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_bmp"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_bmp" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK table_obj = db_obj.create_table( - "test_bmp"+suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, ConflictType.Error) + "test_bmp" + suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, + ConflictType.Error) assert table_obj is not None # CREATE INDEX idx1 ON test_bmp (col2) USING Bmp WITH (block_size = 16, compress_type = compress); @@ -110,16 +114,16 @@ def test_create_index_BMP(self, block_size, compress_type, suffix): res = table_obj.drop_index("idx1") assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_bmp"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_bmp" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_create_index_fulltext(self, suffix): # CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT; db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index_fulltext"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index_fulltext" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK table_obj = db_obj.create_table( - "test_index_fulltext"+suffix, { + "test_index_fulltext" + suffix, { "doctitle": {"type": "varchar"}, "docdate": {"type": "varchar"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -133,7 +137,7 @@ def test_create_index_fulltext(self, suffix): res = table_obj.drop_index("my_index") assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index_fulltext"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index_fulltext" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_drop_index_fulltext(self, suffix): @@ -163,13 +167,13 @@ def test_drop_index_fulltext(self, suffix): res = db_obj.drop_table("test_index_fulltext_drop" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK - def test_create_index_secondary(self ,suffix): + def test_create_index_secondary(self, suffix): # CREATE INDEX idx_secondary ON t(c1); db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index_secondary"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index_secondary" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK table_obj = db_obj.create_table( - "test_index_secondary"+suffix, { + "test_index_secondary" + suffix, { "c1": {"type": "int"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -187,7 +191,7 @@ def test_create_index_secondary(self ,suffix): assert res.error_code == ErrorCode.OK res = table_obj.drop_index("my_index_2") assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index_secondary"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index_secondary" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK # drop non-existent index @@ -195,10 +199,10 @@ def test_create_index_secondary(self ,suffix): def test_create_index_emvb(self, suffix): # CREATE INDEX idx_emvb ON t(c2) USING EMVB; db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index_emvb"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index_emvb" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK table_obj = db_obj.create_table( - "test_index_emvb"+suffix, { + "test_index_emvb" + suffix, { "c1": {"type": "int"}, "c2": {"type": "tensor, 128, float"} }, ConflictType.Error) assert table_obj is not None @@ -210,17 +214,17 @@ def test_create_index_emvb(self, suffix): assert res.error_code == ErrorCode.OK res = table_obj.drop_index("my_index_1") assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index_emvb"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index_emvb" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_drop_non_existent_index(self, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_drop_non_existent_index"+suffix, ConflictType.Ignore) + "test_drop_non_existent_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_drop_non_existent_index"+suffix, { + table_obj = db_obj.create_table("test_drop_non_existent_index" + suffix, { "c1": {"type": "vector,3,float"}}, ConflictType.Error) assert table_obj is not None @@ -232,17 +236,17 @@ def test_drop_non_existent_index(self, suffix): assert e.value.args[0] == ErrorCode.INDEX_NOT_EXIST res = db_obj.drop_table( - "test_drop_non_existent_index"+suffix, ConflictType.Error) + "test_drop_non_existent_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_create_created_index(self, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_drop_non_existent_index"+suffix, ConflictType.Ignore) + "test_drop_non_existent_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_drop_non_existent_index"+suffix, { + table_obj = db_obj.create_table("test_drop_non_existent_index" + suffix, { "c1": {"type": "vector,3,float"}}, ConflictType.Error) assert table_obj is not None @@ -254,7 +258,7 @@ def test_create_created_index(self, suffix): assert e.value.args[0] == ErrorCode.INDEX_NOT_EXIST res = db_obj.drop_table( - "test_drop_non_existent_index"+suffix, ConflictType.Error) + "test_drop_non_existent_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("column_name", @@ -267,7 +271,7 @@ def test_create_created_index(self, suffix): ((1, 2), False), ({"1": 2}, False), (index.IndexType.Hnsw, False), - (index.IndexType.IVFFlat, True) + # (index.IndexType.IVFFlat, False) ]) @pytest.mark.parametrize("params", [ (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((1, 2), False), ({"1": 2}, False), @@ -278,9 +282,10 @@ def test_create_drop_vector_index_invalid_options(self, column_name, index_type, # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_drop_vector_index_invalid_options"+suffix, ConflictType.Ignore) + "test_create_drop_vector_index_invalid_options" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_drop_vector_index_invalid_options"+suffix, {"c1": {"type": types}}, + table_obj = db_obj.create_table("test_create_drop_vector_index_invalid_options" + suffix, + {"c1": {"type": types}}, ConflictType.Error) if not column_name[1] or not index_type[1] or not params[1]: @@ -293,7 +298,7 @@ def test_create_drop_vector_index_invalid_options(self, column_name, index_type, table_obj.create_index("my_index", index_info, ConflictType.Error) res = db_obj.drop_table( - "test_create_drop_vector_index_invalid_options"+suffix, ConflictType.Error) + "test_create_drop_vector_index_invalid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("column_name", [ @@ -317,10 +322,10 @@ def test_create_drop_different_fulltext_index_invalid_options(self, column_name, # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_drop_different_fulltext_index_invalid_options"+suffix, ConflictType.Ignore) + "test_create_drop_different_fulltext_index_invalid_options" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_drop_different_fulltext_index_invalid_options"+suffix, { + table_obj = db_obj.create_table("test_create_drop_different_fulltext_index_invalid_options" + suffix, { "c1": {"type": types}}, ConflictType.Error) if types != "varchar" or not column_name[1] or not index_type[1] or not params[1]: @@ -332,27 +337,33 @@ def test_create_drop_different_fulltext_index_invalid_options(self, column_name, table_obj.create_index("my_index", index_info, ConflictType.Error) res = db_obj.drop_table( - "test_create_drop_different_fulltext_index_invalid_options"+suffix, ConflictType.Error) + "test_create_drop_different_fulltext_index_invalid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_create_index_on_dropped_table(self, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_drop_index_invalid_options"+suffix, ConflictType.Ignore) + "test_create_drop_index_invalid_options" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_drop_index_invalid_options"+suffix, { + table_obj = db_obj.create_table("test_create_drop_index_invalid_options" + suffix, { "c1": {"type": "vector,3,float"}}, ConflictType.Error) db_obj.drop_table( - "test_create_drop_index_invalid_options"+suffix) + "test_create_drop_index_invalid_options" + suffix) # create created index with pytest.raises(InfinityException) as e: table_obj.create_index("my_index", index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), + ConflictType.Error) assert e.type == InfinityException assert e.value.args[0] == ErrorCode.TABLE_NOT_EXIST @@ -361,37 +372,48 @@ def test_create_index_show_index(self, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_index_show_index"+suffix, ConflictType.Ignore) + "test_create_index_show_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_index_show_index"+suffix, { + table_obj = db_obj.create_table("test_create_index_show_index" + suffix, { "c1": {"type": "vector,3,float"}}, ConflictType.Error) # create created index res = table_obj.create_index("my_index", index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), + ConflictType.Error) assert res.error_code == ErrorCode.OK res = table_obj.show_index("my_index") print(res) res = db_obj.drop_table( - "test_create_index_show_index"+suffix, ConflictType.Error) + "test_create_index_show_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_drop_index_show_index(self, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_drop_index_show_index"+suffix, ConflictType.Ignore) + "test_drop_index_show_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_drop_index_show_index"+suffix, { + table_obj = db_obj.create_table("test_drop_index_show_index" + suffix, { "c1": {"type": "vector,3,float"}}, ConflictType.Error) # create created index res = table_obj.create_index("my_index", index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK res = table_obj.show_index("my_index") @@ -404,22 +426,23 @@ def test_drop_index_show_index(self, suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_drop_index_show_index"+suffix, ConflictType.Error) + "test_drop_index_show_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("types", ["vector, 3, float"]) @pytest.mark.parametrize("index_type", [ (index.IndexType.Hnsw, False, ErrorCode.INVALID_INDEX_PARAM), - (index.IndexType.IVFFlat, True), + # (index.IndexType.IVFFlat, True), (index.IndexType.FullText, False, ErrorCode.INVALID_INDEX_DEFINITION) ]) def test_create_index_on_different_type_of_column(self, types, index_type, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_create_index_on_different_type_of_column"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_create_index_on_different_type_of_column" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_index_on_different_type_of_column"+suffix, {"c1": {"type": types}}, + table_obj = db_obj.create_table("test_create_index_on_different_type_of_column" + suffix, + {"c1": {"type": types}}, ConflictType.Error) # create created index @@ -440,17 +463,17 @@ def test_create_index_on_different_type_of_column(self, types, index_type, suffi ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_create_index_on_different_type_of_column"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_create_index_on_different_type_of_column" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_type", [ - index.IndexType.IVFFlat + index.IndexType.Hnsw ]) def test_insert_data_create_index(self, index_type, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_insert_data_create_index"+suffix, ConflictType.Ignore) - table_obj = db_obj.create_table("test_insert_data_create_index"+suffix, { + db_obj.drop_table("test_insert_data_create_index" + suffix, ConflictType.Ignore) + table_obj = db_obj.create_table("test_insert_data_create_index" + suffix, { "c1": {"type": "vector,1024,float"}}, ConflictType.Error) values = [{"c1": [1.1 for _ in range(1024)]}] table_obj.insert(values) @@ -458,14 +481,19 @@ def test_insert_data_create_index(self, index_type, suffix): res = table_obj.create_index("my_index", index.IndexInfo("c1", index_type, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_insert_data_create_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_insert_data_create_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_type", [ - (index.IndexType.IVFFlat, True), + (index.IndexType.Hnsw, True), (index.IndexType.FullText, False, ErrorCode.INVALID_INDEX_DEFINITION) ]) @pytest.mark.parametrize("file_format", ["csv"]) @@ -473,10 +501,10 @@ def test_import_data_create_index(self, index_type, file_format, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_import_data_create_index"+suffix, ConflictType.Ignore) + "test_import_data_create_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_import_data_create_index"+suffix, { + table_obj = db_obj.create_table("test_import_data_create_index" + suffix, { "c1": {"type": "int"}, "c2": {"type": "vector,3,float"}}, ConflictType.Error) @@ -486,7 +514,12 @@ def test_import_data_create_index(self, index_type, file_format, suffix): res = table_obj.create_index("my_index", index.IndexInfo("c2", index_type[0], - {"centroids_count": "128", "metric": "l2"}), + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK else: @@ -500,31 +533,36 @@ def test_import_data_create_index(self, index_type, file_format, suffix): assert e.value.args[0] == index_type[2] res = db_obj.drop_table( - "test_import_data_create_index"+suffix, ConflictType.Error) + "test_import_data_create_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK - @pytest.mark.parametrize("index_type", [index.IndexType.IVFFlat]) + @pytest.mark.parametrize("index_type", [index.IndexType.Hnsw]) @pytest.mark.parametrize("file_format", ["csv"]) def test_create_vector_index_import_data(self, index_type, file_format, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_vector_index_import_data"+suffix, ConflictType.Ignore) + "test_create_vector_index_import_data" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_vector_index_import_data"+suffix, { + table_obj = db_obj.create_table("test_create_vector_index_import_data" + suffix, { "c1": {"type": "int"}, "c2": {"type": "vector,3,float"}}, ConflictType.Error) res = table_obj.create_index("my_index", index.IndexInfo("c2", index_type, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK table_obj.import_data(os.getcwd() + TEST_DATA_DIR + file_format + "/pysdk_test." + file_format) res = db_obj.drop_table( - "test_create_vector_index_import_data"+suffix, ConflictType.Error) + "test_create_vector_index_import_data" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_type", [ @@ -535,9 +573,9 @@ def test_create_index_import_data(self, index_type, file_format, suffix): # connect db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_create_index_import_data"+suffix, ConflictType.Ignore) + "test_create_index_import_data" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_create_index_import_data"+suffix, { + table_obj = db_obj.create_table("test_create_index_import_data" + suffix, { "c1": {"type": "int"}, "c2": {"type": "vector,3,float"}}, ConflictType.Error) @@ -556,7 +594,7 @@ def test_create_index_import_data(self, index_type, file_format, suffix): file_format + "/pysdk_test." + file_format) res = db_obj.drop_table( - "test_create_index_import_data"+suffix, ConflictType.Error) + "test_create_index_import_data" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("file_format", ["csv"]) @@ -572,10 +610,10 @@ def test_insert_data_fulltext_index_search(self, file_format, suffix): db_obj = self.infinity_obj.get_database("default_db") res = db_obj.drop_table( - "test_insert_data_fulltext_index_search"+suffix, ConflictType.Ignore) + "test_insert_data_fulltext_index_search" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_insert_data_fulltext_index_search"+suffix, { + table_obj = db_obj.create_table("test_insert_data_fulltext_index_search" + suffix, { "doctitle": {"type": "varchar"}, "docdate": {"type": "varchar"}, "body": {"type": "varchar"} }, ConflictType.Error) res = table_obj.create_index("body_index", @@ -597,17 +635,17 @@ def test_insert_data_fulltext_index_search(self, file_format, suffix): print(res) res = db_obj.drop_table( - "test_insert_data_fulltext_index_search"+suffix, ConflictType.Error) + "test_insert_data_fulltext_index_search" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("check_data", [{"file_name": "enwiki_9.csv", "data_dir": common_values.TEST_TMP_DIR, }], indirect=True) def test_fulltext_match_with_invalid_analyzer(self, check_data, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_fulltext_match_with_invalid_analyzer"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_fulltext_match_with_invalid_analyzer" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_fulltext_match_with_invalid_analyzer"+suffix, { + table_obj = db_obj.create_table("test_fulltext_match_with_invalid_analyzer" + suffix, { "doctitle": {"type": "varchar"}, "docdate": {"type": "varchar"}, "body": {"type": "varchar"}}) assert res.error_code == ErrorCode.OK @@ -621,15 +659,16 @@ def test_fulltext_match_with_invalid_analyzer(self, check_data, suffix): assert e.type == InfinityException assert e.value.args[0] == ErrorCode.ANALYZER_NOT_FOUND - res = db_obj.drop_table("test_fulltext_match_with_invalid_analyzer"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_fulltext_match_with_invalid_analyzer" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK def test_create_index_on_deleted_table(self, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_index_on_deleted_table"+suffix, ConflictType.Ignore) + "test_create_index_on_deleted_table" + suffix, ConflictType.Ignore) - table_obj = db_obj.create_table("test_create_index_on_deleted_table"+suffix, {"c1": {"type": "vector,128,float"}}, + table_obj = db_obj.create_table("test_create_index_on_deleted_table" + suffix, + {"c1": {"type": "vector,128,float"}}, ConflictType.Error) # insert data embedding_data = [i for i in range(128)] @@ -646,23 +685,28 @@ def test_create_index_on_deleted_table(self, suffix): # create index res = table_obj.create_index("my_index", index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK res = table_obj.drop_index("my_index", ConflictType.Error) assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_create_index_on_deleted_table"+suffix, ConflictType.Error) + "test_create_index_on_deleted_table" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.skip @pytest.mark.xfail(reason="Not support to convert Embedding to Embedding") def test_create_index_on_update_table(self, suffix): db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_create_index_on_update_table"+suffix, + db_obj.drop_table("test_create_index_on_update_table" + suffix, ConflictType.Ignore) - table_obj = db_obj.create_table("test_create_index_on_update_table"+suffix, { + table_obj = db_obj.create_table("test_create_index_on_update_table" + suffix, { "c1": {"type": "vector,128,float"}, "c2": {"type": "int"} }, ConflictType.Error) @@ -682,7 +726,7 @@ def test_create_index_on_update_table(self, suffix): res = table_obj.output(["*"]).to_pl() print(res) res = db_obj.drop_table( - "test_create_index_on_update_table"+suffix, ConflictType.Error) + "test_create_index_on_update_table" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [ConflictType.Ignore, @@ -695,9 +739,9 @@ def test_create_index_on_update_table(self, suffix): def test_create_index_with_valid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_index_with_valid_options"+suffix, ConflictType.Ignore) + "test_create_index_with_valid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_create_index_with_valid_options"+suffix, + "test_create_index_with_valid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -715,7 +759,7 @@ def test_create_index_with_valid_options(self, conflict_type, suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_create_index_with_valid_options"+suffix, ConflictType.Error) + "test_create_index_with_valid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [pytest.param(1.1), @@ -727,9 +771,9 @@ def test_create_index_with_valid_options(self, conflict_type, suffix): def test_create_index_with_invalid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_index_with_invalid_options"+suffix, ConflictType.Ignore) + "test_create_index_with_invalid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_create_index_with_invalid_options"+suffix, + "test_create_index_with_invalid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -747,7 +791,7 @@ def test_create_index_with_invalid_options(self, conflict_type, suffix): # assert e.value.args[0] == ErrorCode.INVALID_CONFLICT_TYPE res = db_obj.drop_table( - "test_create_index_with_invalid_options"+suffix, ConflictType.Error) + "test_create_index_with_invalid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [ConflictType.Ignore, @@ -755,9 +799,9 @@ def test_create_index_with_invalid_options(self, conflict_type, suffix): def test_create_duplicated_index_with_valid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_duplicated_index_with_valid_options"+suffix, ConflictType.Ignore) + "test_create_duplicated_index_with_valid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_create_duplicated_index_with_valid_options"+suffix, + "test_create_duplicated_index_with_valid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None for i in range(10): @@ -776,7 +820,7 @@ def test_create_duplicated_index_with_valid_options(self, conflict_type, suffix) assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_create_duplicated_index_with_valid_options"+suffix, ConflictType.Error) + "test_create_duplicated_index_with_valid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [pytest.param(ConflictType.Error), @@ -788,9 +832,9 @@ def test_create_duplicated_index_with_valid_options(self, conflict_type, suffix) def test_create_duplicated_index_with_valid_error_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_duplicated_index_with_valid_error_options"+suffix, ConflictType.Ignore) + "test_create_duplicated_index_with_valid_error_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_create_duplicated_index_with_valid_error_options"+suffix, + "test_create_duplicated_index_with_valid_error_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -822,7 +866,7 @@ def test_create_duplicated_index_with_valid_error_options(self, conflict_type, s assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_create_duplicated_index_with_valid_error_options"+suffix, ConflictType.Error) + "test_create_duplicated_index_with_valid_error_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [pytest.param(1.1), @@ -834,9 +878,9 @@ def test_create_duplicated_index_with_valid_error_options(self, conflict_type, s def test_create_duplicated_index_with_invalid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_create_duplicated_index_with_invalid_options"+suffix, ConflictType.Ignore) + "test_create_duplicated_index_with_invalid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_create_duplicated_index_with_invalid_options"+suffix, + "test_create_duplicated_index_with_invalid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -869,14 +913,14 @@ def test_create_duplicated_index_with_invalid_options(self, conflict_type, suffi # assert e.value.args[0] == ErrorCode.INVALID_CONFLICT_TYPE res = db_obj.drop_table( - "test_create_duplicated_index_with_invalid_options"+suffix, ConflictType.Error) + "test_create_duplicated_index_with_invalid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_show_index(self, suffix): db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_show_index"+suffix, ConflictType.Ignore) + db_obj.drop_table("test_show_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_show_index"+suffix, + "test_show_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None for i in range(10): @@ -897,15 +941,15 @@ def test_show_index(self, suffix): "my_index_" + str(i), ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_show_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_show_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_name", ["my_index"]) def test_show_valid_name_index(self, index_name, suffix): db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_show_various_name_index"+suffix, ConflictType.Ignore) + db_obj.drop_table("test_show_various_name_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_show_various_name_index"+suffix, + "test_show_various_name_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None res = table_obj.create_index("my_index", @@ -924,7 +968,7 @@ def test_show_valid_name_index(self, index_name, suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_show_various_name_index"+suffix, ConflictType.Error) + "test_show_various_name_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_name", [pytest.param("Invalid name"), @@ -938,9 +982,9 @@ def test_show_valid_name_index(self, index_name, suffix): ]) def test_show_invalid_name_index(self, index_name, suffix): db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_show_invalid_name_index"+suffix, ConflictType.Ignore) + db_obj.drop_table("test_show_invalid_name_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_show_invalid_name_index"+suffix, + "test_show_invalid_name_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -958,14 +1002,14 @@ def test_show_invalid_name_index(self, index_name, suffix): print(res) res = db_obj.drop_table( - "test_show_invalid_name_index"+suffix, ConflictType.Error) + "test_show_invalid_name_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_list_index(self, suffix): db_obj = self.infinity_obj.get_database("default_db") - db_obj.drop_table("test_list_index"+suffix, ConflictType.Ignore) + db_obj.drop_table("test_list_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_list_index"+suffix, + "test_list_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None for i in range(10): @@ -987,7 +1031,7 @@ def test_list_index(self, suffix): "my_index_" + str(i), ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_list_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_list_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [ConflictType.Ignore, @@ -997,9 +1041,9 @@ def test_list_index(self, suffix): def test_drop_index_with_valid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_drop_index_with_valid_options"+suffix, ConflictType.Ignore) + "test_drop_index_with_valid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_drop_index_with_valid_options"+suffix, + "test_drop_index_with_valid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1018,7 +1062,7 @@ def test_drop_index_with_valid_options(self, conflict_type, suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_drop_index_with_valid_options"+suffix, ConflictType.Error) + "test_drop_index_with_valid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("conflict_type", [pytest.param(1.1), @@ -1030,9 +1074,9 @@ def test_drop_index_with_valid_options(self, conflict_type, suffix): def test_drop_index_with_invalid_options(self, conflict_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_drop_index_with_invalid_options"+suffix, ConflictType.Ignore) + "test_drop_index_with_invalid_options" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_drop_index_with_invalid_options"+suffix, + "test_drop_index_with_invalid_options" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1054,17 +1098,17 @@ def test_drop_index_with_invalid_options(self, conflict_type, suffix): # assert e.value.args[0] == ErrorCode.INVALID_CONFLICT_TYPE res = db_obj.drop_table( - "test_drop_index_with_invalid_options"+suffix, ConflictType.Error) + "test_drop_index_with_invalid_options" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_distance_type", ["l2", "ip"]) - def test_supported_vector_index(self, index_distance_type,suffix): + def test_supported_vector_index(self, index_distance_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_supported_vector_index"+suffix, ConflictType.Ignore) + "test_supported_vector_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_supported_vector_index"+suffix, + "test_supported_vector_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1084,16 +1128,16 @@ def test_supported_vector_index(self, index_distance_type,suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_supported_vector_index"+suffix, ConflictType.Error) + "test_supported_vector_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_distance_type", ["hamming"]) def test_unsupported_vector_index(self, index_distance_type, suffix): db_obj = self.infinity_obj.get_database("default_db") db_obj.drop_table( - "test_unsupported_vector_index"+suffix, ConflictType.Ignore) + "test_unsupported_vector_index" + suffix, ConflictType.Ignore) table_obj = db_obj.create_table( - "test_unsupported_vector_index"+suffix, + "test_unsupported_vector_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1115,14 +1159,14 @@ def test_unsupported_vector_index(self, index_distance_type, suffix): assert res.error_code == ErrorCode.OK res = db_obj.drop_table( - "test_unsupported_vector_index"+suffix, ConflictType.Error) + "test_unsupported_vector_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK def test_create_upper_name_index(self, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_upper_name_index"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_upper_name_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK - table_obj = db_obj.create_table("test_upper_name_index"+suffix, { + table_obj = db_obj.create_table("test_upper_name_index" + suffix, { "c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1130,8 +1174,13 @@ def test_create_upper_name_index(self, suffix): lower_name_index = "my_index" res = table_obj.create_index(upper_name_index, index.IndexInfo("c1", - index.IndexType.IVFFlat, - {"centroids_count": "128", "metric": "l2"}), ConflictType.Error) + index.IndexType.Hnsw, + { + "m": "16", + "ef_construction": "200", + "metric": "l2", + "encode": "lvq" + }), ConflictType.Error) assert res.error_code == ErrorCode.OK res = table_obj.show_index(lower_name_index) @@ -1141,11 +1190,11 @@ def test_create_upper_name_index(self, suffix): res = table_obj.drop_index(lower_name_index, ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_upper_name_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_upper_name_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_type", [ - index.IndexType.IVFFlat, + # index.IndexType.IVFFlat, index.IndexType.Hnsw, index.IndexType.BMP, index.IndexType.FullText, @@ -1154,11 +1203,11 @@ def test_create_upper_name_index(self, suffix): ]) def test_create_index_with_converse_param_name(self, index_type, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK if index_type == index.IndexType.IVFFlat: - table_obj = db_obj.create_table("test_index"+suffix, { + table_obj = db_obj.create_table("test_index" + suffix, { "c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1170,7 +1219,7 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.Hnsw: table_obj = db_obj.create_table( - "test_index"+suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) + "test_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None res = table_obj.create_index("my_index", @@ -1185,7 +1234,7 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.BMP: table_obj = db_obj.create_table( - "test_index"+suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, + "test_index" + suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, ConflictType.Error) assert table_obj is not None @@ -1198,7 +1247,7 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.FullText: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "doctitle": {"type": "varchar"}, "docdate": {"type": "varchar"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -1212,7 +1261,7 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.EMVB: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "c1": {"type": "int"}, "c2": {"type": "tensor, 128, float"} }, ConflictType.Error) assert table_obj is not None @@ -1224,7 +1273,7 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.Secondary: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "c1": {"type": "int"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -1239,11 +1288,11 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): res = table_obj.drop_index("my_index", ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK @pytest.mark.parametrize("index_type", [ - index.IndexType.IVFFlat, + # index.IndexType.IVFFlat, index.IndexType.Hnsw, index.IndexType.BMP, index.IndexType.FullText, @@ -1252,10 +1301,10 @@ def test_create_index_with_converse_param_name(self, index_type, suffix): ]) def test_create_index_with_converse_param_value(self, index_type, suffix): db_obj = self.infinity_obj.get_database("default_db") - res = db_obj.drop_table("test_index"+suffix, ConflictType.Ignore) + res = db_obj.drop_table("test_index" + suffix, ConflictType.Ignore) assert res.error_code == ErrorCode.OK if index_type == index.IndexType.IVFFlat: - table_obj = db_obj.create_table("test_index"+suffix, { + table_obj = db_obj.create_table("test_index" + suffix, { "c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None @@ -1267,7 +1316,7 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.Hnsw: table_obj = db_obj.create_table( - "test_index"+suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) + "test_index" + suffix, {"c1": {"type": "vector,1024,float"}}, ConflictType.Error) assert table_obj is not None res = table_obj.create_index("my_index", @@ -1282,7 +1331,7 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.BMP: table_obj = db_obj.create_table( - "test_index"+suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, + "test_index" + suffix, {"col1": {"type": "int"}, "col2": {"type": "sparse,30000,float,int16"}}, ConflictType.Error) assert table_obj is not None @@ -1295,7 +1344,7 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.FullText: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "doctitle": {"type": "varchar"}, "docdate": {"type": "varchar"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -1309,7 +1358,7 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.EMVB: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "c1": {"type": "int"}, "c2": {"type": "tensor, 128, float"} }, ConflictType.Error) assert table_obj is not None @@ -1321,7 +1370,7 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): assert res.error_code == ErrorCode.OK elif index_type == index.IndexType.Secondary: table_obj = db_obj.create_table( - "test_index"+suffix, { + "test_index" + suffix, { "c1": {"type": "int"}, "body": {"type": "varchar"} }, ConflictType.Error) assert table_obj is not None @@ -1336,5 +1385,5 @@ def test_create_index_with_converse_param_value(self, index_type, suffix): res = table_obj.drop_index("my_index", ConflictType.Error) assert res.error_code == ErrorCode.OK - res = db_obj.drop_table("test_index"+suffix, ConflictType.Error) + res = db_obj.drop_table("test_index" + suffix, ConflictType.Error) assert res.error_code == ErrorCode.OK diff --git a/python/test_pysdk/test_knn.py b/python/test_pysdk/test_knn.py index 83f4e92ed5..d338b2f3ac 100644 --- a/python/test_pysdk/test_knn.py +++ b/python/test_pysdk/test_knn.py @@ -997,7 +997,7 @@ def test_with_multiple_fusion(self, check_data, suffix): @pytest.mark.parametrize("knn_column_name", ["gender_vector"]) @pytest.mark.parametrize("index_distance_type", ["l2", "ip", "cosine", "cos"]) @pytest.mark.parametrize("knn_distance_type", ["l2", "ip", "cosine", "cos"]) - @pytest.mark.parametrize("index_type", [index.IndexType.Hnsw, index.IndexType.IVFFlat]) # Remove index.IndexType.IVFFlat, after IVFFlat support cosine metric + @pytest.mark.parametrize("index_type", [index.IndexType.Hnsw]) # Remove index.IndexType.IVFFlat, after IVFFlat support cosine metric def test_with_various_index_knn_distance_combination(self, check_data, index_column_name, knn_column_name, index_distance_type, knn_distance_type, index_type, suffix): db_obj = self.infinity_obj.get_database("default_db") @@ -1340,7 +1340,7 @@ def test_sparse_with_invalid_table_params(self, check_data, table_params ,suffix ConflictType.Error) assert e.value.args[0] == ErrorCode.INVALID_EMBEDDING_DATA_TYPE - @pytest.mark.parametrize("index_type", [index.IndexType.IVFFlat, + @pytest.mark.parametrize("index_type", [#index.IndexType.IVFFlat, index.IndexType.Hnsw, index.IndexType.EMVB, index.IndexType.FullText, diff --git a/src/common/status.cpp b/src/common/status.cpp index 028c69c2c2..e7b62888a6 100644 --- a/src/common/status.cpp +++ b/src/common/status.cpp @@ -344,7 +344,13 @@ Status Status::InvalidConstantType() { return Status(ErrorCode::kInvalidConstant Status Status::InvalidParsedExprType() { return Status(ErrorCode::kInvalidParsedExprType, MakeUnique("Invalid parsed expression type.")); } -Status Status::InvalidIndexType() { return Status(ErrorCode::kInvalidIndexType, MakeUnique("Invalid index type.")); } +Status Status::InvalidIndexType(const String& message) { + if(message.empty()) { + return Status(ErrorCode::kInvalidIndexType, MakeUnique("No index type is given")); + } else { + return Status(ErrorCode::kInvalidIndexType, MakeUnique(fmt::format("Invalid index type: ", message))); + } +} Status Status::InvalidIndexParam(const String ¶m_name) { return Status(ErrorCode::kInvalidIndexParam, MakeUnique(fmt::format("Invalid index parameter type: {}", param_name))); diff --git a/src/common/status.cppm b/src/common/status.cppm index 192ecae29e..808fd9ec72 100644 --- a/src/common/status.cppm +++ b/src/common/status.cppm @@ -262,7 +262,7 @@ public: static Status InvalidEmbeddingDataType(const String& type_str); static Status InvalidConstantType(); static Status InvalidParsedExprType(); - static Status InvalidIndexType(); + static Status InvalidIndexType(const String& message); static Status InvalidIndexParam(const String ¶m_name); static Status LackIndexParam(); static Status InvalidFilterExpression(const String &expr); diff --git a/src/executor/operator/physical_scan/physical_knn_scan.cpp b/src/executor/operator/physical_scan/physical_knn_scan.cpp index 3f48df62b6..c65cd147aa 100644 --- a/src/executor/operator/physical_scan/physical_knn_scan.cpp +++ b/src/executor/operator/physical_scan/physical_knn_scan.cpp @@ -417,7 +417,7 @@ void PhysicalKnnScan::PlanWithIndex(QueryContext *query_context) { // TODO: retu if (auto index_type = table_index_entry->index_base()->index_type_; index_type != IndexType::kIVFFlat and index_type != IndexType::kHnsw) { LOG_ERROR("Invalid index type"); - Status error_status = Status::InvalidIndexType(); + Status error_status = Status::InvalidIndexType("invalid index"); RecoverableError(std::move(error_status)); } diff --git a/src/network/http_server.cpp b/src/network/http_server.cpp index a258030019..bdfa111ad9 100644 --- a/src/network/http_server.cpp +++ b/src/network/http_server.cpp @@ -2507,6 +2507,7 @@ class CreateIndexHandler final : public HttpRequestHandler { }); { index_info->column_name_ = fields[0]; + ToLower(index_info->column_name_); auto index_param_list = new Vector(); DeferFn release_index_param_list([&]() { if(index_param_list != nullptr) { @@ -2520,13 +2521,16 @@ class CreateIndexHandler final : public HttpRequestHandler { for (auto &ele : index.items()) { String name = ele.key(); + ToLower(name); auto value = ele.value(); if (!ele.value().is_string()) { value = ele.value().dump(); } if (strcmp(name.c_str(), "type") == 0) { - index_info->index_type_ = IndexInfo::StringToIndexType(value); + String version_str = value; + ToUpper(version_str); + index_info->index_type_ = IndexInfo::StringToIndexType(version_str); if (index_info->index_type_ == IndexType::kInvalid) { json_response["error_code"] = ErrorCode::kInvalidIndexType; json_response["error_message"] = fmt::format("Invalid index type: {}", name); diff --git a/src/network/infinity_thrift_service.cpp b/src/network/infinity_thrift_service.cpp index c11ee75372..4f77aa4f71 100644 --- a/src/network/infinity_thrift_service.cpp +++ b/src/network/infinity_thrift_service.cpp @@ -83,7 +83,7 @@ ClientVersions::ClientVersions() { client_version_map_[14] = String("0.3.0.dev5"); client_version_map_[15] = String("0.3.0.dev6"); client_version_map_[16] = String("0.3.0.dev7"); - client_version_map_[17] = String("0.3.0.dev8"); + client_version_map_[17] = String("0.3.0"); } Pair ClientVersions::GetVersionByIndex(i64 version_index) { @@ -1070,7 +1070,7 @@ void InfinityThriftService::CreateIndex(infinity_thrift_rpc::CommonResponse &res delete index_info_to_use; index_info_to_use = nullptr; - ProcessStatus(response, Status::InvalidIndexType()); + ProcessStatus(response, Status::InvalidIndexType("Invalid index")); return; } diff --git a/src/network/infinity_thrift_service.cppm b/src/network/infinity_thrift_service.cppm index a561d6b7d7..7cbba8651f 100644 --- a/src/network/infinity_thrift_service.cppm +++ b/src/network/infinity_thrift_service.cppm @@ -57,7 +57,7 @@ struct ClientVersions { export class InfinityThriftService final : public infinity_thrift_rpc::InfinityServiceIf { private: static constexpr std::string_view ErrorMsgHeader = "[THRIFT ERROR]"; - static constexpr i64 current_version_index_{17}; // 0.3.0.dev8 + static constexpr i64 current_version_index_{17}; // 0.3.0 static std::mutex infinity_session_map_mutex_; static HashMap> infinity_session_map_; diff --git a/src/planner/logical_planner.cpp b/src/planner/logical_planner.cpp index 640165907e..5044ecfec6 100644 --- a/src/planner/logical_planner.cpp +++ b/src/planner/logical_planner.cpp @@ -241,7 +241,7 @@ Status LogicalPlanner::BuildInsertValue(const InsertStatement *statement, Shared // Create value list Vector>> value_list_array; SizeT value_count = statement->values_->size(); - if(value_count > INSERT_BATCH_ROW_LIMIT) { + if (value_count > INSERT_BATCH_ROW_LIMIT) { RecoverableError(Status::NotSupport("Insert batch row limit shouldn't more than 8192.")); } @@ -451,23 +451,29 @@ Status LogicalPlanner::BuildCreateTable(const CreateStatement *statement, Shared if (column_count == 0) { return Status::NoColumnDefined(create_table_info->table_name_); } - + Set dedup_set; columns.reserve(column_count); for (SizeT idx = 0; idx < column_count; ++idx) { - - switch (IdentifierValidation(create_table_info->column_defs_[idx]->name())) { + const String &column_name = create_table_info->column_defs_[idx]->name(); + switch (IdentifierValidation(column_name)) { case IdentifierValidationStatus::kOk: break; case IdentifierValidationStatus::kEmpty: return Status::EmptyColumnName(); case IdentifierValidationStatus::kExceedLimit: - return Status::ExceedColumnNameLength(create_table_info->column_defs_[idx]->name().length()); + return Status::ExceedColumnNameLength(column_name.length()); case IdentifierValidationStatus::kInvalidName: { - return Status::InvalidColumnName(create_table_info->column_defs_[idx]->name()); + return Status::InvalidColumnName(column_name); } } - const DataType* data_type = create_table_info->column_defs_[idx]->type().get(); + if (dedup_set.contains(column_name)) { + return Status::DuplicateColumnName(column_name); + } else { + dedup_set.insert(column_name); + } + + const DataType *data_type = create_table_info->column_defs_[idx]->type().get(); switch (data_type->type()) { case LogicalType::kBoolean: case LogicalType::kTinyInt: @@ -486,10 +492,11 @@ Status LogicalPlanner::BuildCreateTable(const CreateStatement *statement, Shared break; } case LogicalType::kEmbedding: { - TypeInfo* type_info_ptr = data_type->type_info().get(); - EmbeddingInfo* embedding_info = static_cast(type_info_ptr); - if(embedding_info->Dimension() > EMBEDDING_LIMIT) { - return Status::NotSupport(fmt::format("Embedding data limit is {}, which larger than limit {}", embedding_info->Dimension(), EMBEDDING_LIMIT)); + TypeInfo *type_info_ptr = data_type->type_info().get(); + EmbeddingInfo *embedding_info = static_cast(type_info_ptr); + if (embedding_info->Dimension() > EMBEDDING_LIMIT) { + return Status::NotSupport( + fmt::format("Embedding data limit is {}, which larger than limit {}", embedding_info->Dimension(), EMBEDDING_LIMIT)); } break; } @@ -522,7 +529,7 @@ Status LogicalPlanner::BuildCreateTable(const CreateStatement *statement, Shared SharedPtr column_def = MakeShared(idx, create_table_info->column_defs_[idx]->type(), - create_table_info->column_defs_[idx]->name(), + column_name, create_table_info->column_defs_[idx]->constraints_, std::move(create_table_info->column_defs_[idx]->default_expr_)); columns.emplace_back(column_def); @@ -542,7 +549,8 @@ Status LogicalPlanner::BuildCreateTable(const CreateStatement *statement, Shared } } - SharedPtr table_def_ptr = TableDef::Make(MakeShared("default_db"), MakeShared(create_table_info->table_name_), std::move(columns)); + SharedPtr table_def_ptr = + TableDef::Make(MakeShared("default_db"), MakeShared(create_table_info->table_name_), std::move(columns)); for (HashSet visited_param_names; auto *property_ptr : create_table_info->properties_) { auto &[param_name, param_value] = *property_ptr; if (auto [_, success] = visited_param_names.insert(param_name); !success) { @@ -744,8 +752,8 @@ Status LogicalPlanner::BuildCreateIndex(const CreateStatement *statement, Shared break; } case IndexType::kInvalid: { - String error_message = "Invalid index type."; - UnrecoverableError(error_message); + Status status = Status::InvalidIndexType("Invalid index"); + RecoverableError(status); break; } } @@ -918,7 +926,7 @@ Status LogicalPlanner::BuildExport(const CopyStatement *statement, SharedPtrcopy_file_type_) { case CopyFileType::kJSONL: case CopyFileType::kFVECS: - case CopyFileType::kCSV: + case CopyFileType::kCSV: case CopyFileType::kPARQUET: { break; } @@ -1696,11 +1704,8 @@ Status LogicalPlanner::BuildShowPersistenceObject(const ShowStatement *statement } Status LogicalPlanner::BuildShowMemory(const ShowStatement *statement, SharedPtr &bind_context_ptr) { - SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), - ShowType::kShowMemory, - "", - "", - bind_context_ptr->GenerateTableIndex()); + SharedPtr logical_show = + MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowMemory, "", "", bind_context_ptr->GenerateTableIndex()); this->logical_plan_ = logical_show; return Status::OK();