From 1f63beb3e3a68651755f93b1ce7edfe07321cb9a Mon Sep 17 00:00:00 2001 From: A Terebey Date: Wed, 17 Apr 2024 17:55:20 +0300 Subject: [PATCH] Add GET /vectors/list endpoint https://docs.pinecone.io/reference/api/data-plane/list --- README.md | 13 ++ lib/pinecone/vector.rb | 11 ++ .../returns_a_response.yml | 137 +++++++++++++ .../with_limit/returns_a_response.yml | 137 +++++++++++++ .../returns_a_response.yml | 181 ++++++++++++++++++ .../with_prefix/returns_a_response.yml | 137 +++++++++++++ spec/cassettes/use_serverless_index.yml | 49 +++++ spec/pinecone/vector_spec.rb | 95 +++++++++ 8 files changed, 760 insertions(+) create mode 100644 spec/cassettes/Pinecone_Vector/_list/successful_response/returns_a_response.yml create mode 100644 spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/returns_a_response.yml create mode 100644 spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/with_pagination_token/returns_a_response.yml create mode 100644 spec/cassettes/Pinecone_Vector/_list/successful_response/with_prefix/returns_a_response.yml create mode 100644 spec/cassettes/use_serverless_index.yml diff --git a/README.md b/README.md index 2e9829f..6791582 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,18 @@ index.fetch( ) ``` +List vector IDs from an index (only for serverless indexes) +```ruby +pinecone = Pinecone::Client.new +index = pinecone.index("example-index") +index.list( + namespace: "example-namespace", + prefix: "example-prefix", + limit: 50, + pagination_token: "example-token" +) +``` + Updating a vector in an index ```ruby pinecone = Pinecone::Client.new @@ -255,6 +267,7 @@ Contributions welcome! - Clone the repo locally - `bundle` to install gems - run tests with `rspec` +- run linter with `standardrb` - `mv .env.copy .env` and add Pinecone API Key if developing a new endpoint or modifying existing ones - to disable VCR and hit real endpoints, `NO_VCR=true rspec` - To setup cloud indexes when writing new tests `ruby spec/support/setup.rb start` and `stop` to delete them diff --git a/lib/pinecone/vector.rb b/lib/pinecone/vector.rb index fc54d8c..de362d7 100644 --- a/lib/pinecone/vector.rb +++ b/lib/pinecone/vector.rb @@ -37,6 +37,17 @@ def fetch(namespace: "", ids: []) self.class.get("#{@base_uri}/vectors/fetch?#{query_string}", options) end + def list(namespace: "", prefix: "", limit: nil, pagination_token: "") + query_params = {} + query_params["namespace"] = namespace unless namespace.empty? + query_params["prefix"] = prefix unless prefix.empty? + query_params["limit"] = limit if limit + query_params["paginationToken"] = pagination_token unless pagination_token.empty? + + query_string = URI.encode_www_form(query_params) + self.class.get("#{@base_uri}/vectors/list?#{query_string}", options) + end + def upsert(body) payload = options.merge(body: body.to_json) self.class.post("#{@base_uri}/vectors/upsert", payload) diff --git a/spec/cassettes/Pinecone_Vector/_list/successful_response/returns_a_response.yml b/spec/cassettes/Pinecone_Vector/_list/successful_response/returns_a_response.yml new file mode 100644 index 0000000..b668a1c --- /dev/null +++ b/spec/cassettes/Pinecone_Vector/_list/successful_response/returns_a_response.yml @@ -0,0 +1,137 @@ +--- +http_interactions: +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/upsert + body: + encoding: UTF-8 + string: '{"vectors":[{"values":[1,2,3],"id":"1"},{"values":[1,2,3],"id":"2"}]}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:33 GMT + Content-Type: + - application/json + Content-Length: + - '19' + Connection: + - keep-alive + X-Pinecone-Request-Lsn: + - '1' + X-Pinecone-Request-Latency-Ms: + - '70' + X-Pinecone-Request-Id: + - '5663984686665354313' + X-Envoy-Upstream-Service-Time: + - '71' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"upsertedCount":2}' + recorded_at: Wed, 17 Apr 2024 14:49:34 GMT +- request: + method: get + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/list + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:39 GMT + Content-Type: + - application/json + Content-Length: + - '74' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '4' + X-Pinecone-Request-Id: + - '5270960153551806981' + X-Envoy-Upstream-Service-Time: + - '5' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"vectors":[{"id":"1"},{"id":"2"}],"namespace":"","usage":{"readUnits":1}}' + recorded_at: Wed, 17 Apr 2024 14:49:39 GMT +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/delete + body: + encoding: UTF-8 + string: '{"namespace":"","ids":[],"deleteAll":true}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:40 GMT + Content-Type: + - application/json + Content-Length: + - '2' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '22' + X-Pinecone-Request-Id: + - '7233596559818886427' + X-Envoy-Upstream-Service-Time: + - '22' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: "{}" + recorded_at: Wed, 17 Apr 2024 14:49:40 GMT +recorded_with: VCR 6.1.0 diff --git a/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/returns_a_response.yml b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/returns_a_response.yml new file mode 100644 index 0000000..3dafb79 --- /dev/null +++ b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/returns_a_response.yml @@ -0,0 +1,137 @@ +--- +http_interactions: +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/upsert + body: + encoding: UTF-8 + string: '{"vectors":[{"values":[1,2,3],"id":"1"},{"values":[1,2,3],"id":"2"}]}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:36 GMT + Content-Type: + - application/json + Content-Length: + - '19' + Connection: + - keep-alive + X-Pinecone-Request-Lsn: + - '1' + X-Pinecone-Request-Latency-Ms: + - '38' + X-Pinecone-Request-Id: + - '4033875309875663471' + X-Envoy-Upstream-Service-Time: + - '38' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"upsertedCount":2}' + recorded_at: Wed, 17 Apr 2024 14:50:36 GMT +- request: + method: get + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/list?limit=1 + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:42 GMT + Content-Type: + - application/json + Content-Length: + - '132' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '7' + X-Pinecone-Request-Id: + - '8561116144363514106' + X-Envoy-Upstream-Service-Time: + - '8' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"vectors":[{"id":"1"}],"pagination":{"next":"eyJza2lwX3Bhc3QiOiIxIiwicHJlZml4IjpudWxsfQ=="},"namespace":"","usage":{"readUnits":1}}' + recorded_at: Wed, 17 Apr 2024 14:50:42 GMT +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/delete + body: + encoding: UTF-8 + string: '{"namespace":"","ids":[],"deleteAll":true}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:42 GMT + Content-Type: + - application/json + Content-Length: + - '2' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '50' + X-Pinecone-Request-Id: + - '6855191500182073582' + X-Envoy-Upstream-Service-Time: + - '51' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: "{}" + recorded_at: Wed, 17 Apr 2024 14:50:42 GMT +recorded_with: VCR 6.1.0 diff --git a/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/with_pagination_token/returns_a_response.yml b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/with_pagination_token/returns_a_response.yml new file mode 100644 index 0000000..6472c96 --- /dev/null +++ b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_limit/with_pagination_token/returns_a_response.yml @@ -0,0 +1,181 @@ +--- +http_interactions: +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/upsert + body: + encoding: UTF-8 + string: '{"vectors":[{"values":[1,2,3],"id":"1"},{"values":[1,2,3],"id":"2"}]}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:53 GMT + Content-Type: + - application/json + Content-Length: + - '19' + Connection: + - keep-alive + X-Pinecone-Request-Lsn: + - '1' + X-Pinecone-Request-Latency-Ms: + - '75' + X-Pinecone-Request-Id: + - '4124490512945968513' + X-Envoy-Upstream-Service-Time: + - '76' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"upsertedCount":2}' + recorded_at: Wed, 17 Apr 2024 14:50:53 GMT +- request: + method: get + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/list?limit=1 + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:58 GMT + Content-Type: + - application/json + Content-Length: + - '132' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '6' + X-Pinecone-Request-Id: + - '6243278115680371296' + X-Envoy-Upstream-Service-Time: + - '7' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"vectors":[{"id":"1"}],"pagination":{"next":"eyJza2lwX3Bhc3QiOiIxIiwicHJlZml4IjpudWxsfQ=="},"namespace":"","usage":{"readUnits":1}}' + recorded_at: Wed, 17 Apr 2024 14:50:59 GMT +- request: + method: get + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/list?limit=1&paginationToken=eyJza2lwX3Bhc3QiOiIxIiwicHJlZml4IjpudWxsfQ== + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:50:59 GMT + Content-Type: + - application/json + Content-Length: + - '63' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '5' + X-Pinecone-Request-Id: + - '5399823328252287626' + X-Envoy-Upstream-Service-Time: + - '6' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"vectors":[{"id":"2"}],"namespace":"","usage":{"readUnits":1}}' + recorded_at: Wed, 17 Apr 2024 14:50:59 GMT +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/delete + body: + encoding: UTF-8 + string: '{"namespace":"","ids":[],"deleteAll":true}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:51:00 GMT + Content-Type: + - application/json + Content-Length: + - '2' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '100' + X-Pinecone-Request-Id: + - '1488100582190574352' + X-Envoy-Upstream-Service-Time: + - '100' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: "{}" + recorded_at: Wed, 17 Apr 2024 14:51:00 GMT +recorded_with: VCR 6.1.0 diff --git a/spec/cassettes/Pinecone_Vector/_list/successful_response/with_prefix/returns_a_response.yml b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_prefix/returns_a_response.yml new file mode 100644 index 0000000..b69d5b1 --- /dev/null +++ b/spec/cassettes/Pinecone_Vector/_list/successful_response/with_prefix/returns_a_response.yml @@ -0,0 +1,137 @@ +--- +http_interactions: +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/upsert + body: + encoding: UTF-8 + string: '{"vectors":[{"values":[1,2,3],"id":"foo#1"},{"values":[1,2,3],"id":"foo#2"},{"values":[1,2,3],"id":"bar#1"}]}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:47 GMT + Content-Type: + - application/json + Content-Length: + - '19' + Connection: + - keep-alive + X-Pinecone-Request-Lsn: + - '1' + X-Pinecone-Request-Latency-Ms: + - '178' + X-Pinecone-Request-Id: + - '5348580298638611894' + X-Envoy-Upstream-Service-Time: + - '178' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"upsertedCount":3}' + recorded_at: Wed, 17 Apr 2024 14:49:47 GMT +- request: + method: get + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/list?prefix=foo%23 + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:52 GMT + Content-Type: + - application/json + Content-Length: + - '82' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '36' + X-Pinecone-Request-Id: + - '4772401632993318618' + X-Envoy-Upstream-Service-Time: + - '37' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: '{"vectors":[{"id":"foo#1"},{"id":"foo#2"}],"namespace":"","usage":{"readUnits":1}}' + recorded_at: Wed, 17 Apr 2024 14:49:52 GMT +- request: + method: post + uri: https://test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io/vectors/delete + body: + encoding: UTF-8 + string: '{"namespace":"","ids":[],"deleteAll":true}' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 17 Apr 2024 14:49:53 GMT + Content-Type: + - application/json + Content-Length: + - '2' + Connection: + - keep-alive + X-Pinecone-Request-Latency-Ms: + - '30' + X-Pinecone-Request-Id: + - '330430087357710490' + X-Envoy-Upstream-Service-Time: + - '30' + Grpc-Status: + - '0' + Server: + - envoy + body: + encoding: UTF-8 + string: "{}" + recorded_at: Wed, 17 Apr 2024 14:49:53 GMT +recorded_with: VCR 6.1.0 diff --git a/spec/cassettes/use_serverless_index.yml b/spec/cassettes/use_serverless_index.yml new file mode 100644 index 0000000..6a459fe --- /dev/null +++ b/spec/cassettes/use_serverless_index.yml @@ -0,0 +1,49 @@ +--- +http_interactions: +- request: + method: get + uri: https://api.pinecone.io/indexes/test-index-serverless + body: + encoding: US-ASCII + string: '' + headers: + Content-Type: + - application/json + Accept: + - application/json + Api-Key: + - "" + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + User-Agent: + - Ruby + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json + Access-Control-Allow-Origin: + - "*" + Vary: + - origin,access-control-request-method,access-control-request-headers + Access-Control-Expose-Headers: + - "*" + X-Cloud-Trace-Context: + - ad1a5890b0d1eaa19ef59c99396ddb9a + Date: + - Wed, 17 Apr 2024 14:50:52 GMT + Server: + - Google Frontend + Content-Length: + - '237' + Via: + - 1.1 google + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + body: + encoding: UTF-8 + string: '{"name":"test-index-serverless","metric":"dotproduct","dimension":3,"status":{"ready":true,"state":"Ready"},"host":"test-index-serverless-d2c1065.svc.aped-4627-b74a.pinecone.io","spec":{"serverless":{"region":"us-east-1","cloud":"aws"}}}' + recorded_at: Wed, 17 Apr 2024 14:50:52 GMT +recorded_with: VCR 6.1.0 diff --git a/spec/pinecone/vector_spec.rb b/spec/pinecone/vector_spec.rb index 54a1c7e..cb40baf 100644 --- a/spec/pinecone/vector_spec.rb +++ b/spec/pinecone/vector_spec.rb @@ -108,6 +108,101 @@ end end + describe "#list", :vcr do + let(:index) { + VCR.use_cassette("use_serverless_index") do + Pinecone::Vector.new("test-index-serverless") + end + } + + let(:data) { + { + vectors: [ + {values: [1, 2, 3], id: "1"}, + {values: [1, 2, 3], id: "2"} + ] + } + } + + before do + index.upsert(data) + end + + describe "successful response" do + let(:response) { index.list } + + it "returns a response" do + expect(response).to be_a(HTTParty::Response) + expect(response.code).to eq(200) + expect(response.parsed_response).to match( + "namespace" => "", + "vectors" => [ + {"id" => "1"}, + {"id" => "2"} + ], + "usage" => {"readUnits" => 1} + ) + end + + describe "with prefix" do + let(:data) { + { + vectors: [ + {values: [1, 2, 3], id: "foo#1"}, + {values: [1, 2, 3], id: "foo#2"}, + {values: [1, 2, 3], id: "bar#1"} + ] + } + } + + let(:response) { index.list(prefix: "foo#") } + + it "returns a response" do + expect(response).to be_a(HTTParty::Response) + expect(response.code).to eq(200) + expect(response.parsed_response).to match( + "namespace" => "", + "vectors" => [ + {"id" => "foo#1"}, + {"id" => "foo#2"} + ], + "usage" => {"readUnits" => 1} + ) + end + end + + describe "with limit" do + let(:response) { index.list(limit: 1) } + + it "returns a response" do + expect(response).to be_a(HTTParty::Response) + expect(response.code).to eq(200) + expect(response.parsed_response).to match( + "namespace" => "", + "vectors" => [{"id" => "1"}], + "pagination" => {"next" => be_a(String)}, + "usage" => {"readUnits" => 1} + ) + end + + describe "with pagination token" do + let(:pagination_token) { index.list(limit: 1).parsed_response.dig("pagination", "next") } + let(:response) { index.list(limit: 1, pagination_token: pagination_token) } + + it "returns a response" do + expect(response).to be_a(HTTParty::Response) + expect(response.code).to eq(200) + expect(response.parsed_response).to match( + "namespace" => "", + "vectors" => [{"id" => "2"}], + "usage" => {"readUnits" => 1} + ) + end + end + end + end + end + describe "#update", :vcr do describe "successful response" do let(:response) {