From 3a52caaea64a8cfb96a9fb7a03cd21fb86708662 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 11 Mar 2021 20:10:30 +0800 Subject: [PATCH 01/17] init --- .../infra/pegasus/client/PException.java | 4 +- .../client/PegasusClientInterface.java | 42 ++++++-------- .../infra/pegasus/client/PegasusTable.java | 22 +++++-- .../pegasus/client/PegasusTableInterface.java | 45 +++++++-------- .../request/{ => batch}/AbstractBatch.java | 2 +- .../client/request/{ => batch}/Batch.java | 2 +- .../{ => batch}/BatchWithResponse.java | 2 +- .../client/request/{ => batch}/Delete.java | 2 +- .../request/{ => batch}/DeleteBatch.java | 2 +- .../client/request/{ => batch}/Get.java | 2 +- .../client/request/{ => batch}/GetBatch.java | 2 +- .../request/{ => batch}/MultiDelete.java | 2 +- .../request/{ => batch}/MultiDeleteBatch.java | 2 +- .../client/request/{ => batch}/MultiGet.java | 2 +- .../request/{ => batch}/MultiGetBatch.java | 2 +- .../client/request/{ => batch}/MultiSet.java | 2 +- .../request/{ => batch}/MultiSetBatch.java | 2 +- .../client/request/{ => batch}/Set.java | 2 +- .../client/request/{ => batch}/SetBatch.java | 2 +- .../client/request/range/DeleteRange.java | 10 ++++ .../client/request/range/FullScanRange.java | 9 +++ .../client/request/range/HashScanRange.java | 32 +++++++++++ .../client/request/range/ScannerWrapper.java | 57 +++++++++++++++++++ .../infra/pegasus/example/BatchSample.java | 10 ++-- .../infra/pegasus/client/TestBatch.java | 26 ++++----- 25 files changed, 199 insertions(+), 88 deletions(-) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/AbstractBatch.java (97%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/Batch.java (98%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/BatchWithResponse.java (98%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/Delete.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/DeleteBatch.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/Get.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/GetBatch.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiDelete.java (96%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiDeleteBatch.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiGet.java (96%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiGetBatch.java (96%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiSet.java (97%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/MultiSetBatch.java (95%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/Set.java (96%) rename src/main/java/com/xiaomi/infra/pegasus/client/request/{ => batch}/SetBatch.java (95%) create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PException.java b/src/main/java/com/xiaomi/infra/pegasus/client/PException.java index 4483bcf3..c95bb096 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PException.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PException.java @@ -34,14 +34,14 @@ public PException(Throwable cause) { super(versionPrefix + cause.toString(), cause); } - static PException threadInterrupted(String tableName, InterruptedException e) { + public static PException threadInterrupted(String tableName, InterruptedException e) { return new PException( new ReplicationException( error_code.error_types.ERR_THREAD_INTERRUPTED, String.format("[table=%s] Thread was interrupted: %s", tableName, e.getMessage()))); } - static PException timeout( + public static PException timeout( String metaList, String tableName, Request request, int timeout, TimeoutException e) { return new PException( new ReplicationException( diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java index 3af2de70..dae28e9c 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java @@ -3,6 +3,8 @@ // can be found in the LICENSE file in the root directory of this source tree. package com.xiaomi.infra.pegasus.client; +import com.xiaomi.infra.pegasus.client.request.batch.Batch; +import com.xiaomi.infra.pegasus.client.request.batch.BatchWithResponse; import java.util.*; import org.apache.commons.lang3.tuple.Pair; @@ -111,8 +113,7 @@ public PegasusTableInterface openTable(String tableName, TableOptions tableOptio * Batch get values of different keys. Will terminate immediately if any error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commit(List, List)} + * more. The latest batch operation please see {@link BatchWithResponse#commit(List, List)} * @param tableName table name * @param keys hashKey and sortKey pair list. * @param values output values; should be created by caller; if succeed, the size of values will @@ -131,8 +132,7 @@ public void batchGet(String tableName, List> keys, List> values) throws PException; + @Deprecated public boolean multiGet( String tableName, byte[] hashKey, @@ -220,8 +223,7 @@ public boolean multiGet( * occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commit(List, List)} + * more. The latest batch operation please see {@link BatchWithResponse#commit(List, List)} * @param tableName table name * @param keys List{hashKey,List{sortKey}}; if List{sortKey} is null or empty, means fetch all * sortKeys under the hashKey. @@ -242,8 +244,7 @@ public void batchMultiGet( * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commitWaitAllComplete(List, - * List)} + * BatchWithResponse#commitWaitAllComplete(List, List)} * @param tableName table name * @param keys List{hashKey,List{sortKey}}; if List{sortKey} is null or empty, means fetch all * sortKeys under the hashKey. @@ -307,8 +308,7 @@ public void set(String tableName, byte[] hashKey, byte[] sortKey, byte[] value, * Batch set lots of values. Will terminate immediately if any error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param tableName TableHandler name * @param items list of items. * @throws PException throws exception if any error occurs. @@ -322,8 +322,7 @@ public void set(String tableName, byte[] hashKey, byte[] sortKey, byte[] value, * Batch set lots of values. Will wait for all requests done even if some error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param tableName table name * @param items list of items. * @param results output results; should be created by caller; after call done, the size of @@ -360,8 +359,7 @@ public void multiSet(String tableName, byte[] hashKey, List * occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param tableName TableHandler name * @param items list of items. * @param ttlSeconds time to live in seconds, 0 means no ttl. default value is 0. @@ -381,8 +379,7 @@ public void batchMultiSet(String tableName, List items, int ttlSeco * error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param tableName table name * @param items list of items. * @param ttlSeconds time to live in seconds, 0 means no ttl. default value is 0. @@ -420,8 +417,7 @@ public int batchMultiSet2(String tableName, List items, List items, List>> key * if some error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param tableName table name * @param keys List{hashKey,List{sortKey}} * @param results output results; should be created by caller; after call done, the size of diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java index f2f69654..ff66161a 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java @@ -43,6 +43,18 @@ public PegasusTable(PegasusClient client, Table table) { this.metaList = client.getMetaList(); } + public Table getTable() { + return table; + } + + public int getDefaultTimeout() { + return defaultTimeout; + } + + public String getMetaList() { + return metaList; + } + @Override public Future asyncExist(byte[] hashKey, byte[] sortKey, int timeout) { final DefaultPromise promise = table.newPromise(); @@ -1822,7 +1834,7 @@ public List getUnorderedScanners( * {@linkplain #scanRange(byte[], byte[], byte[], ScanOptions, int, int)} result, if fetch all * data for {startSortKey, stopSortKey}, ScanRangeResult.allFetched=true */ - static class ScanRangeResult { + public static class ScanRangeResult { public List, byte[]>> results; public boolean allFetched; } @@ -1934,21 +1946,21 @@ private void handleWriteLimiterException(DefaultPromise promise, String message) promise.setFailure(new PException("Exceed write limit threshold:" + message)); } - static class Request { + public static class Request { byte[] hashKey = null; byte[] sortKey = null; int sortKeyCount = 0; - Request(byte[] hashKey) { + public Request(byte[] hashKey) { this.hashKey = hashKey; } - Request(byte[] hashKey, byte[] sortKey) { + public Request(byte[] hashKey, byte[] sortKey) { this.hashKey = hashKey; this.sortKey = sortKey; } - Request(byte[] hashKey, int sortKeyCount) { + public Request(byte[] hashKey, int sortKeyCount) { this.hashKey = hashKey; this.sortKeyCount = sortKeyCount; } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java index f54fbec7..30632685 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java @@ -3,6 +3,8 @@ // can be found in the LICENSE file in the root directory of this source tree. package com.xiaomi.infra.pegasus.client; +import com.xiaomi.infra.pegasus.client.request.batch.Batch; +import com.xiaomi.infra.pegasus.client.request.batch.BatchWithResponse; import io.netty.util.concurrent.Future; import io.netty.util.concurrent.GenericFutureListener; import java.util.List; @@ -219,6 +221,7 @@ public Future asyncMultiGet( /** * get multiple key-values under the same hashKey with sortKey range limited, async version * + * @deprecated the API may can't get all records, please use //TODO * @param hashKey used to decide which partition the key may exist should not be null or empty. * @param startSortKey the start sort key. null means "". * @param stopSortKey the stop sort key. null or "" means fetch to the last sort key. @@ -238,6 +241,7 @@ public Future asyncMultiGet( * the same order as the listeners added. But listeners for different tables are not * guaranteed to be dispatched in the same thread. */ + @Deprecated public Future asyncMultiGet( byte[] hashKey, byte[] startSortKey, @@ -247,6 +251,7 @@ public Future asyncMultiGet( int maxFetchSize, int timeout /*ms*/); + @Deprecated public Future asyncMultiGet( byte[] hashKey, byte[] startSortKey, @@ -730,8 +735,7 @@ public static interface TTLListener extends GenericFutureListener> keys, List values, int t * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commitWaitAllComplete(List, - * List)} + * BatchWithResponse#commitWaitAllComplete(List, List)} * @param keys hashKey and sortKey pair list. * @param results output results; should be created by caller; after call done, the size of * results will be same with keys; the results[i] is a Pair: - if Pair.left != null : means @@ -791,7 +794,10 @@ public MultiGetResult multiGet(byte[] hashKey, List sortKeys, int timeou * sync version of MultiGet, please refer to the async version {@link #asyncMultiGet(byte[], * byte[], byte[], MultiGetOptions, int, int, int)} and {@link #asyncMultiGet(byte[], byte[], * byte[], MultiGetOptions, int)} + * + * @deprecated The API may can't get all records */ + @Deprecated public MultiGetResult multiGet( byte[] hashKey, byte[] startSortKey, @@ -802,6 +808,7 @@ public MultiGetResult multiGet( int timeout /*ms*/) throws PException; + @Deprecated public MultiGetResult multiGet( byte[] hashKey, byte[] startSortKey, @@ -815,8 +822,7 @@ public MultiGetResult multiGet( * occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commit(List, List)} + * more. The latest batch operation please see {@link BatchWithResponse#commit(List, List)} * @param keys List{hashKey,List{sortKey}} * @param values output values; should be created by caller; if succeed, the size of values will * be same with keys; the data for keys[i] is stored in values[i]. @@ -837,8 +843,7 @@ public void batchMultiGet( * error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.BatchWithResponse#commit(List, List)} + * more. The latest batch operation please see {@link BatchWithResponse#commit(List, List)} * @param keys List{hashKey,List{sortKey}}; if List{sortKey} is null or empty, means fetch all * sortKeys under the hashKey. * @param results output results; should be created by caller; after call done, the size of @@ -890,8 +895,7 @@ public void set(byte[] hashKey, byte[] sortKey, byte[] value, int timeout /*ms*/ * Batch set lots of values. Will terminate immediately if any error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param items list of items. * @param timeout how long will the operation timeout in milliseconds. if timeout > 0, it is a * timeout value for current op, else the timeout value in the configuration file will be @@ -907,8 +911,7 @@ public void set(byte[] hashKey, byte[] sortKey, byte[] value, int timeout /*ms*/ * Batch set lots of values. Will wait for all requests done even if some error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param items list of items. * @param results output results; should be created by caller; after call done, the size of * results will be same with items; the results[i] is a PException: - if results[i] != null : @@ -942,8 +945,7 @@ public void multiSet(byte[] hashKey, List> values, int time * occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param items list of items. * @param ttlSeconds time to live in seconds, 0 means no ttl. * @param timeout how long will the operation timeout in milliseconds. if timeout > 0, it is a @@ -962,8 +964,7 @@ public void batchMultiSet(List items, int ttlSeconds, int timeout / * error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param items list of items. * @param ttlSeconds time to live in seconds, 0 means no ttl. default value is 0. * @param results output results; should be created by caller; after call done, the size of @@ -992,8 +993,7 @@ public int batchMultiSet2( * Batch delete values of different keys. Will terminate immediately if any error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param keys hashKey and sortKey pair list. * @param timeout how long will the operation timeout in milliseconds. if timeout > 0, it is a * timeout value for current op, else the timeout value in the configuration file will be @@ -1010,8 +1010,7 @@ public int batchMultiSet2( * occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param keys hashKey and sortKey pair list. * @param results output results; should be created by caller; after call done, the size of * results will be same with keys; the results[i] is a PException: - if results[i] != null : @@ -1062,8 +1061,7 @@ public void delRange( * error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commit(List)} + * more. The latest batch operation please see {@link Batch#commit(List)} * @param keys List{hashKey,List{sortKey}} * @param timeout how long will the operation timeout in milliseconds. if timeout > 0, it is a * timeout value for current op, else the timeout value in the configuration file will be @@ -1081,8 +1079,7 @@ public void batchMultiDel(List>> keys, int timeout /*m * if some error occurs. * * @deprecated Retained only for backward compatibility, will be removed later. Don't use it any - * more. The latest batch operation please see {@link - * com.xiaomi.infra.pegasus.client.request.Batch#commitWaitAllComplete(List, List)} + * more. The latest batch operation please see {@link Batch#commitWaitAllComplete(List, List)} * @param keys List{hashKey,List{sortKey}} * @param results output results; should be created by caller; after call done, the size of * results will be same with keys; the results[i] is a PException: - if results[i] != null : diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/AbstractBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/AbstractBatch.java similarity index 97% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/AbstractBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/AbstractBatch.java index 20b65fe1..8e52dfae 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/AbstractBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/AbstractBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.FutureGroup; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/Batch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Batch.java similarity index 98% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/Batch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Batch.java index 1e2b693c..068056c6 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/Batch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Batch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/BatchWithResponse.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/BatchWithResponse.java similarity index 98% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/BatchWithResponse.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/BatchWithResponse.java index 0de78de8..50de7d2e 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/BatchWithResponse.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/BatchWithResponse.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/Delete.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Delete.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/Delete.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Delete.java index 733dd4db..c70b7581 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/Delete.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Delete.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/DeleteBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/DeleteBatch.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/DeleteBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/DeleteBatch.java index 5c9059b3..a5634474 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/DeleteBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/DeleteBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import io.netty.util.concurrent.Future; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/Get.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Get.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/Get.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Get.java index fd1cd23d..b0203ac5 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/Get.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Get.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/GetBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/GetBatch.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/GetBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/GetBatch.java index 6791bdc9..048d3b51 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/GetBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/GetBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import io.netty.util.concurrent.Future; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDelete.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDelete.java similarity index 96% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDelete.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDelete.java index c6ceedf1..073677a4 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDelete.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDelete.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; import java.util.ArrayList; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDeleteBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDeleteBatch.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDeleteBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDeleteBatch.java index 7c006755..d3a95391 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiDeleteBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiDeleteBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import io.netty.util.concurrent.Future; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGet.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGet.java similarity index 96% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGet.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGet.java index d14d8ae2..fbc43096 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGet.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGet.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; import java.util.ArrayList; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGetBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGetBatch.java similarity index 96% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGetBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGetBatch.java index 7a940f53..c42bf621 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiGetBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiGetBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSet.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSet.java similarity index 97% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSet.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSet.java index 28b641c9..7b128ba4 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSet.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSet.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; import java.util.ArrayList; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSetBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSetBatch.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSetBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSetBatch.java index 9e5ec035..f8feedc7 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/MultiSetBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/MultiSetBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import io.netty.util.concurrent.Future; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/Set.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Set.java similarity index 96% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/Set.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Set.java index 40728fee..190db8c9 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/Set.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/Set.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import java.io.Serializable; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/SetBatch.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/SetBatch.java similarity index 95% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/SetBatch.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/batch/SetBatch.java index d77213c8..4c2a1e7d 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/SetBatch.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/batch/SetBatch.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.xiaomi.infra.pegasus.client.request; +package com.xiaomi.infra.pegasus.client.request.batch; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import io.netty.util.concurrent.Future; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java new file mode 100644 index 00000000..46051433 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -0,0 +1,10 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.DelRangeOptions; + +public class DeleteRange { + private DelRangeOptions delRangeOptions; + private byte[] startSortKey; + private byte[] stopSortKey; + private int timeout; +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java new file mode 100644 index 00000000..5088c13c --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java @@ -0,0 +1,9 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.ScanOptions; + +public class FullScanRange { + private ScanOptions scanOptions; + private int maxSplitCount; + private int timeout; +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java new file mode 100644 index 00000000..2a5b3baf --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java @@ -0,0 +1,32 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusTable; +import com.xiaomi.infra.pegasus.client.PegasusTableInterface; +import com.xiaomi.infra.pegasus.client.ScanOptions; + +public class HashScanRange { + public PegasusTableInterface table; + + private ScanOptions scanOptions; + private byte[] hashKey; + private byte[] startSortKey; + private byte[] stopSortKey; + private int maxFetchCount; + private int timeout; + + public HashScanRange(PegasusTableInterface table, ScanOptions scanOptions, byte[] hashKey, byte[] startSortKey, byte[] stopSortKey, int maxFetchCount, int timeout) { + this.table = table; + this.scanOptions = scanOptions; + this.startSortKey = startSortKey; + this.hashKey = hashKey; + this.stopSortKey = stopSortKey; + this.maxFetchCount = maxFetchCount; + this.timeout = timeout; + } + + public PegasusTable.ScanRangeResult commitAndWait() throws PException { + ScannerWrapper scannerWrapper = new ScannerWrapper(table); + return scannerWrapper.hashScan(hashKey, startSortKey,stopSortKey,scanOptions, maxFetchCount, timeout); + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java new file mode 100644 index 00000000..6f1182a7 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -0,0 +1,57 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.*; +import com.xiaomi.infra.pegasus.rpc.Table; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.concurrent.TimeoutException; + +public class ScannerWrapper { + private final PegasusTableInterface table; + + public ScannerWrapper(PegasusTableInterface table) { + this.table = table; + } + + PegasusTable.ScanRangeResult hashScan( + byte[] hashKey, + byte[] startSortKey, + byte[] stopSortKey, + ScanOptions options, + int maxFetchCount, + int timeout /*ms*/) + throws PException { + if (timeout <= 0) timeout = ((PegasusTable)table).getDefaultTimeout(); + long deadlineTime = System.currentTimeMillis() + timeout; + + PegasusScannerInterface pegasusScanner = + table.getScanner(hashKey, startSortKey, stopSortKey, options); + PegasusTable.ScanRangeResult scanRangeResult = new PegasusTable.ScanRangeResult(); + scanRangeResult.allFetched = false; + scanRangeResult.results = new ArrayList<>(); + if (System.currentTimeMillis() >= deadlineTime) { + throw PException.timeout( + ((PegasusTable)table).getMetaList(), + ((PegasusTable)table).getTable().getTableName(), + new PegasusTable.Request(hashKey), timeout, new TimeoutException()); + } + + Pair, byte[]> pair; + while ((pair = pegasusScanner.next()) != null + && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { + if (System.currentTimeMillis() >= deadlineTime) { + throw PException.timeout( + ((PegasusTable)table).getMetaList(), + ((PegasusTable)table).getTable().getTableName(), + new PegasusTable.Request(hashKey), timeout, new TimeoutException()); + } + scanRangeResult.results.add(pair); + } + + if (pegasusScanner.next() == null) { + scanRangeResult.allFetched = true; + } + return scanRangeResult; + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/example/BatchSample.java b/src/main/java/com/xiaomi/infra/pegasus/example/BatchSample.java index d8c93664..9a9552af 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/example/BatchSample.java +++ b/src/main/java/com/xiaomi/infra/pegasus/example/BatchSample.java @@ -22,11 +22,11 @@ import com.xiaomi.infra.pegasus.client.PegasusClientFactory; import com.xiaomi.infra.pegasus.client.PegasusClientInterface; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -import com.xiaomi.infra.pegasus.client.request.BatchWithResponse; -import com.xiaomi.infra.pegasus.client.request.Get; -import com.xiaomi.infra.pegasus.client.request.GetBatch; -import com.xiaomi.infra.pegasus.client.request.Set; -import com.xiaomi.infra.pegasus.client.request.SetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.BatchWithResponse; +import com.xiaomi.infra.pegasus.client.request.batch.Get; +import com.xiaomi.infra.pegasus.client.request.batch.GetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.Set; +import com.xiaomi.infra.pegasus.client.request.batch.SetBatch; import io.netty.util.concurrent.Future; import java.util.ArrayList; import java.util.List; diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestBatch.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestBatch.java index 6f7d0e30..040e0ebf 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestBatch.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestBatch.java @@ -17,19 +17,19 @@ package com.xiaomi.infra.pegasus.client; import com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; -import com.xiaomi.infra.pegasus.client.request.BatchWithResponse; -import com.xiaomi.infra.pegasus.client.request.Delete; -import com.xiaomi.infra.pegasus.client.request.DeleteBatch; -import com.xiaomi.infra.pegasus.client.request.Get; -import com.xiaomi.infra.pegasus.client.request.GetBatch; -import com.xiaomi.infra.pegasus.client.request.MultiDelete; -import com.xiaomi.infra.pegasus.client.request.MultiDeleteBatch; -import com.xiaomi.infra.pegasus.client.request.MultiGet; -import com.xiaomi.infra.pegasus.client.request.MultiGetBatch; -import com.xiaomi.infra.pegasus.client.request.MultiSet; -import com.xiaomi.infra.pegasus.client.request.MultiSetBatch; -import com.xiaomi.infra.pegasus.client.request.Set; -import com.xiaomi.infra.pegasus.client.request.SetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.BatchWithResponse; +import com.xiaomi.infra.pegasus.client.request.batch.Delete; +import com.xiaomi.infra.pegasus.client.request.batch.DeleteBatch; +import com.xiaomi.infra.pegasus.client.request.batch.Get; +import com.xiaomi.infra.pegasus.client.request.batch.GetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.MultiDelete; +import com.xiaomi.infra.pegasus.client.request.batch.MultiDeleteBatch; +import com.xiaomi.infra.pegasus.client.request.batch.MultiGet; +import com.xiaomi.infra.pegasus.client.request.batch.MultiGetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.MultiSet; +import com.xiaomi.infra.pegasus.client.request.batch.MultiSetBatch; +import com.xiaomi.infra.pegasus.client.request.batch.Set; +import com.xiaomi.infra.pegasus.client.request.batch.SetBatch; import io.netty.util.concurrent.Future; import java.util.ArrayList; import java.util.List; From c1365134fd62a3218abf0e106373fac1ce2780de Mon Sep 17 00:00:00 2001 From: jiashuo Date: Mon, 15 Mar 2021 17:33:31 +0800 Subject: [PATCH 02/17] init --- .../com/xiaomi/infra/pegasus/TestScan.java | 81 +++++++++++++++++++ .../client/request/range/DeleteRange.java | 14 ++-- .../client/request/range/FullScanRange.java | 9 --- .../client/request/range/GetRange.java | 22 +++++ .../client/request/range/HashScanRange.java | 32 -------- .../pegasus/client/request/range/Scan.java | 31 +++++++ 6 files changed, 142 insertions(+), 47 deletions(-) create mode 100644 src/main/java/com/xiaomi/infra/pegasus/TestScan.java delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/TestScan.java b/src/main/java/com/xiaomi/infra/pegasus/TestScan.java new file mode 100644 index 00000000..0678fd06 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/TestScan.java @@ -0,0 +1,81 @@ +package com.xiaomi.infra.pegasus; + +import static com.xiaomi.infra.pegasus.client.FilterType.*; + +import com.xiaomi.infra.pegasus.client.*; +import org.apache.commons.lang3.tuple.Pair; + +import java.time.Duration; + +public class TestScan { + private static final String metaServerAddrs = "127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603"; + private static final String tableName = "temp"; + + private static final String hashKey = "scan"; + private static final String startSortKey = ""; + private static final String stopSortKey = ""; + + private static final int batchSize = 10; + private static final int scanTimeoutMillis = 10000; + private static final Boolean startInclusive = true; + private static final Boolean stopInclusive = true; + private static final FilterType hashKeyFilterType = FT_NO_FILTER; + private static final String hashKeyFilterPattern = ""; + private static final FilterType sortKeyFilterType = FT_NO_FILTER; + private static final String sortKeyFilterPattern = ""; + private static final Boolean noValue = false; + + private static PegasusClientInterface client;//如果维护的table = client.openTable为全局变量，请务必确保client实例也是全局变量 + + public static void main(String[] args) throws InterruptedException { + ClientOptions clientOptions = + ClientOptions.builder() + .metaServers(metaServerAddrs) + .operationTimeout(Duration.ofMillis(1000)) + .build(); + // 1.也可以使用PegasusClientFactory.getSingletonClient(String configPath)创建单例 + // 2.如不希望构建单例，可以使用createClient(String configPath)或者createClient(ClientOptions options)创建 + try { + client = PegasusClientFactory.getSingletonClient(clientOptions); + } catch (PException e) { + e.printStackTrace(); + } + + ScanOptions options = new ScanOptions(); + options.timeoutMillis = 100; + options.batchSize = batchSize; + options.startInclusive = startInclusive; + options.stopInclusive = stopInclusive; + options.hashKeyFilterType = hashKeyFilterType; + options.hashKeyFilterPattern = hashKeyFilterPattern.getBytes(); + options.sortKeyFilterType = sortKeyFilterType; + options.sortKeyFilterPattern = sortKeyFilterPattern.getBytes(); + options.noValue = noValue; + + PegasusScannerInterface scanner = null; + try { + scanner = client.getScanner(tableName, hashKey.getBytes(), + startSortKey.getBytes(), stopSortKey.getBytes(), options); + } catch (PException e) { + e.printStackTrace(); + } + System.out.println("hash scan successfully:"); + Pair, byte[]> result = null; + try { + result = scanner.next(); + } catch (PException e) { + e.printStackTrace(); + } + while (result != null) { + try { + result = scanner.next(); + } catch (PException e) { + System.out.println(e.getMessage()); + } + + System.out.printf("hashKey=\"%s\", sortKey=\"%s\", value=\"%s\"\n", + new String(result.getKey().getKey()), new String(result.getKey().getValue()), new String(result.getValue())); + Thread.sleep(5000); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java index 46051433..3d21da1c 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -1,10 +1,12 @@ package com.xiaomi.infra.pegasus.client.request.range; -import com.xiaomi.infra.pegasus.client.DelRangeOptions; +import com.xiaomi.infra.pegasus.client.ScanOptions; + +public class DeleteRange extends Scan{ + + public DeleteRange(byte[] hashKey, int timeout) { + super(hashKey, timeout); + } + -public class DeleteRange { - private DelRangeOptions delRangeOptions; - private byte[] startSortKey; - private byte[] stopSortKey; - private int timeout; } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java deleted file mode 100644 index 5088c13c..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/FullScanRange.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.xiaomi.infra.pegasus.client.request.range; - -import com.xiaomi.infra.pegasus.client.ScanOptions; - -public class FullScanRange { - private ScanOptions scanOptions; - private int maxSplitCount; - private int timeout; -} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java new file mode 100644 index 00000000..4f260c2a --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -0,0 +1,22 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusTable; +import com.xiaomi.infra.pegasus.client.PegasusTableInterface; +import com.xiaomi.infra.pegasus.client.ScanOptions; + +public class GetRange extends Scan{ + public PegasusTableInterface table; + public int maxFetchCount; + + public GetRange(PegasusTableInterface table, byte[] hashKey, int maxFetchCount, int timeout) { + super(hashKey, timeout); + this.table = table; + this.maxFetchCount = maxFetchCount; + } + + public PegasusTable.ScanRangeResult commitAndWait() throws PException { + ScannerWrapper scannerWrapper = new ScannerWrapper(table); + return scannerWrapper.hashScan(hashKey, startSortKey, stopSortKey, scanOptions, maxFetchCount, timeout); + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java deleted file mode 100644 index 2a5b3baf..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/HashScanRange.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.xiaomi.infra.pegasus.client.request.range; - -import com.xiaomi.infra.pegasus.client.PException; -import com.xiaomi.infra.pegasus.client.PegasusTable; -import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -import com.xiaomi.infra.pegasus.client.ScanOptions; - -public class HashScanRange { - public PegasusTableInterface table; - - private ScanOptions scanOptions; - private byte[] hashKey; - private byte[] startSortKey; - private byte[] stopSortKey; - private int maxFetchCount; - private int timeout; - - public HashScanRange(PegasusTableInterface table, ScanOptions scanOptions, byte[] hashKey, byte[] startSortKey, byte[] stopSortKey, int maxFetchCount, int timeout) { - this.table = table; - this.scanOptions = scanOptions; - this.startSortKey = startSortKey; - this.hashKey = hashKey; - this.stopSortKey = stopSortKey; - this.maxFetchCount = maxFetchCount; - this.timeout = timeout; - } - - public PegasusTable.ScanRangeResult commitAndWait() throws PException { - ScannerWrapper scannerWrapper = new ScannerWrapper(table); - return scannerWrapper.hashScan(hashKey, startSortKey,stopSortKey,scanOptions, maxFetchCount, timeout); - } -} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java new file mode 100644 index 00000000..1ee37a97 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java @@ -0,0 +1,31 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.ScanOptions; + +public class Scan { + public byte[] hashKey; + public int timeout; + + public ScanOptions scanOptions = new ScanOptions(); + public byte[] startSortKey; + public byte[] stopSortKey; + + public Scan(byte[] hashKey, int timeout) { + this.hashKey = hashKey; + this.timeout = timeout; + } + + public Scan withOptions(ScanOptions scanOptions) { + this.scanOptions = scanOptions; + return this; + } + + public Scan withStartSortKey(byte[] startSortKey) { + this.startSortKey = startSortKey; + return this; + } + + public void withStopSortKey(byte[] stopSortKey) { + this.stopSortKey = stopSortKey; + } +} From 014752d0fcb14df3d17255f09f1e106211d0a1dd Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 16 Mar 2021 12:01:02 +0800 Subject: [PATCH 03/17] init --- scripts/format-all.sh | 3 +- .../client/request/range/DeleteRange.java | 69 +++++++++++++- .../client/request/range/GetRange.java | 23 ++--- .../pegasus/client/request/range/Range.java | 38 ++++++++ .../pegasus/client/request/range/Scan.java | 31 ------- .../client/request/range/ScannerWrapper.java | 93 +++++++++---------- 6 files changed, 161 insertions(+), 96 deletions(-) create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java diff --git a/scripts/format-all.sh b/scripts/format-all.sh index 9209c354..ae95b433 100755 --- a/scripts/format-all.sh +++ b/scripts/format-all.sh @@ -24,13 +24,14 @@ PROJECT_DIR=$(dirname "${SCRIPT_DIR}") cd "${PROJECT_DIR}" || exit 1 SRC_FILES=(src/main/java/com/xiaomi/infra/pegasus/client/*.java + src/main/java/com/xiaomi/infra/pegasus/client/request/batch/*.java + src/main/java/com/xiaomi/infra/pegasus/client/request/range/*.java src/main/java/com/xiaomi/infra/pegasus/metrics/*.java src/main/java/com/xiaomi/infra/pegasus/rpc/*.java src/main/java/com/xiaomi/infra/pegasus/rpc/async/*.java src/main/java/com/xiaomi/infra/pegasus/operator/*.java src/main/java/com/xiaomi/infra/pegasus/tools/*.java src/main/java/com/xiaomi/infra/pegasus/rpc/interceptor/*.java - src/main/java/com/xiaomi/infra/pegasus/client/request/*.java src/main/java/com/xiaomi/infra/pegasus/base/*.java src/main/java/com/xiaomi/infra/pegasus/example/*.java src/main/java/com/xiaomi/infra/pegasus/security/*.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java index 3d21da1c..6fe5e505 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -1,12 +1,73 @@ package com.xiaomi.infra.pegasus.client.request.range; -import com.xiaomi.infra.pegasus.client.ScanOptions; +import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusTable; +import com.xiaomi.infra.pegasus.client.PegasusTableInterface; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.commons.lang3.tuple.Pair; -public class DeleteRange extends Scan{ +public class DeleteRange extends Range { + public PegasusTableInterface table; + private byte[] nextSortKey; - public DeleteRange(byte[] hashKey, int timeout) { - super(hashKey, timeout); + public DeleteRange(PegasusTableInterface table, byte[] hashKey, int timeout) { + super(hashKey, timeout); + this.table = table; + } + + public Boolean commitAndWait(int maxDeleteCount) + throws PException, InterruptedException, ExecutionException, TimeoutException { + this.scanOptions.noValue = true; + ScannerWrapper scannerWrapper = new ScannerWrapper<>(table, this); + + if (timeout <= 0) timeout = ((PegasusTable) table).getDefaultTimeout(); + long deadlineTime = System.currentTimeMillis() + timeout; + PegasusTable.ScanRangeResult result = scannerWrapper.hashScan(maxDeleteCount, timeout); + + if (System.currentTimeMillis() >= deadlineTime) { + throw PException.timeout( + ((PegasusTable) table).getMetaList(), + ((PegasusTable) table).getTable().getTableName(), + new PegasusTable.Request(hashKey), + timeout, + new TimeoutException()); } + List sortKeys = new ArrayList<>(); + int remainingTime = (int) (deadlineTime - System.currentTimeMillis()); + for (Pair, byte[]> pair : result.results) { + remainingTime = (int) (deadlineTime - System.currentTimeMillis()); + sortKeys.add(pair.getKey().getValue()); + if (sortKeys.size() == this.scanOptions.batchSize) { + nextSortKey = sortKeys.get(0); + table + .asyncMultiDel(hashKey, sortKeys, remainingTime) + .get(remainingTime, TimeUnit.MILLISECONDS); + if (remainingTime <= 0) { + throw PException.timeout( + ((PegasusTable) table).getMetaList(), + ((PegasusTable) table).getTable().getTableName(), + new PegasusTable.Request(hashKey), + timeout, + new TimeoutException()); + } + sortKeys.clear(); + } + } + if (!sortKeys.isEmpty()) { + table + .asyncMultiDel(hashKey, sortKeys, remainingTime) + .get(remainingTime, TimeUnit.MILLISECONDS); + nextSortKey = null; + } + return result.allFetched; + } + public byte[] getNextSortKey() { + return nextSortKey; + } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index 4f260c2a..0a2d3dae 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -3,20 +3,17 @@ import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusTable; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -import com.xiaomi.infra.pegasus.client.ScanOptions; -public class GetRange extends Scan{ - public PegasusTableInterface table; - public int maxFetchCount; +public class GetRange extends Range { + public PegasusTableInterface table; - public GetRange(PegasusTableInterface table, byte[] hashKey, int maxFetchCount, int timeout) { - super(hashKey, timeout); - this.table = table; - this.maxFetchCount = maxFetchCount; - } + public GetRange(PegasusTableInterface table, byte[] hashKey, int timeout) { + super(hashKey, timeout); + this.table = table; + } - public PegasusTable.ScanRangeResult commitAndWait() throws PException { - ScannerWrapper scannerWrapper = new ScannerWrapper(table); - return scannerWrapper.hashScan(hashKey, startSortKey, stopSortKey, scanOptions, maxFetchCount, timeout); - } + public PegasusTable.ScanRangeResult commitAndWait(int maxFetchCount) throws PException { + ScannerWrapper scannerWrapper = new ScannerWrapper<>(table, this); + return scannerWrapper.hashScan(maxFetchCount, timeout); + } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java new file mode 100644 index 00000000..33a4b224 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java @@ -0,0 +1,38 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.ScanOptions; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; + +public abstract class Range { + protected byte[] hashKey; + protected int timeout; + + protected ScanOptions scanOptions = new ScanOptions(); + protected byte[] startSortKey; + protected byte[] stopSortKey; + + protected Range(byte[] hashKey, int timeout) { + this.hashKey = hashKey; + this.timeout = timeout; + } + + protected abstract Response commitAndWait(int maxRangeCount) + throws PException, InterruptedException, ExecutionException, TimeoutException; + + protected Range withOptions(ScanOptions scanOptions) { + this.scanOptions = scanOptions; + return this; + } + + protected Range withStartSortKey(byte[] startSortKey) { + this.startSortKey = startSortKey; + return this; + } + + protected Range withStopSortKey(byte[] stopSortKey) { + this.stopSortKey = stopSortKey; + return this; + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java deleted file mode 100644 index 1ee37a97..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Scan.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.xiaomi.infra.pegasus.client.request.range; - -import com.xiaomi.infra.pegasus.client.ScanOptions; - -public class Scan { - public byte[] hashKey; - public int timeout; - - public ScanOptions scanOptions = new ScanOptions(); - public byte[] startSortKey; - public byte[] stopSortKey; - - public Scan(byte[] hashKey, int timeout) { - this.hashKey = hashKey; - this.timeout = timeout; - } - - public Scan withOptions(ScanOptions scanOptions) { - this.scanOptions = scanOptions; - return this; - } - - public Scan withStartSortKey(byte[] startSortKey) { - this.startSortKey = startSortKey; - return this; - } - - public void withStopSortKey(byte[] stopSortKey) { - this.stopSortKey = stopSortKey; - } -} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java index 6f1182a7..d87ab50b 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -1,57 +1,56 @@ package com.xiaomi.infra.pegasus.client.request.range; import com.xiaomi.infra.pegasus.client.*; -import com.xiaomi.infra.pegasus.rpc.Table; -import org.apache.commons.lang3.tuple.Pair; - import java.util.ArrayList; import java.util.concurrent.TimeoutException; +import org.apache.commons.lang3.tuple.Pair; -public class ScannerWrapper { - private final PegasusTableInterface table; - - public ScannerWrapper(PegasusTableInterface table) { - this.table = table; +public class ScannerWrapper { + private final Range request; + private final PegasusTableInterface table; + private final PegasusScannerInterface scanner; + + public ScannerWrapper(PegasusTableInterface table, Range request) throws PException { + this.table = table; + this.request = request; + this.scanner = + table.getScanner( + request.hashKey, request.startSortKey, request.stopSortKey, request.scanOptions); + } + + PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) throws PException { + if (timeout <= 0) timeout = ((PegasusTable) table).getDefaultTimeout(); + long deadlineTime = System.currentTimeMillis() + timeout; + + PegasusTable.ScanRangeResult scanRangeResult = new PegasusTable.ScanRangeResult(); + scanRangeResult.allFetched = false; + scanRangeResult.results = new ArrayList<>(); + if (System.currentTimeMillis() >= deadlineTime) { + throw PException.timeout( + ((PegasusTable) table).getMetaList(), + ((PegasusTable) table).getTable().getTableName(), + new PegasusTable.Request(request.hashKey), + timeout, + new TimeoutException()); } - PegasusTable.ScanRangeResult hashScan( - byte[] hashKey, - byte[] startSortKey, - byte[] stopSortKey, - ScanOptions options, - int maxFetchCount, - int timeout /*ms*/) - throws PException { - if (timeout <= 0) timeout = ((PegasusTable)table).getDefaultTimeout(); - long deadlineTime = System.currentTimeMillis() + timeout; - - PegasusScannerInterface pegasusScanner = - table.getScanner(hashKey, startSortKey, stopSortKey, options); - PegasusTable.ScanRangeResult scanRangeResult = new PegasusTable.ScanRangeResult(); - scanRangeResult.allFetched = false; - scanRangeResult.results = new ArrayList<>(); - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - ((PegasusTable)table).getMetaList(), - ((PegasusTable)table).getTable().getTableName(), - new PegasusTable.Request(hashKey), timeout, new TimeoutException()); - } - - Pair, byte[]> pair; - while ((pair = pegasusScanner.next()) != null - && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - ((PegasusTable)table).getMetaList(), - ((PegasusTable)table).getTable().getTableName(), - new PegasusTable.Request(hashKey), timeout, new TimeoutException()); - } - scanRangeResult.results.add(pair); - } + Pair, byte[]> pair; + while ((pair = scanner.next()) != null + && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { + if (System.currentTimeMillis() >= deadlineTime) { + throw PException.timeout( + ((PegasusTable) table).getMetaList(), + ((PegasusTable) table).getTable().getTableName(), + new PegasusTable.Request(request.hashKey), + timeout, + new TimeoutException()); + } + scanRangeResult.results.add(pair); + } - if (pegasusScanner.next() == null) { - scanRangeResult.allFetched = true; - } - return scanRangeResult; - } + if (scanner.next() == null) { + scanRangeResult.allFetched = true; + } + return scanRangeResult; + } } From caf8f404e7dd1655a8ef16317cee95b15d867080 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 16 Mar 2021 12:06:16 +0800 Subject: [PATCH 04/17] init --- .../com/xiaomi/infra/pegasus/TestScan.java | 81 ------------------- .../client/request/range/ScannerWrapper.java | 2 +- 2 files changed, 1 insertion(+), 82 deletions(-) delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/TestScan.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/TestScan.java b/src/main/java/com/xiaomi/infra/pegasus/TestScan.java deleted file mode 100644 index 0678fd06..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/TestScan.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.xiaomi.infra.pegasus; - -import static com.xiaomi.infra.pegasus.client.FilterType.*; - -import com.xiaomi.infra.pegasus.client.*; -import org.apache.commons.lang3.tuple.Pair; - -import java.time.Duration; - -public class TestScan { - private static final String metaServerAddrs = "127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603"; - private static final String tableName = "temp"; - - private static final String hashKey = "scan"; - private static final String startSortKey = ""; - private static final String stopSortKey = ""; - - private static final int batchSize = 10; - private static final int scanTimeoutMillis = 10000; - private static final Boolean startInclusive = true; - private static final Boolean stopInclusive = true; - private static final FilterType hashKeyFilterType = FT_NO_FILTER; - private static final String hashKeyFilterPattern = ""; - private static final FilterType sortKeyFilterType = FT_NO_FILTER; - private static final String sortKeyFilterPattern = ""; - private static final Boolean noValue = false; - - private static PegasusClientInterface client;//如果维护的table = client.openTable为全局变量，请务必确保client实例也是全局变量 - - public static void main(String[] args) throws InterruptedException { - ClientOptions clientOptions = - ClientOptions.builder() - .metaServers(metaServerAddrs) - .operationTimeout(Duration.ofMillis(1000)) - .build(); - // 1.也可以使用PegasusClientFactory.getSingletonClient(String configPath)创建单例 - // 2.如不希望构建单例，可以使用createClient(String configPath)或者createClient(ClientOptions options)创建 - try { - client = PegasusClientFactory.getSingletonClient(clientOptions); - } catch (PException e) { - e.printStackTrace(); - } - - ScanOptions options = new ScanOptions(); - options.timeoutMillis = 100; - options.batchSize = batchSize; - options.startInclusive = startInclusive; - options.stopInclusive = stopInclusive; - options.hashKeyFilterType = hashKeyFilterType; - options.hashKeyFilterPattern = hashKeyFilterPattern.getBytes(); - options.sortKeyFilterType = sortKeyFilterType; - options.sortKeyFilterPattern = sortKeyFilterPattern.getBytes(); - options.noValue = noValue; - - PegasusScannerInterface scanner = null; - try { - scanner = client.getScanner(tableName, hashKey.getBytes(), - startSortKey.getBytes(), stopSortKey.getBytes(), options); - } catch (PException e) { - e.printStackTrace(); - } - System.out.println("hash scan successfully:"); - Pair, byte[]> result = null; - try { - result = scanner.next(); - } catch (PException e) { - e.printStackTrace(); - } - while (result != null) { - try { - result = scanner.next(); - } catch (PException e) { - System.out.println(e.getMessage()); - } - - System.out.printf("hashKey=\"%s\", sortKey=\"%s\", value=\"%s\"\n", - new String(result.getKey().getKey()), new String(result.getKey().getValue()), new String(result.getValue())); - Thread.sleep(5000); - } - } -} \ No newline at end of file diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java index d87ab50b..48cfc738 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -10,7 +10,7 @@ public class ScannerWrapper { private final PegasusTableInterface table; private final PegasusScannerInterface scanner; - public ScannerWrapper(PegasusTableInterface table, Range request) throws PException { + ScannerWrapper(PegasusTableInterface table, Range request) throws PException { this.table = table; this.request = request; this.scanner = From 084d73d3ffb6aa012c73ff69a89625cae6e68112 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 16 Mar 2021 17:10:58 +0800 Subject: [PATCH 05/17] format --- .../infra/pegasus/client/request/range/ScannerWrapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java index 48cfc738..4d49faed 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -18,6 +18,7 @@ public class ScannerWrapper { request.hashKey, request.startSortKey, request.stopSortKey, request.scanOptions); } + // TODO(jiashuo1) scanner.next()'s bug need be fixed, it will be put next pr PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) throws PException { if (timeout <= 0) timeout = ((PegasusTable) table).getDefaultTimeout(); long deadlineTime = System.currentTimeMillis() + timeout; From 36ec8473fcdce0a5be546f2b4b95d70dc1143389 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 16 Mar 2021 19:29:28 +0800 Subject: [PATCH 06/17] format --- .../infra/pegasus/client/PegasusTable.java | 61 +-------- .../client/request/range/DeleteRange.java | 57 +++------ .../client/request/range/GetRange.java | 17 +-- .../request/range/GetRangeWithValue.java | 18 +++ .../pegasus/client/request/range/Range.java | 19 +-- .../client/request/range/ScannerWrapper.java | 50 ++++++-- .../infra/pegasus/client/TestBasic.java | 97 ++------------- .../infra/pegasus/client/TestRange.java | 116 ++++++++++++++++++ 8 files changed, 224 insertions(+), 211 deletions(-) create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java create mode 100644 src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java index ff66161a..1483663e 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java @@ -7,6 +7,7 @@ import com.xiaomi.infra.pegasus.base.blob; import com.xiaomi.infra.pegasus.base.error_code; import com.xiaomi.infra.pegasus.base.gpid; +import com.xiaomi.infra.pegasus.client.request.range.GetRange; import com.xiaomi.infra.pegasus.operator.*; import com.xiaomi.infra.pegasus.rpc.ReplicationException; import com.xiaomi.infra.pegasus.rpc.Table; @@ -1185,16 +1186,9 @@ public int batchMultiGet2( public MultiGetSortKeysResult multiGetSortKeys( byte[] hashKey, int maxFetchCount, int maxFetchSize, int timeout) throws PException { if (timeout <= 0) timeout = defaultTimeout; - MultiGetSortKeysResult sortKeysResult = new MultiGetSortKeysResult(); - sortKeysResult.keys = new ArrayList<>(); - ScanOptions options = new ScanOptions(); - options.noValue = true; - ScanRangeResult result = scanRange(hashKey, null, null, options, maxFetchCount, timeout); - for (Pair, byte[]> pair : result.results) { - sortKeysResult.keys.add(pair.getLeft().getValue()); - } - sortKeysResult.allFetched = result.allFetched; - return sortKeysResult; + ; + GetRange getRange = new GetRange(this, hashKey, timeout); + return getRange.commitAndWait(maxFetchCount).convertMultiGetSortKeysResult(); } @Override @@ -1830,18 +1824,10 @@ public List getUnorderedScanners( return ret; } - /** - * {@linkplain #scanRange(byte[], byte[], byte[], ScanOptions, int, int)} result, if fetch all - * data for {startSortKey, stopSortKey}, ScanRangeResult.allFetched=true - */ - public static class ScanRangeResult { - public List, byte[]>> results; - public boolean allFetched; - } - /** * get scan result for {startSortKey, stopSortKey} within hashKey * + * @deprecated it will be replace * @param hashKey used to decide which partition to put this k-v, * @param startSortKey start sort key scan from if null or length == 0, means start from begin * @param stopSortKey stop sort key scan to if null or length == 0, means stop to end @@ -1854,43 +1840,6 @@ public static class ScanRangeResult { * {startSortKey, stopSortKey}, ScanRangeResult.allFetched=true * @throws PException */ - ScanRangeResult scanRange( - byte[] hashKey, - byte[] startSortKey, - byte[] stopSortKey, - ScanOptions options, - int maxFetchCount, - int timeout /*ms*/) - throws PException { - if (timeout <= 0) timeout = defaultTimeout; - long deadlineTime = System.currentTimeMillis() + timeout; - - PegasusScannerInterface pegasusScanner = - getScanner(hashKey, startSortKey, stopSortKey, options); - ScanRangeResult scanRangeResult = new ScanRangeResult(); - scanRangeResult.allFetched = false; - scanRangeResult.results = new ArrayList<>(); - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - metaList, table.getTableName(), new Request(hashKey), timeout, new TimeoutException()); - } - - Pair, byte[]> pair; - while ((pair = pegasusScanner.next()) != null - && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - metaList, table.getTableName(), new Request(hashKey), timeout, new TimeoutException()); - } - scanRangeResult.results.add(pair); - } - - if (pegasusScanner.next() == null) { - scanRangeResult.allFetched = true; - } - return scanRangeResult; - } - public void handleReplicaException( Request request, DefaultPromise promise, client_operator op, Table table, int timeout) { if (timeout <= 0) timeout = defaultTimeout; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java index 6fe5e505..a2bdf13e 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -1,34 +1,28 @@ package com.xiaomi.infra.pegasus.client.request.range; +import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; + import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusTable; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.commons.lang3.tuple.Pair; public class DeleteRange extends Range { - public PegasusTableInterface table; - private byte[] nextSortKey; public DeleteRange(PegasusTableInterface table, byte[] hashKey, int timeout) { - super(hashKey, timeout); - this.table = table; + super(table, hashKey, timeout); } - public Boolean commitAndWait(int maxDeleteCount) - throws PException, InterruptedException, ExecutionException, TimeoutException { - this.scanOptions.noValue = true; - ScannerWrapper scannerWrapper = new ScannerWrapper<>(table, this); - - if (timeout <= 0) timeout = ((PegasusTable) table).getDefaultTimeout(); + public Boolean commitAndWait(int maxDeleteCount) throws PException { long deadlineTime = System.currentTimeMillis() + timeout; - PegasusTable.ScanRangeResult result = scannerWrapper.hashScan(maxDeleteCount, timeout); - if (System.currentTimeMillis() >= deadlineTime) { + this.scanOptions.noValue = true; + ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); + Result result = scannerWrapper.hashScan(maxDeleteCount); + int remainingTime = (int) (deadlineTime - System.currentTimeMillis()); + if (remainingTime <= 0) { throw PException.timeout( ((PegasusTable) table).getMetaList(), ((PegasusTable) table).getTable().getTableName(), @@ -37,37 +31,14 @@ public Boolean commitAndWait(int maxDeleteCount) new TimeoutException()); } - List sortKeys = new ArrayList<>(); - int remainingTime = (int) (deadlineTime - System.currentTimeMillis()); - for (Pair, byte[]> pair : result.results) { - remainingTime = (int) (deadlineTime - System.currentTimeMillis()); - sortKeys.add(pair.getKey().getValue()); - if (sortKeys.size() == this.scanOptions.batchSize) { - nextSortKey = sortKeys.get(0); - table - .asyncMultiDel(hashKey, sortKeys, remainingTime) - .get(remainingTime, TimeUnit.MILLISECONDS); - if (remainingTime <= 0) { - throw PException.timeout( - ((PegasusTable) table).getMetaList(), - ((PegasusTable) table).getTable().getTableName(), - new PegasusTable.Request(hashKey), - timeout, - new TimeoutException()); - } - sortKeys.clear(); - } - } - if (!sortKeys.isEmpty()) { + try { table - .asyncMultiDel(hashKey, sortKeys, remainingTime) + .asyncMultiDel(hashKey, result.convertMultiGetSortKeysResult().keys, remainingTime) .get(remainingTime, TimeUnit.MILLISECONDS); - nextSortKey = null; + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new PException("Delete range failed!", e); } - return result.allFetched; - } - public byte[] getNextSortKey() { - return nextSortKey; + return result.allFetched; } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index 0a2d3dae..8c43943c 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -1,19 +1,20 @@ package com.xiaomi.infra.pegasus.client.request.range; +import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; + import com.xiaomi.infra.pegasus.client.PException; -import com.xiaomi.infra.pegasus.client.PegasusTable; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -public class GetRange extends Range { - public PegasusTableInterface table; +public class GetRange extends Range { public GetRange(PegasusTableInterface table, byte[] hashKey, int timeout) { - super(hashKey, timeout); - this.table = table; + super(table, hashKey, timeout); } - public PegasusTable.ScanRangeResult commitAndWait(int maxFetchCount) throws PException { - ScannerWrapper scannerWrapper = new ScannerWrapper<>(table, this); - return scannerWrapper.hashScan(maxFetchCount, timeout); + @Override + public Result commitAndWait(int maxFetchCount) throws PException { + this.scanOptions.noValue = true; + ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); + return scannerWrapper.hashScan(maxFetchCount); } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java new file mode 100644 index 00000000..78d78c85 --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java @@ -0,0 +1,18 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; + +import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusTableInterface; + +public class GetRangeWithValue extends Range { + + public GetRangeWithValue(PegasusTableInterface table, byte[] hashKey, int timeout) { + super(table, hashKey, timeout); + } + + public Result commitAndWait(int maxFetchCount) throws PException { + ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); + return scannerWrapper.hashScan(maxFetchCount); + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java index 33a4b224..4c87f70a 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java @@ -1,11 +1,12 @@ package com.xiaomi.infra.pegasus.client.request.range; import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusTable; +import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import com.xiaomi.infra.pegasus.client.ScanOptions; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; public abstract class Range { + public PegasusTableInterface table; protected byte[] hashKey; protected int timeout; @@ -13,25 +14,25 @@ public abstract class Range { protected byte[] startSortKey; protected byte[] stopSortKey; - protected Range(byte[] hashKey, int timeout) { + public Range(PegasusTableInterface table, byte[] hashKey, int timeout) { + this.table = table; this.hashKey = hashKey; - this.timeout = timeout; + this.timeout = timeout <= 0 ? ((PegasusTable) table).getDefaultTimeout() : timeout; } - protected abstract Response commitAndWait(int maxRangeCount) - throws PException, InterruptedException, ExecutionException, TimeoutException; + public abstract Response commitAndWait(int maxRangeCount) throws PException; - protected Range withOptions(ScanOptions scanOptions) { + public Range withOptions(ScanOptions scanOptions) { this.scanOptions = scanOptions; return this; } - protected Range withStartSortKey(byte[] startSortKey) { + public Range withStartSortKey(byte[] startSortKey) { this.startSortKey = startSortKey; return this; } - protected Range withStopSortKey(byte[] stopSortKey) { + public Range withStopSortKey(byte[] stopSortKey) { this.stopSortKey = stopSortKey; return this; } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java index 4d49faed..b66221fe 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -1,7 +1,11 @@ package com.xiaomi.infra.pegasus.client.request.range; +import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; +import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetSortKeysResult; + import com.xiaomi.infra.pegasus.client.*; import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.tuple.Pair; @@ -10,20 +14,19 @@ public class ScannerWrapper { private final PegasusTableInterface table; private final PegasusScannerInterface scanner; - ScannerWrapper(PegasusTableInterface table, Range request) throws PException { - this.table = table; + public ScannerWrapper(Range request) throws PException { this.request = request; + this.table = request.table; this.scanner = table.getScanner( request.hashKey, request.startSortKey, request.stopSortKey, request.scanOptions); } // TODO(jiashuo1) scanner.next()'s bug need be fixed, it will be put next pr - PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) throws PException { - if (timeout <= 0) timeout = ((PegasusTable) table).getDefaultTimeout(); - long deadlineTime = System.currentTimeMillis() + timeout; + public Result hashScan(int maxFetchCount) throws PException { + long deadlineTime = System.currentTimeMillis() + request.timeout; - PegasusTable.ScanRangeResult scanRangeResult = new PegasusTable.ScanRangeResult(); + Result scanRangeResult = new Result(); scanRangeResult.allFetched = false; scanRangeResult.results = new ArrayList<>(); if (System.currentTimeMillis() >= deadlineTime) { @@ -31,7 +34,7 @@ PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) thr ((PegasusTable) table).getMetaList(), ((PegasusTable) table).getTable().getTableName(), new PegasusTable.Request(request.hashKey), - timeout, + request.timeout, new TimeoutException()); } @@ -43,7 +46,7 @@ PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) thr ((PegasusTable) table).getMetaList(), ((PegasusTable) table).getTable().getTableName(), new PegasusTable.Request(request.hashKey), - timeout, + request.timeout, new TimeoutException()); } scanRangeResult.results.add(pair); @@ -54,4 +57,35 @@ PegasusTable.ScanRangeResult hashScan(int maxFetchCount, int timeout /*ms*/) thr } return scanRangeResult; } + + public static class Result { + public List, byte[]>> results; + public boolean allFetched; + + public MultiGetResult convertMultiGetResult() { + MultiGetResult multiGetResult = new MultiGetResult(); + if (results == null) { + return multiGetResult; + } + multiGetResult.values = new ArrayList<>(); + for (Pair, byte[]> pair : results) { + multiGetResult.values.add(Pair.of(pair.getLeft().getValue(), pair.getValue())); + } + multiGetResult.allFetched = allFetched; + return multiGetResult; + } + + public MultiGetSortKeysResult convertMultiGetSortKeysResult() { + MultiGetSortKeysResult multiGetSortKeysResult = new MultiGetSortKeysResult(); + if (results == null) { + return multiGetSortKeysResult; + } + multiGetSortKeysResult.keys = new ArrayList<>(); + for (Pair, byte[]> pair : results) { + multiGetSortKeysResult.keys.add(pair.getLeft().getValue()); + } + multiGetSortKeysResult.allFetched = allFetched; + return multiGetSortKeysResult; + } + } } diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestBasic.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestBasic.java index 563e7899..740b4462 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestBasic.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestBasic.java @@ -4,7 +4,6 @@ package com.xiaomi.infra.pegasus.client; /** @author qinzuoyan */ -import com.xiaomi.infra.pegasus.client.PegasusTable.ScanRangeResult; import com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetSortKeysResult; import io.netty.util.concurrent.Future; import java.util.ArrayList; @@ -2632,8 +2631,8 @@ private void testWriteSizeLimit(PegasusClientInterface client) { } } - @Test // test for making sure return "maxFetchCount" if has "maxFetchCount" valid record - public void testScanRangeWithValueExpired() throws PException, InterruptedException { + @Test + public void testGetSortKeysWithExpiredValue() throws PException, InterruptedException { String tableName = "temp"; String hashKey = "hashKey"; // generate records: sortKeys=[expired_0....expired_999,persistent_0...persistent_9] @@ -2641,78 +2640,18 @@ public void testScanRangeWithValueExpired() throws PException, InterruptedExcept PegasusTable table = (PegasusTable) PegasusClientFactory.getSingletonClient().openTable(tableName); - // case A: scan all record - // case A1: scan all record: if persistent record count >= maxFetchCount, it must return - // maxFetchCount records - ScanRangeResult caseA1 = - table.scanRange(hashKey.getBytes(), null, null, new ScanOptions(), 5, 0); - assertScanResult(0, 4, false, caseA1); - // case A2: scan all record: if persistent record count < maxFetchCount, it only return - // persistent count records - ScanRangeResult caseA2 = - table.scanRange(hashKey.getBytes(), null, null, new ScanOptions(), 100, 0); - assertScanResult(0, 9, true, caseA2); - - // case B: scan limit record by "startSortKey" and "": - // case B1: scan limit record by "expired_0" and "", if persistent record count >= - // maxFetchCount, it must return maxFetchCount records - ScanRangeResult caseB1 = - table.scanRange( - hashKey.getBytes(), "expired_0".getBytes(), "".getBytes(), new ScanOptions(), 5, 0); - assertScanResult(0, 4, false, caseB1); - // case B2: scan limit record by "expired_0" and "", if persistent record count < maxFetchCount, - // it only return valid records - ScanRangeResult caseB2 = - table.scanRange( - hashKey.getBytes(), "expired_0".getBytes(), "".getBytes(), new ScanOptions(), 50, 0); - assertScanResult(0, 9, true, caseB2); - // case B3: scan limit record by "persistent_5" and "", if following persistent record count < - // maxFetchCount, it only return valid records - ScanRangeResult caseB3 = - table.scanRange( - hashKey.getBytes(), "persistent_5".getBytes(), "".getBytes(), new ScanOptions(), 50, 0); - assertScanResult(5, 9, true, caseB3); - // case B4: scan limit record by "persistent_5" and "", if following persistent record count > - // maxFetchCount, it only return valid records - ScanRangeResult caseB4 = - table.scanRange( - hashKey.getBytes(), "persistent_5".getBytes(), "".getBytes(), new ScanOptions(), 3, 0); - assertScanResult(5, 7, false, caseB4); - - // case C: scan limit record by "" and "stopSortKey": - // case C1: scan limit record by "" and "expired_7", if will return 0 record - ScanRangeResult caseC1 = - table.scanRange( - hashKey.getBytes(), "".getBytes(), "expired_7".getBytes(), new ScanOptions(), 3, 0); - Assert.assertTrue(caseC1.allFetched); - Assert.assertEquals(0, caseC1.results.size()); // among "" and "expired_7" has 0 valid record - // case C2: scan limit record by "" and "persistent_7", if valid record count < maxFetchCount, - // it only return valid record - ScanRangeResult caseC2 = - table.scanRange( - hashKey.getBytes(), "".getBytes(), "persistent_7".getBytes(), new ScanOptions(), 10, 0); - assertScanResult(0, 6, true, caseC2); - // case C3: scan limit record by "" and "persistent_7", if valid record count > maxFetchCount, - // it only return valid record - ScanRangeResult caseC3 = - table.scanRange( - hashKey.getBytes(), "".getBytes(), "persistent_7".getBytes(), new ScanOptions(), 2, 0); - assertScanResult(0, 1, false, caseC3); - - // case D: use multiGetSortKeys, which actually equal with case A but no value - // case D1: maxFetchCount > 0, return maxFetchCount valid record - MultiGetSortKeysResult caseD1 = table.multiGetSortKeys(hashKey.getBytes(), 5, -1, 0); - Assert.assertFalse(caseD1.allFetched); - Assert.assertEquals(5, caseD1.keys.size()); + MultiGetSortKeysResult result1 = table.multiGetSortKeys(hashKey.getBytes(), 5, -1, 0); + Assertions.assertFalse(result1.allFetched); + Assertions.assertEquals(5, result1.keys.size()); for (int i = 0; i <= 4; i++) { - Assertions.assertEquals("persistent_" + i, new String(caseD1.keys.get(i))); + Assertions.assertEquals("persistent_" + i, new String(result1.keys.get(i))); } // case D1: maxFetchCount < 0, return all valid record - MultiGetSortKeysResult caseD2 = table.multiGetSortKeys(hashKey.getBytes(), 10, -1, 0); - Assert.assertTrue(caseD2.allFetched); - Assert.assertEquals(10, caseD2.keys.size()); + MultiGetSortKeysResult result2 = table.multiGetSortKeys(hashKey.getBytes(), 10, -1, 0); + Assertions.assertTrue(result2.allFetched); + Assertions.assertEquals(10, result2.keys.size()); for (int i = 0; i <= 9; i++) { - Assertions.assertEquals("persistent_" + i, new String(caseD2.keys.get(i))); + Assertions.assertEquals("persistent_" + i, new String(result2.keys.get(i))); } } @@ -2742,20 +2681,4 @@ private void generateRecordsWithExpired( } PegasusClientFactory.closeSingletonClient(); } - - private void assertScanResult( - int startIndex, int stopIndex, boolean expectAllFetched, ScanRangeResult actuallyRes) { - Assertions.assertEquals(expectAllFetched, actuallyRes.allFetched); - Assertions.assertEquals(stopIndex - startIndex + 1, actuallyRes.results.size()); - for (int i = startIndex; i <= stopIndex; i++) { - Assertions.assertEquals( - "hashKey", new String(actuallyRes.results.get(i - startIndex).getLeft().getKey())); - Assertions.assertEquals( - "persistent_" + i, - new String(actuallyRes.results.get(i - startIndex).getLeft().getValue())); - Assertions.assertEquals( - "persistent_" + i + "_value", - new String(actuallyRes.results.get(i - startIndex).getRight())); - } - } } diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java new file mode 100644 index 00000000..f5d49d28 --- /dev/null +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -0,0 +1,116 @@ +package com.xiaomi.infra.pegasus.client; + +import com.xiaomi.infra.pegasus.client.request.range.GetRangeWithValue; +import com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestRange { + + @Test // test for making sure return "maxFetchCount" if has "maxFetchCount" valid record + public void testScanRangeWithValueExpired() + throws PException, InterruptedException, TimeoutException, ExecutionException { + String tableName = "temp"; + String hashKey = "hashKey"; + // generate records: sortKeys=[expired_0....expired_999,persistent_0...persistent_9] + generateRecordsWithExpired(tableName, hashKey, 1000, 10); + + PegasusTable table = + (PegasusTable) PegasusClientFactory.getSingletonClient().openTable(tableName); + GetRangeWithValue getRangeWithValue = new GetRangeWithValue(table, hashKey.getBytes(), 0); + // case A: scan all record + // case A1: scan all record: if persistent record count >= maxFetchCount, it must return + // maxFetchCount records + ScannerWrapper.Result caseA1 = getRangeWithValue.commitAndWait(5); + assertScanResult(0, 4, false, caseA1); + // case A2: scan all record: if persistent record count < maxFetchCount, it only return + // persistent count records + ScannerWrapper.Result caseA2 = getRangeWithValue.commitAndWait(100); + assertScanResult(0, 9, true, caseA2); + + // case B: scan limit record by "startSortKey" and "": + // case B1: scan limit record by "expired_0" and "", if persistent record count >= + // maxFetchCount, it must return maxFetchCount records + ScannerWrapper.Result caseB1 = + getRangeWithValue.withStartSortKey("expired_0".getBytes()).commitAndWait(5); + assertScanResult(0, 4, false, caseB1); + // case B2: scan limit record by "expired_0" and "", if persistent record count < maxFetchCount, + // it only return valid records + ScannerWrapper.Result caseB2 = + getRangeWithValue.withStartSortKey("expired_0".getBytes()).commitAndWait(50); + assertScanResult(0, 9, true, caseB2); + // case B3: scan limit record by "persistent_5" and "", if following persistent record count < + // maxFetchCount, it only return valid records + ScannerWrapper.Result caseB3 = + getRangeWithValue.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); + assertScanResult(5, 9, true, caseB3); + // case B4: scan limit record by "persistent_5" and "", if following persistent record count > + // maxFetchCount, it only return valid records + ScannerWrapper.Result caseB4 = + getRangeWithValue.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); + assertScanResult(5, 7, false, caseB4); + + // case C: scan limit record by "" and "stopSortKey": + // case C1: scan limit record by "" and "expired_7", if will return 0 record + ScannerWrapper.Result caseC1 = + getRangeWithValue.withStopSortKey("expired_7".getBytes()).commitAndWait(3); + Assertions.assertTrue(caseC1.allFetched); + Assertions.assertEquals( + 0, caseC1.results.size()); // among "" and "expired_7" has 0 valid record + // case C2: scan limit record by "" and "persistent_7", if valid record count < maxFetchCount, + // it only return valid record + ScannerWrapper.Result caseC2 = + getRangeWithValue.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); + assertScanResult(0, 6, true, caseC2); + // case C3: scan limit record by "" and "persistent_7", if valid record count > maxFetchCount, + // it only return valid record + ScannerWrapper.Result caseC3 = + getRangeWithValue.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); + assertScanResult(0, 1, false, caseC3); + } + + private void generateRecordsWithExpired( + String tableName, String hashKey, int expiredCount, int persistentCount) + throws PException, InterruptedException { + PegasusClientInterface client = PegasusClientFactory.getSingletonClient(); + // assign prefix to make sure the expire record is stored front of persistent + String expiredSortKeyPrefix = "expired_"; + String persistentSortKeyPrefix = "persistent_"; + while (expiredCount-- > 0) { + client.set( + tableName, + hashKey.getBytes(), + (expiredSortKeyPrefix + expiredCount).getBytes(), + (expiredSortKeyPrefix + expiredCount + "_value").getBytes(), + 1); + } + // sleep to make sure the record is expired + Thread.sleep(1000); + while (persistentCount-- > 0) { + client.set( + tableName, + hashKey.getBytes(), + (persistentSortKeyPrefix + persistentCount).getBytes(), + (persistentSortKeyPrefix + persistentCount + "_value").getBytes()); + } + PegasusClientFactory.closeSingletonClient(); + } + + private void assertScanResult( + int startIndex, int stopIndex, boolean expectAllFetched, ScannerWrapper.Result actuallyRes) { + Assertions.assertEquals(expectAllFetched, actuallyRes.allFetched); + Assertions.assertEquals(stopIndex - startIndex + 1, actuallyRes.results.size()); + for (int i = startIndex; i <= stopIndex; i++) { + Assertions.assertEquals( + "hashKey", new String(actuallyRes.results.get(i - startIndex).getLeft().getKey())); + Assertions.assertEquals( + "persistent_" + i, + new String(actuallyRes.results.get(i - startIndex).getLeft().getValue())); + Assertions.assertEquals( + "persistent_" + i + "_value", + new String(actuallyRes.results.get(i - startIndex).getRight())); + } + } +} From f99619818d2142b651a3a252f8b07a311c62f49a Mon Sep 17 00:00:00 2001 From: jiashuo Date: Tue, 16 Mar 2021 19:33:57 +0800 Subject: [PATCH 07/17] format --- .../infra/pegasus/client/PegasusTable.java | 7 +++--- .../client/request/range/GetRange.java | 2 -- ...ngeWithValue.java => GetRangeNoValue.java} | 6 +++-- .../infra/pegasus/client/TestRange.java | 22 +++++++++---------- 4 files changed, 18 insertions(+), 19 deletions(-) rename src/main/java/com/xiaomi/infra/pegasus/client/request/range/{GetRangeWithValue.java => GetRangeNoValue.java} (72%) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java index 1483663e..8ede1daf 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java @@ -7,7 +7,7 @@ import com.xiaomi.infra.pegasus.base.blob; import com.xiaomi.infra.pegasus.base.error_code; import com.xiaomi.infra.pegasus.base.gpid; -import com.xiaomi.infra.pegasus.client.request.range.GetRange; +import com.xiaomi.infra.pegasus.client.request.range.GetRangeNoValue; import com.xiaomi.infra.pegasus.operator.*; import com.xiaomi.infra.pegasus.rpc.ReplicationException; import com.xiaomi.infra.pegasus.rpc.Table; @@ -1186,9 +1186,8 @@ public int batchMultiGet2( public MultiGetSortKeysResult multiGetSortKeys( byte[] hashKey, int maxFetchCount, int maxFetchSize, int timeout) throws PException { if (timeout <= 0) timeout = defaultTimeout; - ; - GetRange getRange = new GetRange(this, hashKey, timeout); - return getRange.commitAndWait(maxFetchCount).convertMultiGetSortKeysResult(); + GetRangeNoValue getRangeNoValue = new GetRangeNoValue(this, hashKey, timeout); + return getRangeNoValue.commitAndWait(maxFetchCount).convertMultiGetSortKeysResult(); } @Override diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index 8c43943c..b250b3da 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -11,9 +11,7 @@ public GetRange(PegasusTableInterface table, byte[] hashKey, int timeout) { super(table, hashKey, timeout); } - @Override public Result commitAndWait(int maxFetchCount) throws PException { - this.scanOptions.noValue = true; ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); return scannerWrapper.hashScan(maxFetchCount); } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java similarity index 72% rename from src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java rename to src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java index 78d78c85..0362a755 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeWithValue.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java @@ -5,13 +5,15 @@ import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; -public class GetRangeWithValue extends Range { +public class GetRangeNoValue extends Range { - public GetRangeWithValue(PegasusTableInterface table, byte[] hashKey, int timeout) { + public GetRangeNoValue(PegasusTableInterface table, byte[] hashKey, int timeout) { super(table, hashKey, timeout); } + @Override public Result commitAndWait(int maxFetchCount) throws PException { + this.scanOptions.noValue = true; ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); return scannerWrapper.hashScan(maxFetchCount); } diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java index f5d49d28..4c14823c 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -1,6 +1,6 @@ package com.xiaomi.infra.pegasus.client; -import com.xiaomi.infra.pegasus.client.request.range.GetRangeWithValue; +import com.xiaomi.infra.pegasus.client.request.range.GetRange; import com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -19,55 +19,55 @@ public void testScanRangeWithValueExpired() PegasusTable table = (PegasusTable) PegasusClientFactory.getSingletonClient().openTable(tableName); - GetRangeWithValue getRangeWithValue = new GetRangeWithValue(table, hashKey.getBytes(), 0); + GetRange getRange = new GetRange(table, hashKey.getBytes(), 0); // case A: scan all record // case A1: scan all record: if persistent record count >= maxFetchCount, it must return // maxFetchCount records - ScannerWrapper.Result caseA1 = getRangeWithValue.commitAndWait(5); + ScannerWrapper.Result caseA1 = getRange.commitAndWait(5); assertScanResult(0, 4, false, caseA1); // case A2: scan all record: if persistent record count < maxFetchCount, it only return // persistent count records - ScannerWrapper.Result caseA2 = getRangeWithValue.commitAndWait(100); + ScannerWrapper.Result caseA2 = getRange.commitAndWait(100); assertScanResult(0, 9, true, caseA2); // case B: scan limit record by "startSortKey" and "": // case B1: scan limit record by "expired_0" and "", if persistent record count >= // maxFetchCount, it must return maxFetchCount records ScannerWrapper.Result caseB1 = - getRangeWithValue.withStartSortKey("expired_0".getBytes()).commitAndWait(5); + getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(5); assertScanResult(0, 4, false, caseB1); // case B2: scan limit record by "expired_0" and "", if persistent record count < maxFetchCount, // it only return valid records ScannerWrapper.Result caseB2 = - getRangeWithValue.withStartSortKey("expired_0".getBytes()).commitAndWait(50); + getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(50); assertScanResult(0, 9, true, caseB2); // case B3: scan limit record by "persistent_5" and "", if following persistent record count < // maxFetchCount, it only return valid records ScannerWrapper.Result caseB3 = - getRangeWithValue.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); + getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); assertScanResult(5, 9, true, caseB3); // case B4: scan limit record by "persistent_5" and "", if following persistent record count > // maxFetchCount, it only return valid records ScannerWrapper.Result caseB4 = - getRangeWithValue.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); + getRange.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); assertScanResult(5, 7, false, caseB4); // case C: scan limit record by "" and "stopSortKey": // case C1: scan limit record by "" and "expired_7", if will return 0 record ScannerWrapper.Result caseC1 = - getRangeWithValue.withStopSortKey("expired_7".getBytes()).commitAndWait(3); + getRange.withStopSortKey("expired_7".getBytes()).commitAndWait(3); Assertions.assertTrue(caseC1.allFetched); Assertions.assertEquals( 0, caseC1.results.size()); // among "" and "expired_7" has 0 valid record // case C2: scan limit record by "" and "persistent_7", if valid record count < maxFetchCount, // it only return valid record ScannerWrapper.Result caseC2 = - getRangeWithValue.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); + getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); assertScanResult(0, 6, true, caseC2); // case C3: scan limit record by "" and "persistent_7", if valid record count > maxFetchCount, // it only return valid record ScannerWrapper.Result caseC3 = - getRangeWithValue.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); + getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); assertScanResult(0, 1, false, caseC3); } From 3fcb09709ac56dc78a19e69bc359f41505f19b84 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Wed, 17 Mar 2021 10:46:30 +0800 Subject: [PATCH 08/17] refactor --- .../client/request/range/GetRange.java | 5 ++- .../client/request/range/GetRangeNoValue.java | 20 ------------ .../client/request/range/ScannerWrapper.java | 31 ++----------------- 3 files changed, 7 insertions(+), 49 deletions(-) delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index b250b3da..b0c7e95f 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -3,6 +3,7 @@ import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; import com.xiaomi.infra.pegasus.client.PException; +import com.xiaomi.infra.pegasus.client.PegasusScannerInterface; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; public class GetRange extends Range { @@ -12,7 +13,9 @@ public GetRange(PegasusTableInterface table, byte[] hashKey, int timeout) { } public Result commitAndWait(int maxFetchCount) throws PException { - ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); + PegasusScannerInterface scanner = + table.getScanner(hashKey, startSortKey, stopSortKey, scanOptions); + ScannerWrapper scannerWrapper = new ScannerWrapper(scanner); return scannerWrapper.hashScan(maxFetchCount); } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java deleted file mode 100644 index 0362a755..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRangeNoValue.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.xiaomi.infra.pegasus.client.request.range; - -import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; - -import com.xiaomi.infra.pegasus.client.PException; -import com.xiaomi.infra.pegasus.client.PegasusTableInterface; - -public class GetRangeNoValue extends Range { - - public GetRangeNoValue(PegasusTableInterface table, byte[] hashKey, int timeout) { - super(table, hashKey, timeout); - } - - @Override - public Result commitAndWait(int maxFetchCount) throws PException { - this.scanOptions.noValue = true; - ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); - return scannerWrapper.hashScan(maxFetchCount); - } -} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java index b66221fe..8d8eb917 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java @@ -6,49 +6,24 @@ import com.xiaomi.infra.pegasus.client.*; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.tuple.Pair; -public class ScannerWrapper { - private final Range request; - private final PegasusTableInterface table; +public class ScannerWrapper { private final PegasusScannerInterface scanner; - public ScannerWrapper(Range request) throws PException { - this.request = request; - this.table = request.table; - this.scanner = - table.getScanner( - request.hashKey, request.startSortKey, request.stopSortKey, request.scanOptions); + public ScannerWrapper(PegasusScannerInterface scanner) throws PException { + this.scanner = scanner; } // TODO(jiashuo1) scanner.next()'s bug need be fixed, it will be put next pr public Result hashScan(int maxFetchCount) throws PException { - long deadlineTime = System.currentTimeMillis() + request.timeout; - Result scanRangeResult = new Result(); scanRangeResult.allFetched = false; scanRangeResult.results = new ArrayList<>(); - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - ((PegasusTable) table).getMetaList(), - ((PegasusTable) table).getTable().getTableName(), - new PegasusTable.Request(request.hashKey), - request.timeout, - new TimeoutException()); - } Pair, byte[]> pair; while ((pair = scanner.next()) != null && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { - if (System.currentTimeMillis() >= deadlineTime) { - throw PException.timeout( - ((PegasusTable) table).getMetaList(), - ((PegasusTable) table).getTable().getTableName(), - new PegasusTable.Request(request.hashKey), - request.timeout, - new TimeoutException()); - } scanRangeResult.results.add(pair); } From 4bda7bb4bb4814f4ca3ef03d044af0d7e4e744b2 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Wed, 17 Mar 2021 16:08:56 +0800 Subject: [PATCH 09/17] add timeou log --- LICENSE | 2 +- pom.xml | 2 +- .../infra/pegasus/client/PegasusTable.java | 27 +++----- .../client/request/range/DeleteRange.java | 62 ++++++++++------- .../client/request/range/GetRange.java | 43 +++++++++--- .../pegasus/client/request/range/Range.java | 35 ++++++++++ .../client/request/range/ScanResult.java | 54 +++++++++++++++ .../client/request/range/ScannerWrapper.java | 66 ------------------- .../infra/pegasus/client/TestRange.java | 53 +++++++++------ 9 files changed, 204 insertions(+), 140 deletions(-) create mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java delete mode 100644 src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java diff --git a/LICENSE b/LICENSE index 6efe7f33..8bfcc1db 100644 --- a/LICENSE +++ b/LICENSE @@ -157,7 +157,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the + scanResult of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor diff --git a/pom.xml b/pom.xml index 6854c7e1..240abde7 100644 --- a/pom.xml +++ b/pom.xml @@ -382,7 +382,7 @@ Use Rat to improve accuracy and efficiency when checking releases for licenses. Usage: mvn apache-rat:check - # the result will be generated to ./target/rat.txt + # the scanResult will be generated to ./target/rat.txt --> org.apache.rat diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java index 8ede1daf..04eccda6 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java @@ -7,7 +7,7 @@ import com.xiaomi.infra.pegasus.base.blob; import com.xiaomi.infra.pegasus.base.error_code; import com.xiaomi.infra.pegasus.base.gpid; -import com.xiaomi.infra.pegasus.client.request.range.GetRangeNoValue; +import com.xiaomi.infra.pegasus.client.request.range.GetRange; import com.xiaomi.infra.pegasus.operator.*; import com.xiaomi.infra.pegasus.rpc.ReplicationException; import com.xiaomi.infra.pegasus.rpc.Table; @@ -1186,8 +1186,13 @@ public int batchMultiGet2( public MultiGetSortKeysResult multiGetSortKeys( byte[] hashKey, int maxFetchCount, int maxFetchSize, int timeout) throws PException { if (timeout <= 0) timeout = defaultTimeout; - GetRangeNoValue getRangeNoValue = new GetRangeNoValue(this, hashKey, timeout); - return getRangeNoValue.commitAndWait(maxFetchCount).convertMultiGetSortKeysResult(); + ScanOptions scanOptions = new ScanOptions(); + scanOptions.noValue = true; + GetRange getRange = new GetRange(this, hashKey, timeout); + return getRange + .withOptions(scanOptions) + .commitAndWait(maxFetchCount) + .convertMultiGetSortKeysResult(); } @Override @@ -1823,22 +1828,6 @@ public List getUnorderedScanners( return ret; } - /** - * get scan result for {startSortKey, stopSortKey} within hashKey - * - * @deprecated it will be replace - * @param hashKey used to decide which partition to put this k-v, - * @param startSortKey start sort key scan from if null or length == 0, means start from begin - * @param stopSortKey stop sort key scan to if null or length == 0, means stop to end - * @param options scan options like endpoint inclusive/exclusive - * @param maxFetchCount max count of k-v pairs to be fetched. if <=0 means fetch all data for - * {startSortKey, stopSortKey} - * @param timeout if exceed the timeout will throw timeout exception, if <=0, it is equal with - * "timeout" of config - * @return ScanRangeResult result{pair((hashKey, sortKey), value}, if fetch all data for - * {startSortKey, stopSortKey}, ScanRangeResult.allFetched=true - * @throws PException - */ public void handleReplicaException( Request request, DefaultPromise promise, client_operator op, Table table, int timeout) { if (timeout <= 0) timeout = defaultTimeout; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java index a2bdf13e..938acdaf 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -1,13 +1,13 @@ package com.xiaomi.infra.pegasus.client.request.range; -import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; - -import com.xiaomi.infra.pegasus.client.PException; -import com.xiaomi.infra.pegasus.client.PegasusTable; -import com.xiaomi.infra.pegasus.client.PegasusTableInterface; +import com.xiaomi.infra.pegasus.client.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.commons.lang3.tuple.Pair; public class DeleteRange extends Range { @@ -16,29 +16,41 @@ public DeleteRange(PegasusTableInterface table, byte[] hashKey, int timeout) { } public Boolean commitAndWait(int maxDeleteCount) throws PException { - long deadlineTime = System.currentTimeMillis() + timeout; - - this.scanOptions.noValue = true; - ScannerWrapper scannerWrapper = new ScannerWrapper<>(this); - Result result = scannerWrapper.hashScan(maxDeleteCount); - int remainingTime = (int) (deadlineTime - System.currentTimeMillis()); - if (remainingTime <= 0) { - throw PException.timeout( - ((PegasusTable) table).getMetaList(), - ((PegasusTable) table).getTable().getTableName(), - new PegasusTable.Request(hashKey), - timeout, - new TimeoutException()); - } - + ScanResult scanResult = new ScanResult(); try { - table - .asyncMultiDel(hashKey, result.convertMultiGetSortKeysResult().keys, remainingTime) - .get(remainingTime, TimeUnit.MILLISECONDS); + scanResult = + CompletableFuture.supplyAsync( + () -> { + try { + ScanOptions scanOptions = new ScanOptions(); + scanOptions.noValue = true; + PegasusScannerInterface scanner = + table.getScanner(hashKey, startSortKey, stopSortKey, scanOptions); + List sortKeys = new ArrayList<>(); + Pair, byte[]> pairs; + while ((pairs = scanner.next()) != null) { + sortKeys.add(pairs.getKey().getValue()); + if (sortKeys.size() == scanOptions.batchSize) { + table.multiDel(hashKey, sortKeys, 0); + sortKeys.clear(); + } + } + if (!sortKeys.isEmpty()) { + table.multiDel(hashKey, sortKeys, 0); + } + } catch (PException e) { + return new ScanResult(e); + } + return new ScanResult(); + }) + .get(scanOptions.timeoutMillis, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { - throw new PException("Delete range failed!", e); + throw new PException(e); } - return result.allFetched; + if (scanResult.exception != null) { + throw new PException(scanResult.exception); + } + return true; } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index b0c7e95f..e00ec525 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -1,21 +1,48 @@ package com.xiaomi.infra.pegasus.client.request.range; -import static com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper.Result; - import com.xiaomi.infra.pegasus.client.PException; import com.xiaomi.infra.pegasus.client.PegasusScannerInterface; import com.xiaomi.infra.pegasus.client.PegasusTableInterface; +import java.util.concurrent.*; +import org.apache.commons.lang3.tuple.Pair; -public class GetRange extends Range { +public class GetRange extends Range { public GetRange(PegasusTableInterface table, byte[] hashKey, int timeout) { super(table, hashKey, timeout); } - public Result commitAndWait(int maxFetchCount) throws PException { - PegasusScannerInterface scanner = - table.getScanner(hashKey, startSortKey, stopSortKey, scanOptions); - ScannerWrapper scannerWrapper = new ScannerWrapper(scanner); - return scannerWrapper.hashScan(maxFetchCount); + public ScanResult commitAndWait(int maxFetchCount) throws PException { + ScanResult scanResult; + try { + scanResult = + CompletableFuture.supplyAsync( + () -> { + ScanResult res = new ScanResult(); + try { + PegasusScannerInterface scanner = + table.getScanner(hashKey, startSortKey, stopSortKey, scanOptions); + Pair, byte[]> pair; + while ((pair = scanner.next()) != null + && (maxFetchCount <= 0 || res.results.size() < maxFetchCount)) { + res.results.add(pair); + } + if (scanner.next() == null) { + res.allFetched = true; + } + return res; + } catch (PException e) { + return new ScanResult(e); + } + }) + .get(scanOptions.timeoutMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new PException(e); + } + + if (scanResult != null && scanResult.exception != null) { + throw scanResult.exception; + } + return scanResult; } } diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java index 4c87f70a..3101c720 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java @@ -5,6 +5,12 @@ import com.xiaomi.infra.pegasus.client.PegasusTableInterface; import com.xiaomi.infra.pegasus.client.ScanOptions; +/** + * range operation abstract class based {@linkplain PegasusTableInterface#getScanner(byte[], byte[], + * byte[], ScanOptions)} it is implemented by {@linkplain DeleteRange} and {@linkplain GetRange} + * + * @param generic type for response + */ public abstract class Range { public PegasusTableInterface table; protected byte[] hashKey; @@ -14,24 +20,53 @@ public abstract class Range { protected byte[] startSortKey; protected byte[] stopSortKey; + /** + * @param table table opened + * @param hashKey hashKey used to decide which partition to put this k-v + * @param timeout if exceed the timeout will throw timeout exception, if <=0, it is equal with + * "timeout" of config + */ public Range(PegasusTableInterface table, byte[] hashKey, int timeout) { this.table = table; this.hashKey = hashKey; this.timeout = timeout <= 0 ? ((PegasusTable) table).getDefaultTimeout() : timeout; } + /** + * @param maxRangeCount the max count of range operation result + * @return generic type for response + * @throws PException + */ public abstract Response commitAndWait(int maxRangeCount) throws PException; + /** + * set scan options + * + * @param scanOptions see {@linkplain ScanOptions} + * @return this + */ public Range withOptions(ScanOptions scanOptions) { this.scanOptions = scanOptions; return this; } + /** + * set startSortKey + * + * @param startSortKey start sort key scan from if null or length == 0, means start from begin + * @return this + */ public Range withStartSortKey(byte[] startSortKey) { this.startSortKey = startSortKey; return this; } + /** + * set stopSortKey + * + * @param stopSortKey stop sort key scan to if null or length == 0, means stop to end + * @return this + */ public Range withStopSortKey(byte[] stopSortKey) { this.stopSortKey = stopSortKey; return this; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java new file mode 100644 index 00000000..514e3fbb --- /dev/null +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java @@ -0,0 +1,54 @@ +package com.xiaomi.infra.pegasus.client.request.range; + +import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; +import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetSortKeysResult; + +import com.xiaomi.infra.pegasus.client.*; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; + +public class ScanResult { + public List, byte[]>> results; + public boolean allFetched; + public PException exception; + + public ScanResult() { + this(new ArrayList<>(), false); + } + + public ScanResult(List, byte[]>> results, boolean allFetched) { + this.results = results; + this.allFetched = allFetched; + } + + public ScanResult(PException exception) { + this.exception = exception; + } + + public MultiGetResult convertMultiGetResult() { + MultiGetResult multiGetResult = new MultiGetResult(); + if (results == null) { + return multiGetResult; + } + multiGetResult.values = new ArrayList<>(); + for (Pair, byte[]> pair : results) { + multiGetResult.values.add(Pair.of(pair.getLeft().getValue(), pair.getValue())); + } + multiGetResult.allFetched = allFetched; + return multiGetResult; + } + + public MultiGetSortKeysResult convertMultiGetSortKeysResult() { + MultiGetSortKeysResult multiGetSortKeysResult = new MultiGetSortKeysResult(); + if (results == null) { + return multiGetSortKeysResult; + } + multiGetSortKeysResult.keys = new ArrayList<>(); + for (Pair, byte[]> pair : results) { + multiGetSortKeysResult.keys.add(pair.getLeft().getValue()); + } + multiGetSortKeysResult.allFetched = allFetched; + return multiGetSortKeysResult; + } +} diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java deleted file mode 100644 index 8d8eb917..00000000 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScannerWrapper.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.xiaomi.infra.pegasus.client.request.range; - -import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; -import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetSortKeysResult; - -import com.xiaomi.infra.pegasus.client.*; -import java.util.ArrayList; -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; - -public class ScannerWrapper { - private final PegasusScannerInterface scanner; - - public ScannerWrapper(PegasusScannerInterface scanner) throws PException { - this.scanner = scanner; - } - - // TODO(jiashuo1) scanner.next()'s bug need be fixed, it will be put next pr - public Result hashScan(int maxFetchCount) throws PException { - Result scanRangeResult = new Result(); - scanRangeResult.allFetched = false; - scanRangeResult.results = new ArrayList<>(); - - Pair, byte[]> pair; - while ((pair = scanner.next()) != null - && (maxFetchCount <= 0 || scanRangeResult.results.size() < maxFetchCount)) { - scanRangeResult.results.add(pair); - } - - if (scanner.next() == null) { - scanRangeResult.allFetched = true; - } - return scanRangeResult; - } - - public static class Result { - public List, byte[]>> results; - public boolean allFetched; - - public MultiGetResult convertMultiGetResult() { - MultiGetResult multiGetResult = new MultiGetResult(); - if (results == null) { - return multiGetResult; - } - multiGetResult.values = new ArrayList<>(); - for (Pair, byte[]> pair : results) { - multiGetResult.values.add(Pair.of(pair.getLeft().getValue(), pair.getValue())); - } - multiGetResult.allFetched = allFetched; - return multiGetResult; - } - - public MultiGetSortKeysResult convertMultiGetSortKeysResult() { - MultiGetSortKeysResult multiGetSortKeysResult = new MultiGetSortKeysResult(); - if (results == null) { - return multiGetSortKeysResult; - } - multiGetSortKeysResult.keys = new ArrayList<>(); - for (Pair, byte[]> pair : results) { - multiGetSortKeysResult.keys.add(pair.getLeft().getValue()); - } - multiGetSortKeysResult.allFetched = allFetched; - return multiGetSortKeysResult; - } - } -} diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java index 4c14823c..43b5474a 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -1,7 +1,8 @@ package com.xiaomi.infra.pegasus.client; +import com.xiaomi.infra.pegasus.client.request.range.DeleteRange; import com.xiaomi.infra.pegasus.client.request.range.GetRange; -import com.xiaomi.infra.pegasus.client.request.range.ScannerWrapper; +import com.xiaomi.infra.pegasus.client.request.range.ScanResult; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Assertions; @@ -17,60 +18,72 @@ public void testScanRangeWithValueExpired() // generate records: sortKeys=[expired_0....expired_999,persistent_0...persistent_9] generateRecordsWithExpired(tableName, hashKey, 1000, 10); - PegasusTable table = - (PegasusTable) PegasusClientFactory.getSingletonClient().openTable(tableName); + PegasusTableInterface table = PegasusClientFactory.getSingletonClient().openTable(tableName); GetRange getRange = new GetRange(table, hashKey.getBytes(), 0); // case A: scan all record // case A1: scan all record: if persistent record count >= maxFetchCount, it must return // maxFetchCount records - ScannerWrapper.Result caseA1 = getRange.commitAndWait(5); + ScanResult caseA1 = getRange.commitAndWait(5); assertScanResult(0, 4, false, caseA1); // case A2: scan all record: if persistent record count < maxFetchCount, it only return // persistent count records - ScannerWrapper.Result caseA2 = getRange.commitAndWait(100); + ScanResult caseA2 = getRange.commitAndWait(100); assertScanResult(0, 9, true, caseA2); // case B: scan limit record by "startSortKey" and "": // case B1: scan limit record by "expired_0" and "", if persistent record count >= // maxFetchCount, it must return maxFetchCount records - ScannerWrapper.Result caseB1 = - getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(5); + ScanResult caseB1 = getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(5); assertScanResult(0, 4, false, caseB1); // case B2: scan limit record by "expired_0" and "", if persistent record count < maxFetchCount, // it only return valid records - ScannerWrapper.Result caseB2 = - getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(50); + ScanResult caseB2 = getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(50); assertScanResult(0, 9, true, caseB2); // case B3: scan limit record by "persistent_5" and "", if following persistent record count < // maxFetchCount, it only return valid records - ScannerWrapper.Result caseB3 = - getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); + ScanResult caseB3 = getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); assertScanResult(5, 9, true, caseB3); // case B4: scan limit record by "persistent_5" and "", if following persistent record count > // maxFetchCount, it only return valid records - ScannerWrapper.Result caseB4 = - getRange.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); + ScanResult caseB4 = getRange.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); assertScanResult(5, 7, false, caseB4); // case C: scan limit record by "" and "stopSortKey": // case C1: scan limit record by "" and "expired_7", if will return 0 record - ScannerWrapper.Result caseC1 = - getRange.withStopSortKey("expired_7".getBytes()).commitAndWait(3); + ScanResult caseC1 = getRange.withStopSortKey("expired_7".getBytes()).commitAndWait(3); Assertions.assertTrue(caseC1.allFetched); Assertions.assertEquals( 0, caseC1.results.size()); // among "" and "expired_7" has 0 valid record // case C2: scan limit record by "" and "persistent_7", if valid record count < maxFetchCount, // it only return valid record - ScannerWrapper.Result caseC2 = - getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); + ScanResult caseC2 = getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); assertScanResult(0, 6, true, caseC2); // case C3: scan limit record by "" and "persistent_7", if valid record count > maxFetchCount, // it only return valid record - ScannerWrapper.Result caseC3 = - getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); + ScanResult caseC3 = getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); assertScanResult(0, 1, false, caseC3); } + @Test // test for making sure return "maxFetchCount" if has "maxFetchCount" valid record + public void testDeleteRangeWithValueExpired() + throws PException, InterruptedException, TimeoutException, ExecutionException { + String tableName = "temp"; + String hashKey = "hashKey"; + // generate records: sortKeys=[expired_0....expired_999,persistent_0...persistent_9] + generateRecordsWithExpired(tableName, hashKey, 1000, 10); + PegasusTableInterface table = PegasusClientFactory.getSingletonClient().openTable(tableName); + DeleteRange deleteRange = new DeleteRange(table, hashKey.getBytes(), 0); + Boolean res1 = deleteRange.commitAndWait(5); + Assertions.assertTrue(res1); + + Boolean res2 = deleteRange.commitAndWait(10); + Assertions.assertTrue(res2); + GetRange getRange = new GetRange(table, hashKey.getBytes(), 0); + ScanResult getRes = getRange.commitAndWait(-1); + Assertions.assertTrue(getRes.allFetched); + Assertions.assertEquals(0, getRes.results.size()); + } + private void generateRecordsWithExpired( String tableName, String hashKey, int expiredCount, int persistentCount) throws PException, InterruptedException { @@ -99,7 +112,7 @@ private void generateRecordsWithExpired( } private void assertScanResult( - int startIndex, int stopIndex, boolean expectAllFetched, ScannerWrapper.Result actuallyRes) { + int startIndex, int stopIndex, boolean expectAllFetched, ScanResult actuallyRes) { Assertions.assertEquals(expectAllFetched, actuallyRes.allFetched); Assertions.assertEquals(stopIndex - startIndex + 1, actuallyRes.results.size()); for (int i = startIndex; i <= stopIndex; i++) { From d7a5e6f4211f4faea9333ae742e6bd05ee527b43 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Wed, 17 Mar 2021 17:33:56 +0800 Subject: [PATCH 10/17] add timeou log --- .../xiaomi/infra/pegasus/client/request/range/GetRange.java | 2 +- src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index e00ec525..65857d7f 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -27,7 +27,7 @@ public ScanResult commitAndWait(int maxFetchCount) throws PException { && (maxFetchCount <= 0 || res.results.size() < maxFetchCount)) { res.results.add(pair); } - if (scanner.next() == null) { + if (pair == null) { res.allFetched = true; } return res; diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java index 43b5474a..3c38fb16 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -45,11 +45,12 @@ public void testScanRangeWithValueExpired() assertScanResult(5, 9, true, caseB3); // case B4: scan limit record by "persistent_5" and "", if following persistent record count > // maxFetchCount, it only return valid records - ScanResult caseB4 = getRange.withStopSortKey("persistent_5".getBytes()).commitAndWait(3); + ScanResult caseB4 = getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(3); assertScanResult(5, 7, false, caseB4); // case C: scan limit record by "" and "stopSortKey": // case C1: scan limit record by "" and "expired_7", if will return 0 record + getRange.withStartSortKey("".getBytes()); ScanResult caseC1 = getRange.withStopSortKey("expired_7".getBytes()).commitAndWait(3); Assertions.assertTrue(caseC1.allFetched); Assertions.assertEquals( @@ -66,7 +67,7 @@ public void testScanRangeWithValueExpired() @Test // test for making sure return "maxFetchCount" if has "maxFetchCount" valid record public void testDeleteRangeWithValueExpired() - throws PException, InterruptedException, TimeoutException, ExecutionException { + throws PException, InterruptedException, TimeoutException, ExecutionException { String tableName = "temp"; String hashKey = "hashKey"; // generate records: sortKeys=[expired_0....expired_999,persistent_0...persistent_9] From 788523b40df78a3a4f733e2e981c9864786da467 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 11:20:18 +0800 Subject: [PATCH 11/17] fix comment --- pom.xml | 2 +- .../xiaomi/infra/pegasus/client/PegasusClientInterface.java | 3 ++- .../com/xiaomi/infra/pegasus/client/PegasusTableInterface.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 240abde7..8a84bc57 100644 --- a/pom.xml +++ b/pom.xml @@ -363,7 +363,7 @@ src/test/resource/pegasus.properties src/test/java/com/xiaomi/infra/pegasus/client/* src/test/java/com/xiaomi/infra/pegasus/operator/* - src/main/java/com/xiaomi/infra/pegasus/client/** + src/main/java/com/xiaomi/infra/pegasus/client/*** src/main/java/com/xiaomi/infra/pegasus/tools/* src/main/java/com/xiaomi/infra/pegasus/operator/* src/main/java/com/xiaomi/infra/pegasus/base/* diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java index dae28e9c..7bd602b6 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java @@ -182,7 +182,8 @@ public boolean multiGet( /** * Get multiple key-values under the same hashKey with sortKey range limited. * - * @deprecated The API may can't get all records, please use //TODO + * @deprecated The API may can't get all records, please use {@linkplain + * com.xiaomi.infra.pegasus.client.request.range.GetRange} * @param tableName table name * @param hashKey used to decide which partition the key may exist should not be null or empty. * @param startSortKey the start sort key. null means "". diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java index 30632685..58276e3e 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTableInterface.java @@ -221,7 +221,8 @@ public Future asyncMultiGet( /** * get multiple key-values under the same hashKey with sortKey range limited, async version * - * @deprecated the API may can't get all records, please use //TODO + * @deprecated the API may can't get all records, please use {@linkplain + * com.xiaomi.infra.pegasus.client.request.range.GetRange} * @param hashKey used to decide which partition the key may exist should not be null or empty. * @param startSortKey the start sort key. null means "". * @param stopSortKey the stop sort key. null or "" means fetch to the last sort key. From db79239fd16806029ec52b93c6017a32c8eb4a8c Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 11:27:36 +0800 Subject: [PATCH 12/17] fix comment --- .../client/request/range/DeleteRange.java | 16 ++++++++++++++++ .../pegasus/client/request/range/GetRange.java | 16 ++++++++++++++++ .../pegasus/client/request/range/Range.java | 16 ++++++++++++++++ .../pegasus/client/request/range/ScanResult.java | 16 ++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java index 938acdaf..a961e81a 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/DeleteRange.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.xiaomi.infra.pegasus.client.request.range; import com.xiaomi.infra.pegasus.client.*; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java index 65857d7f..4aacece5 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/GetRange.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.xiaomi.infra.pegasus.client.request.range; import com.xiaomi.infra.pegasus.client.PException; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java index 3101c720..8e484a3f 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.xiaomi.infra.pegasus.client.request.range; import com.xiaomi.infra.pegasus.client.PException; diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java index 514e3fbb..1511f66b 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/ScanResult.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.xiaomi.infra.pegasus.client.request.range; import static com.xiaomi.infra.pegasus.client.PegasusTableInterface.MultiGetResult; From 6600b598b758b4541d70e34667660fe0e4efddea Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 11:45:17 +0800 Subject: [PATCH 13/17] fix comment --- LICENSE | 2 +- dev-support/findbugs-exclude.xml | 3 ++- pom.xml | 2 +- .../xiaomi/infra/pegasus/client/PException.java | 4 ++-- .../infra/pegasus/client/PegasusTable.java | 16 ++++------------ 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/LICENSE b/LICENSE index 8bfcc1db..6efe7f33 100644 --- a/LICENSE +++ b/LICENSE @@ -157,7 +157,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a - scanResult of this License or out of the use or inability to use the + result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor diff --git a/dev-support/findbugs-exclude.xml b/dev-support/findbugs-exclude.xml index 1f18a83d..01eef014 100644 --- a/dev-support/findbugs-exclude.xml +++ b/dev-support/findbugs-exclude.xml @@ -15,7 +15,8 @@ - + + diff --git a/pom.xml b/pom.xml index 8a84bc57..cf6e06e6 100644 --- a/pom.xml +++ b/pom.xml @@ -382,7 +382,7 @@ Use Rat to improve accuracy and efficiency when checking releases for licenses. Usage: mvn apache-rat:check - # the scanResult will be generated to ./target/rat.txt + # the result will be generated to ./target/rat.txt --> org.apache.rat diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PException.java b/src/main/java/com/xiaomi/infra/pegasus/client/PException.java index c95bb096..4483bcf3 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PException.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PException.java @@ -34,14 +34,14 @@ public PException(Throwable cause) { super(versionPrefix + cause.toString(), cause); } - public static PException threadInterrupted(String tableName, InterruptedException e) { + static PException threadInterrupted(String tableName, InterruptedException e) { return new PException( new ReplicationException( error_code.error_types.ERR_THREAD_INTERRUPTED, String.format("[table=%s] Thread was interrupted: %s", tableName, e.getMessage()))); } - public static PException timeout( + static PException timeout( String metaList, String tableName, Request request, int timeout, TimeoutException e) { return new PException( new ReplicationException( diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java index 04eccda6..ce830a0b 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusTable.java @@ -44,18 +44,10 @@ public PegasusTable(PegasusClient client, Table table) { this.metaList = client.getMetaList(); } - public Table getTable() { - return table; - } - public int getDefaultTimeout() { return defaultTimeout; } - public String getMetaList() { - return metaList; - } - @Override public Future asyncExist(byte[] hashKey, byte[] sortKey, int timeout) { final DefaultPromise promise = table.newPromise(); @@ -1883,21 +1875,21 @@ private void handleWriteLimiterException(DefaultPromise promise, String message) promise.setFailure(new PException("Exceed write limit threshold:" + message)); } - public static class Request { + static class Request { byte[] hashKey = null; byte[] sortKey = null; int sortKeyCount = 0; - public Request(byte[] hashKey) { + Request(byte[] hashKey) { this.hashKey = hashKey; } - public Request(byte[] hashKey, byte[] sortKey) { + Request(byte[] hashKey, byte[] sortKey) { this.hashKey = hashKey; this.sortKey = sortKey; } - public Request(byte[] hashKey, int sortKeyCount) { + Request(byte[] hashKey, int sortKeyCount) { this.hashKey = hashKey; this.sortKeyCount = sortKeyCount; } From c4c62a83d430643e9a15e085f134ee3fcdc7bbf3 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 11:49:27 +0800 Subject: [PATCH 14/17] fix comment --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cf6e06e6..6854c7e1 100644 --- a/pom.xml +++ b/pom.xml @@ -363,7 +363,7 @@ src/test/resource/pegasus.properties src/test/java/com/xiaomi/infra/pegasus/client/* src/test/java/com/xiaomi/infra/pegasus/operator/* - src/main/java/com/xiaomi/infra/pegasus/client/*** + src/main/java/com/xiaomi/infra/pegasus/client/** src/main/java/com/xiaomi/infra/pegasus/tools/* src/main/java/com/xiaomi/infra/pegasus/operator/* src/main/java/com/xiaomi/infra/pegasus/base/* From 7b7d58168413e632b01c90070480f8a0a6f8748c Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 11:52:32 +0800 Subject: [PATCH 15/17] fix comment --- .../xiaomi/infra/pegasus/client/TestRange.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java index 3c38fb16..566fc32e 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.xiaomi.infra.pegasus.client; import com.xiaomi.infra.pegasus.client.request.range.DeleteRange; From 3a718ac833321f53f2999d419878e447ccbb22b4 Mon Sep 17 00:00:00 2001 From: jiashuo Date: Thu, 18 Mar 2021 12:35:30 +0800 Subject: [PATCH 16/17] fix comment --- .../xiaomi/infra/pegasus/client/PegasusClientInterface.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java index 7bd602b6..55f3a3eb 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/PegasusClientInterface.java @@ -464,6 +464,8 @@ public int batchDel2(String tableName, List> keys, List> keys, List Date: Mon, 29 Mar 2021 14:59:49 +0800 Subject: [PATCH 17/17] update sortKeys --- .../pegasus/client/request/range/Range.java | 13 ++----------- .../xiaomi/infra/pegasus/client/TestRange.java | 17 +++++++++-------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java index 8e484a3f..cf61283a 100644 --- a/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java +++ b/src/main/java/com/xiaomi/infra/pegasus/client/request/range/Range.java @@ -70,20 +70,11 @@ public Range withOptions(ScanOptions scanOptions) { * set startSortKey * * @param startSortKey start sort key scan from if null or length == 0, means start from begin - * @return this - */ - public Range withStartSortKey(byte[] startSortKey) { - this.startSortKey = startSortKey; - return this; - } - - /** - * set stopSortKey - * * @param stopSortKey stop sort key scan to if null or length == 0, means stop to end * @return this */ - public Range withStopSortKey(byte[] stopSortKey) { + public Range withSortKeyRange(byte[] startSortKey, byte[] stopSortKey) { + this.startSortKey = startSortKey; this.stopSortKey = stopSortKey; return this; } diff --git a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java index 566fc32e..6a3781ab 100644 --- a/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java +++ b/src/test/java/com/xiaomi/infra/pegasus/client/TestRange.java @@ -49,35 +49,36 @@ public void testScanRangeWithValueExpired() // case B: scan limit record by "startSortKey" and "": // case B1: scan limit record by "expired_0" and "", if persistent record count >= // maxFetchCount, it must return maxFetchCount records - ScanResult caseB1 = getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(5); + ScanResult caseB1 = getRange.withSortKeyRange("expired_0".getBytes(), null).commitAndWait(5); assertScanResult(0, 4, false, caseB1); // case B2: scan limit record by "expired_0" and "", if persistent record count < maxFetchCount, // it only return valid records - ScanResult caseB2 = getRange.withStartSortKey("expired_0".getBytes()).commitAndWait(50); + ScanResult caseB2 = getRange.withSortKeyRange("expired_0".getBytes(), null).commitAndWait(50); assertScanResult(0, 9, true, caseB2); // case B3: scan limit record by "persistent_5" and "", if following persistent record count < // maxFetchCount, it only return valid records - ScanResult caseB3 = getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(50); + ScanResult caseB3 = + getRange.withSortKeyRange("persistent_5".getBytes(), null).commitAndWait(50); assertScanResult(5, 9, true, caseB3); // case B4: scan limit record by "persistent_5" and "", if following persistent record count > // maxFetchCount, it only return valid records - ScanResult caseB4 = getRange.withStartSortKey("persistent_5".getBytes()).commitAndWait(3); + ScanResult caseB4 = getRange.withSortKeyRange("persistent_5".getBytes(), null).commitAndWait(3); assertScanResult(5, 7, false, caseB4); // case C: scan limit record by "" and "stopSortKey": // case C1: scan limit record by "" and "expired_7", if will return 0 record - getRange.withStartSortKey("".getBytes()); - ScanResult caseC1 = getRange.withStopSortKey("expired_7".getBytes()).commitAndWait(3); + ScanResult caseC1 = getRange.withSortKeyRange(null, "expired_7".getBytes()).commitAndWait(3); Assertions.assertTrue(caseC1.allFetched); Assertions.assertEquals( 0, caseC1.results.size()); // among "" and "expired_7" has 0 valid record // case C2: scan limit record by "" and "persistent_7", if valid record count < maxFetchCount, // it only return valid record - ScanResult caseC2 = getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(10); + ScanResult caseC2 = + getRange.withSortKeyRange(null, "persistent_7".getBytes()).commitAndWait(10); assertScanResult(0, 6, true, caseC2); // case C3: scan limit record by "" and "persistent_7", if valid record count > maxFetchCount, // it only return valid record - ScanResult caseC3 = getRange.withStopSortKey("persistent_7".getBytes()).commitAndWait(2); + ScanResult caseC3 = getRange.withSortKeyRange(null, "persistent_7".getBytes()).commitAndWait(2); assertScanResult(0, 1, false, caseC3); }