Skip to content

Commit 7c85cca

Browse files
contrueCTcontrueCT
committed
fix(hstore): preserve range index paging order
1 parent 9300691 commit 7c85cca

11 files changed

Lines changed: 461 additions & 344 deletions

File tree

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/page/IdHolder.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,17 @@
3535
public abstract class IdHolder {
3636

3737
protected final Query query;
38+
private final boolean keepOrder;
3839
protected boolean exhausted;
3940

4041
public IdHolder(Query query) {
42+
this(query, false);
43+
}
44+
45+
public IdHolder(Query query, boolean keepOrder) {
4146
E.checkNotNull(query, "query");
4247
this.query = query;
48+
this.keepOrder = keepOrder;
4349
this.exhausted = false;
4450
}
4551

@@ -48,7 +54,7 @@ public Query query() {
4854
}
4955

5056
public boolean keepOrder() {
51-
return false;
57+
return this.keepOrder;
5258
}
5359

5460
@Override
@@ -97,7 +103,13 @@ public static class PagingIdHolder extends IdHolder {
97103

98104
public PagingIdHolder(ConditionQuery query,
99105
Function<ConditionQuery, PageIds> fetcher) {
100-
super(query.copy());
106+
this(query, fetcher, false);
107+
}
108+
109+
public PagingIdHolder(ConditionQuery query,
110+
Function<ConditionQuery, PageIds> fetcher,
111+
boolean keepOrder) {
112+
super(query.copy(), keepOrder);
101113
E.checkArgument(query.paging(),
102114
"Query '%s' must include page info", query);
103115
this.fetcher = fetcher;
@@ -142,7 +154,14 @@ public static class BatchIdHolder extends IdHolder
142154
public BatchIdHolder(ConditionQuery query,
143155
Iterator<BackendEntry> entries,
144156
Function<Long, Set<Id>> fetcher) {
145-
super(query);
157+
this(query, entries, fetcher, false);
158+
}
159+
160+
public BatchIdHolder(ConditionQuery query,
161+
Iterator<BackendEntry> entries,
162+
Function<Long, Set<Id>> fetcher,
163+
boolean keepOrder) {
164+
super(query, keepOrder);
146165
this.entries = entries;
147166
this.fetcher = fetcher;
148167
this.count = 0L;

‎hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java‎

Lines changed: 3 additions & 245 deletions
Original file line numberDiff line numberDiff line change
@@ -652,262 +652,20 @@ private void storeSelectedIndexField(IndexLabel indexLabel,
652652

653653
@Watched(prefix = "index")
654654
private IdHolder doIndexQuery(IndexLabel indexLabel, ConditionQuery query) {
655-
if (this.needHstoreRangeIndexOrder(indexLabel)) {
656-
return this.doHstoreRangeIndexQuery(indexLabel, query);
657-
}
658655
if (!query.paging()) {
659656
return this.doIndexQueryBatch(indexLabel, query);
660657
} else {
661658
return new PagingIdHolder(query, q -> {
662659
return this.doIndexQueryOnce(indexLabel, q);
663-
});
660+
}, this.keepBackendIndexOrder(indexLabel));
664661
}
665662
}
666663

667-
private boolean needHstoreRangeIndexOrder(IndexLabel indexLabel) {
664+
private boolean keepBackendIndexOrder(IndexLabel indexLabel) {
668665
return this.store().provider().isHstore() &&
669666
indexLabel.indexType().isRange();
670667
}
671668

672-
private IdHolder doHstoreRangeIndexQuery(IndexLabel indexLabel,
673-
ConditionQuery query) {
674-
if (!query.paging()) {
675-
if (query.noLimitAndOffset()) {
676-
return this.doIndexQueryBatch(indexLabel, query);
677-
}
678-
Set<Id> ids = this.querySortedRangeIndexIds(indexLabel, query);
679-
return this.newSortedRangeIndexBatchHolder(query, ids);
680-
}
681-
return new SortedRangePagingIdHolder(query, q -> {
682-
return this.querySortedRangeIndexPage(indexLabel, q);
683-
});
684-
}
685-
686-
private BatchIdHolder newSortedRangeIndexBatchHolder(ConditionQuery query,
687-
Set<Id> ids) {
688-
return new SortedRangeBatchIdHolder(query, ids);
689-
}
690-
691-
private Set<Id> querySortedRangeIndexIds(IndexLabel indexLabel,
692-
ConditionQuery query) {
693-
List<HugeIndex> indexes = this.querySortedRangeIndexes(indexLabel,
694-
query);
695-
Set<Id> ids = InsertionOrderUtil.newSet();
696-
for (HugeIndex index : indexes) {
697-
ids.addAll(index.elementIds());
698-
Query.checkForceCapacity(ids.size());
699-
}
700-
return ids;
701-
}
702-
703-
private PageIds querySortedRangeIndexPage(IndexLabel indexLabel,
704-
ConditionQuery query) {
705-
List<HugeIndex> indexes = this.querySortedRangeIndexes(indexLabel,
706-
query);
707-
Set<Id> allIds = InsertionOrderUtil.newSet();
708-
for (HugeIndex index : indexes) {
709-
allIds.addAll(index.elementIds());
710-
Query.checkForceCapacity(allIds.size());
711-
}
712-
if (allIds.isEmpty()) {
713-
return PageIds.EMPTY;
714-
}
715-
716-
int start = 0;
717-
if (!query.page().isEmpty()) {
718-
start = PageState.fromString(query.page()).offset();
719-
}
720-
if (start >= allIds.size()) {
721-
return PageIds.EMPTY;
722-
}
723-
724-
long total = allIds.size();
725-
long end = query.noLimit() ? total :
726-
Math.min(total, (long) start + query.limit());
727-
Set<Id> pageIds = CollectionUtil.subSet(allIds, start, (int) end);
728-
if (pageIds.isEmpty()) {
729-
return PageIds.EMPTY;
730-
}
731-
732-
int next = (int) end;
733-
PageState pageState;
734-
if (next < total) {
735-
pageState = new PageState(new byte[]{1}, next, pageIds.size());
736-
} else {
737-
pageState = new PageState(PageState.EMPTY_BYTES, 0,
738-
pageIds.size());
739-
}
740-
return new PageIds(pageIds, pageState);
741-
}
742-
743-
private List<HugeIndex> querySortedRangeIndexes(IndexLabel indexLabel,
744-
ConditionQuery query) {
745-
List<HugeIndex> indexes = new ArrayList<>();
746-
Iterator<BackendEntry> entries = null;
747-
String spaceGraph = this.params()
748-
.graph().spaceGraphName();
749-
LockUtil.Locks locks = new LockUtil.Locks(spaceGraph);
750-
ConditionQuery scanQuery = query.copy();
751-
scanQuery.page(null);
752-
scanQuery.offset(0L);
753-
scanQuery.limit(Query.NO_LIMIT);
754-
try {
755-
locks.lockReads(LockUtil.INDEX_LABEL_DELETE, indexLabel.id());
756-
locks.lockReads(LockUtil.INDEX_LABEL_REBUILD, indexLabel.id());
757-
if (!indexLabel.system()) {
758-
graph().indexLabel(indexLabel.id());
759-
}
760-
761-
entries = super.query(scanQuery).iterator();
762-
while (entries.hasNext()) {
763-
HugeIndex index = this.readMatchedIndex(indexLabel, scanQuery,
764-
entries.next());
765-
if (index == null) {
766-
continue;
767-
}
768-
this.removeExpiredIndexIfNeeded(index, scanQuery.showExpired());
769-
this.recordIndexValue(scanQuery, index);
770-
indexes.add(index);
771-
Query.checkForceCapacity(indexes.size());
772-
}
773-
} finally {
774-
locks.unlock();
775-
CloseableIterator.closeIterator(entries);
776-
}
777-
778-
Collections.sort(indexes, (a, b) -> {
779-
return this.compareRangeIndexValues(a, b);
780-
});
781-
return indexes;
782-
}
783-
784-
@SuppressWarnings({"rawtypes", "unchecked"})
785-
private int compareRangeIndexValues(HugeIndex left, HugeIndex right) {
786-
Object leftValue = left.fieldValues();
787-
Object rightValue = right.fieldValues();
788-
E.checkArgument(leftValue instanceof Comparable,
789-
"Invalid range index value '%s'", leftValue);
790-
E.checkArgument(rightValue instanceof Comparable,
791-
"Invalid range index value '%s'", rightValue);
792-
return ((Comparable) leftValue).compareTo(rightValue);
793-
}
794-
795-
static class SortedRangeBatchIdHolder extends BatchIdHolder {
796-
797-
private final List<Id> idList;
798-
private int offset;
799-
private PageIds pendingBatch;
800-
801-
SortedRangeBatchIdHolder(ConditionQuery query, Set<Id> ids) {
802-
super(query, Collections.emptyIterator(), batch -> {
803-
throw new IllegalStateException("Unexpected sorted index fetcher");
804-
});
805-
this.idList = new ArrayList<>(ids);
806-
this.offset = 0;
807-
this.pendingBatch = null;
808-
}
809-
810-
@Override
811-
public boolean keepOrder() {
812-
return true;
813-
}
814-
815-
@Override
816-
public boolean hasNext() {
817-
if (this.pendingBatch != null) {
818-
return true;
819-
}
820-
if (this.exhausted) {
821-
return false;
822-
}
823-
return this.offset < this.idList.size();
824-
}
825-
826-
@Override
827-
public IdHolder next() {
828-
if (!this.hasNext()) {
829-
throw new java.util.NoSuchElementException();
830-
}
831-
return this;
832-
}
833-
834-
@Override
835-
public PageIds fetchNext(String page, long batchSize) {
836-
E.checkArgument(page == null,
837-
"Not support page parameter by BatchIdHolder");
838-
E.checkArgument(batchSize >= 0L,
839-
"Invalid batch size value: %s", batchSize);
840-
if (this.pendingBatch != null) {
841-
PageIds result = this.pendingBatch;
842-
this.pendingBatch = null;
843-
return result;
844-
}
845-
return this.fetchBatch(batchSize);
846-
}
847-
848-
@Override
849-
public Set<Id> all() {
850-
Set<Id> allIds = InsertionOrderUtil.newSet();
851-
if (this.pendingBatch != null) {
852-
allIds.addAll(this.pendingBatch.ids());
853-
}
854-
if (this.offset < this.idList.size()) {
855-
allIds.addAll(this.idList.subList(this.offset,
856-
this.idList.size()));
857-
}
858-
this.close();
859-
return allIds;
860-
}
861-
862-
@Override
863-
public PageIds peekNext(long size) {
864-
E.checkArgument(this.pendingBatch == null,
865-
"Can't call peekNext() twice");
866-
this.pendingBatch = this.fetchBatch(size);
867-
return this.pendingBatch;
868-
}
869-
870-
@Override
871-
public void close() {
872-
this.exhausted = true;
873-
this.pendingBatch = null;
874-
this.offset = this.idList.size();
875-
}
876-
877-
private PageIds fetchBatch(long batchSize) {
878-
if (this.offset >= this.idList.size() || batchSize == 0L) {
879-
this.close();
880-
return PageIds.EMPTY;
881-
}
882-
883-
int end;
884-
if (batchSize == Query.NO_LIMIT) {
885-
end = this.idList.size();
886-
} else {
887-
end = (int) Math.min((long) this.idList.size(),
888-
this.offset + batchSize);
889-
}
890-
Set<Id> batchIds = InsertionOrderUtil.newSet();
891-
batchIds.addAll(this.idList.subList(this.offset, end));
892-
this.offset = end;
893-
this.exhausted = this.offset >= this.idList.size();
894-
return new PageIds(batchIds, PageState.EMPTY);
895-
}
896-
}
897-
898-
private static class SortedRangePagingIdHolder extends PagingIdHolder {
899-
900-
SortedRangePagingIdHolder(ConditionQuery query,
901-
Function<ConditionQuery, PageIds> fetcher) {
902-
super(query, fetcher);
903-
}
904-
905-
@Override
906-
public boolean keepOrder() {
907-
return true;
908-
}
909-
}
910-
911669
@Watched(prefix = "index")
912670
private IdHolder doIndexQueryBatch(IndexLabel indexLabel,
913671
ConditionQuery query) {
@@ -947,7 +705,7 @@ private IdHolder doIndexQueryBatch(IndexLabel indexLabel,
947705
} finally {
948706
locks.unlock();
949707
}
950-
});
708+
}, this.keepBackendIndexOrder(indexLabel));
951709
}
952710

953711
private void recordIndexValue(ConditionQuery query, HugeIndex index) {

‎hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreSessions.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,18 @@ public abstract BackendColumnIterator scan(String table,
150150
int scanType,
151151
byte[] query);
152152

153+
public BackendColumnIterator scan(String table,
154+
byte[] ownerKeyFrom,
155+
byte[] ownerKeyTo,
156+
byte[] keyFrom,
157+
byte[] keyTo,
158+
int scanType,
159+
byte[] query,
160+
long limit) {
161+
return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo,
162+
scanType, query);
163+
}
164+
153165
public abstract BackendColumnIterator scan(String table,
154166
byte[] ownerKeyFrom,
155167
byte[] ownerKeyTo,
@@ -159,6 +171,19 @@ public abstract BackendColumnIterator scan(String table,
159171
byte[] query,
160172
byte[] position);
161173

174+
public BackendColumnIterator scan(String table,
175+
byte[] ownerKeyFrom,
176+
byte[] ownerKeyTo,
177+
byte[] keyFrom,
178+
byte[] keyTo,
179+
int scanType,
180+
byte[] query,
181+
byte[] position,
182+
long limit) {
183+
return this.scan(table, ownerKeyFrom, ownerKeyTo, keyFrom, keyTo,
184+
scanType, query, position);
185+
}
186+
162187
public abstract BackendColumnIterator scan(String table,
163188
int codeFrom,
164189
int codeTo,

0 commit comments

Comments
 (0)