Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance enhancements for high cardinality types #480

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Performance enhancements for high cardinality types #480

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bbeeabd
expose read states in producer
zurdoron May 6, 2020
ad57fa0
expose current in read state helper
zurdoron May 6, 2020
6b6873b
parallel enhancements for high cardinality types
zurdoron May 6, 2020
f166ed9
add import
zurdoron May 6, 2020
cc741f0
parallelism enhancements for high cardinality types
zurdoron Jul 25, 2020
466f4ab
remove unused imports
zurdoron Jul 25, 2020
68cb89c
revert
zurdoron Jul 25, 2020
93111b9
Revert "revert"
zurdoron Jul 27, 2020
7b5ade3
Producer Listener support refactored for code reuse (#448)
Sunjeet Feb 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 25 additions & 14 deletions hollow/src/main/java/com/netflix/hollow/core/memory/ByteArrayOrdinalMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@
*/
package com.netflix.hollow.core.memory;

import com.netflix.hollow.api.error.HollowException;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.memory.encoding.VarInt;
import com.netflix.hollow.core.memory.pool.WastefulRecycler;

import java.util.Arrays;
import java.util.BitSet;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.stream.IntStream;

/**
* This data structure maps byte sequences to ordinals. This is a hash table.
Expand Down Expand Up @@ -371,7 +375,7 @@ public void compact(ThreadSafeBitSet usedOrdinals) {
}
}

Arrays.sort(populatedReverseKeys);
Arrays.parallelSort(populatedReverseKeys);

SegmentedByteArray arr = byteData.getUnderlyingArray();
long currentCopyPointer = 0;
Expand Down Expand Up @@ -543,18 +547,25 @@ private void populateNewHashArray(AtomicLongArray newKeys, long[] valuesToAdd, i

int modBitmask = newKeys.length() - 1;

for (int i = 0; i < length; i++) {
long value = valuesToAdd[i];
if (value != EMPTY_BUCKET_VALUE) {
int hash = rehashPreviouslyAddedData(value);
int bucket = hash & modBitmask;
while (newKeys.get(bucket) != EMPTY_BUCKET_VALUE) {
bucket = (bucket + 1) & modBitmask;
}
// Volatile store not required, could use plain store
// See VarHandles for JDK >= 9
newKeys.lazySet(bucket, value);
}
ForkJoinPool forkJoinPool = new ForkJoinPool();

try {
forkJoinPool.submit(()->{
IntStream.range(0,length).parallel().forEach(i->{
long value = valuesToAdd[i];
if (value != EMPTY_BUCKET_VALUE) {
int hash = rehashPreviouslyAddedData(value);
int bucket = hash & modBitmask;
boolean isSet = false;
while(!isSet){
isSet = newKeys.compareAndSet(bucket,EMPTY_BUCKET_VALUE,value);
bucket = (bucket + 1) & modBitmask;
}
}
});
}).get();
} catch (Exception e) {
throw new HollowException("Unable to populate byte array map", e);
}
}

Expand Down Expand Up @@ -603,4 +614,4 @@ public static int getOrdinal(long pointerAndOrdinal) {
return (int) (pointerAndOrdinal >>> BITS_PER_POINTER);
}

}
}