Skip to content

Commit

Permalink
Sanity Changes
Browse files Browse the repository at this point in the history
  • Loading branch information
Satyam Saxena committed Jul 10, 2024
1 parent 7094594 commit 5f632ec
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 18 deletions.
10 changes: 4 additions & 6 deletions ...-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import javax.validation.Valid;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.Comparator;
Expand All @@ -22,14 +21,15 @@
@AllArgsConstructor
@NoArgsConstructor
public class IdGeneratorConfig {
@NotEmpty
@NotNull
@Valid
private IdGeneratorRetryConfig retryConfig;

@Valid
private WeightedIdConfig weightedIdConfig;

@NotNull
@Min(1)
@Min(2)
private int idPoolSize;

@NotNull
Expand All @@ -46,9 +46,7 @@ public boolean isPartitionCountValid() {
if (weightedIdConfig != null) {
List<WeightedPartition> sortedPartitions = new ArrayList<>(weightedIdConfig.getPartitions());
sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart()));
if (sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getEnd() - sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getStart() != partitionCount) {
return false;
}
return sortedPartitions.get(sortedPartitions.size() - 1).getPartitionRange().getEnd() - sortedPartitions.get(0).getPartitionRange().getStart() + 1 == partitionCount;
}
return true;
}
Expand Down
2 changes: 2 additions & 0 deletions ...le/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ void testGenerateWithBenchmark() throws IOException {
val id = partitionAwareIdGenerator.generate("P");
id.ifPresent(value -> allIdsList.add(value.getId()));
},
true,
this.getClass().getName() + ".testGenerateWithBenchmark");
Assertions.assertEquals(numThreads * iterationCount, allIdsList.size());
checkUniqueIds(allIdsList);
Expand All @@ -84,6 +85,7 @@ void testGenerateWithConstraints() throws IOException {
val id = partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false);
id.ifPresent(value -> allIdsList.add(value.getId()));
},
false,
this.getClass().getName() + ".testGenerateWithConstraints");

Assertions.assertEquals(numThreads * iterationCount, allIdsList.size());
Expand Down
6 changes: 4 additions & 2 deletions ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
public class TestUtil {
private static final String OUTPUT_PATH = "perf/results/%s.json";

public double runMTTest(int numThreads, int iterationCount, final Consumer<Integer> supplier, String outputFileName) throws IOException {
public double runMTTest(int numThreads, int iterationCount, final Consumer<Integer> supplier, final boolean save_output, final String outputFileName) throws IOException {
final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
final List<Future<Long>> futures = IntStream.range(0, numThreads)
.mapToObj(i -> executorService.submit(() -> {
Expand All @@ -50,7 +50,9 @@ public double runMTTest(int numThreads, int iterationCount, final Consumer<Integ
}
})
.sum();
writeToFile(numThreads, iterationCount, total, outputFileName);
if (save_output){
writeToFile(numThreads, iterationCount, total, outputFileName);
}
log.warn("Finished Execution for {} iterations in avg time: {}", iterationCount, ((double) total) / numThreads);
return total;
}
Expand Down
22 changes: 12 additions & 10 deletions ...y-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
class WeightedIdGeneratorTest {
final int numThreads = 5;
final int iterationCount = 100000;
final int partitionCount = 64;
final int partitionCount = 1024;
final Function<String, Integer> partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount;
private WeightedIdGenerator weightedIdGenerator;
private IdGeneratorConfig idGeneratorConfig;
Expand All @@ -52,10 +52,10 @@ void setup() {
doNothing().when(meter).mark();
List<WeightedPartition> partitionConfigList = new ArrayList<>();
partitionConfigList.add(WeightedPartition.builder()
.partitionRange(PartitionRange.builder().start(0).end(31).build())
.partitionRange(PartitionRange.builder().start(0).end(511).build())
.weight(400).build());
partitionConfigList.add(WeightedPartition.builder()
.partitionRange(PartitionRange.builder().start(32).end(63).build())
.partitionRange(PartitionRange.builder().start(512).end(1023).build())
.weight(600).build());
val weightedIdConfig = WeightedIdConfig.builder()
.partitions(partitionConfigList)
Expand All @@ -80,16 +80,16 @@ void testGenerateWithBenchmark() throws IOException {
val id = weightedIdGenerator.generate("P");
id.ifPresent(value -> allIdsList.add(value.getId()));
},
true,
this.getClass().getName() + ".testGenerateWithBenchmark");
Assertions.assertEquals(numThreads * iterationCount, allIdsList.size());
checkUniqueIds(allIdsList);
checkDistribution(allIdsList);
}

@Test
void testGenerateAccuracy() throws IOException {
val allIdsList = Collections.synchronizedList(new ArrayList<String>());
val numThreads = 1;
val iterationCount = 400000;
val totalIdCount = numThreads * iterationCount;
val totalTime = TestUtil.runMTTest(
numThreads,
Expand All @@ -98,8 +98,8 @@ void testGenerateAccuracy() throws IOException {
val id = weightedIdGenerator.generate("P");
id.ifPresent(value -> allIdsList.add(value.getId()));
},
this.getClass().getName() + ".testGenerateWithBenchmark");
Assertions.assertEquals(numThreads * iterationCount, allIdsList.size());
false,
this.getClass().getName() + ".testGenerateAccuracy");
checkUniqueIds(allIdsList);
checkDistribution(allIdsList);
}
Expand All @@ -116,10 +116,11 @@ void testGenerateWithConstraints() throws IOException {
val id = weightedIdGenerator.generateWithConstraints("P", (String) null, false);
id.ifPresent(value -> allIdsList.add(value.getId()));
},
false,
this.getClass().getName() + ".testGenerateWithConstraints");
checkUniqueIds(allIdsList);

// Assert No ID was generated for Invalid partitions
// Assert no ID was generated for Invalid partitions
for (val id: allIdsList) {
val partitionId = partitionResolverSupplier.apply(id);
Assertions.assertTrue(partitionConstraint.isValid(partitionId));
Expand All @@ -142,8 +143,9 @@ void checkDistribution(List<String> allIdsList) {
val expectedIdCount = ((double) partition.getWeight() / weightedIdGenerator.getMaxShardWeight()) * ((double) allIdsList.size() / (partition.getPartitionRange().getEnd()-partition.getPartitionRange().getStart()+1));
int idCountForPartition = 0;
for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) {
Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId));
Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2);
log.warn("{} - {} - {}", expectedIdCount * 0.9, idCountMap.get(partitionId), expectedIdCount * 1.1);
Assertions.assertTrue(expectedIdCount * 0.9 <= idCountMap.get(partitionId));
Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.1);
idCountForPartition += idCountMap.get(partitionId);
}
log.debug("Partition ID Count: {} - Percentage: {}", idCountForPartition, (double) idCountForPartition * 100 / allIdsList.size());
Expand Down

0 comments on commit 5f632ec

Please sign in to comment.