Skip to content

Commit d4b0770

Browse files
george0stgeorge0st
authored
Merge pull request #105 from george0st/change
Tune performance tests
2 parents 1620e01 + 218dfe4 commit d4b0770

File tree

3 files changed

+97
-16
lines changed

3 files changed

+97
-16
lines changed

‎nifi/cql-processor/docs/useful-links.md‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
## Useful links
22

33
- Custom processor
4-
- https://medium.com/@mr.sinchan.banerjee/nifi-custom-processor-series-part-3-junit-test-with-nifi-mock-a935a1a4e3e5
4+
- https://medium.com/@mr.sinchan.banerjee/nifi-custom-processor-series-part-1-writing-your-first-custom-processor-dd7aa901c896
5+
- https://medium.com/@mr.sinchan.banerjee/nifi-custom-processor-series-part-2-add-custom-validation-to-custom-processor-95bb56763834
6+
- https://medium.com/@mr.sinchan.banerjee/nifi-custom-processor-series-part-3-junit-test-with-nifi-mock-a935a1a4e3e5
57
- Custom processor and controller
6-
- https://medium.com/hashmapinc/creating-custom-processors-and-controllers-in-apache-nifi-e14148740ea
8+
- https://medium.com/hashmapinc/creating-custom-processors-and-controllers-in-apache-nifi-e14148740ea
79
- Sample of code from Github
8-
- MongoDB ControllerService
9-
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBControllerService.java#L187
10-
- MongoDB with SSL_CONTEXT_SERVICE
11-
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-mongodb-bundle/nifi-mongodb-client-service-api/src/main/java/org/apache/nifi/mongodb/MongoDBClientService.java
12-
- DBCPConnectionPool
13-
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
14-
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/AbstractDBCPConnectionPool.java
10+
- MongoDB ControllerService
11+
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-mongodb-bundle/nifi-mongodb-services/src/main/java/org/apache/nifi/mongodb/MongoDBControllerService.java#L187
12+
- MongoDB with SSL_CONTEXT_SERVICE
13+
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-mongodb-bundle/nifi-mongodb-client-service-api/src/main/java/org/apache/nifi/mongodb/MongoDBClientService.java
14+
- DBCPConnectionPool
15+
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
16+
- https://github.com/apache/nifi/blob/main/nifi-extension-bundles/nifi-extension-utils/nifi-dbcp-base/src/main/java/org/apache/nifi/dbcp/AbstractDBCPConnectionPool.java

‎nifi/cql-processor/nifi-cql-processors/src/test/java/org/george0st/processors/cql/GetCQLBase.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -122,16 +122,14 @@ protected List<MockFlowFile> runTestParallelWithProperty(TestSetupRead setup, St
122122
}
123123

124124
private void printResult(MockFlowFile result, TestSetupRead setup, long start, long finish){
125-
long countWrite;
126-
127-
countWrite = Long.parseLong(result.getAttribute(CQLAttributes.READ_COUNT));
125+
long countRead = Long.parseLong(result.getAttribute(CQLAttributes.READ_COUNT));
128126
System.out.printf("Source: '%s'; READ; '%s': %s (%d ms); Items: %d; Perf: %.1f [calls/sec]%s",
129127
setup.name,
130128
"FlowFile",
131129
ReadableValue.fromMillisecond(finish - start),
132130
finish - start,
133-
countWrite,
134-
countWrite / ((finish - start) / 1000.0),
131+
countRead,
132+
countRead / ((finish - start) / 1000.0),
135133
System.lineSeparator());
136134
}
137135

‎nifi/cql-processor/nifi-cql-processors/src/test/java/org/george0st/processors/cql/GetCQLPerformance.java‎

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ public void testRead150K() throws Exception {
120120
}
121121

122122
@Test
123-
public void testRead10KMultiple() throws Exception {
123+
public void testRead10KMultipleWithContent() throws Exception {
124124
String content = "\"colbigint\",\"colint\",\"coltext\",\"colfloat\",\"coldouble\",\"coldate\",\"coltime\",\"coltimestamp\",\"colboolean\",\"coluuid\",\"colsmallint\",\"coltinyint\",\"coltimeuuid\",\"colvarchar\"\n" +
125125
"\"0\",\"1064\",\"zeVOKGnORq\",\"627.6811\",\"395.8522407512559\",\"1971-11-12\",\"03:37:15\",\"2000-09-25T22:18:45Z\",\"false\",\"6080071f-4dd1-4ea5-b711-9ad0716e242a\",\"8966\",\"55\",\"f45e58f5-c3b7-11ef-8d19-97ae87be7c54\",\"Tzxsw\"\n" +
126126
"\"1\",\"1709\",\"7By0z5QEXh\",\"652.03955\",\"326.9081263857284\",\"2013-12-17\",\"08:43:09\",\"2010-04-27T07:02:27Z\",\"false\",\"7d511666-2f81-41c4-9d5c-a5fa87f7d1c3\",\"24399\",\"38\",\"f45e8006-c3b7-11ef-8d19-172ff8d0d752\",\"exAbN\"\n";
@@ -138,7 +138,32 @@ public void testRead10KMultiple() throws Exception {
138138
for (TestSetupRead setup : setups) {
139139
setup.columnNames="colbigint, colint";
140140

141-
//results = runTestParallel(setup, content, 5);
141+
results = runTestParallel(setup, content, 5);
142+
143+
// check amount of read items
144+
for (MockFlowFile result: results) {
145+
assertNotNull(result, String.format("Issue with processing in '%s'", setup.name));
146+
assertEquals(10_000, Long.parseLong(result.getAttribute(CQLAttributes.READ_COUNT)));
147+
}
148+
}
149+
}
150+
151+
@Test
152+
public void testRead10KMultiple() throws Exception {
153+
List<MockFlowFile> results;
154+
155+
// Prepare data for performance test
156+
PutCQLPerformance putCQL = new PutCQLPerformance();
157+
putCQL.init();
158+
putCQL.csvRandomWrite10K();
159+
160+
// sleep before read (2 seconds)
161+
Thread.sleep(2000);
162+
163+
// Read data
164+
for (TestSetupRead setup : setups) {
165+
setup.columnNames="colbigint, colint";
166+
142167
results = runTestParallel(setup, 5);
143168

144169
// check amount of read items
@@ -148,4 +173,60 @@ public void testRead10KMultiple() throws Exception {
148173
}
149174
}
150175
}
176+
177+
@Test
178+
public void testRead100KMultipleWithContent() throws Exception {
179+
String content = "\"colbigint\",\"colint\",\"coltext\",\"colfloat\",\"coldouble\",\"coldate\",\"coltime\",\"coltimestamp\",\"colboolean\",\"coluuid\",\"colsmallint\",\"coltinyint\",\"coltimeuuid\",\"colvarchar\"\n" +
180+
"\"0\",\"1064\",\"zeVOKGnORq\",\"627.6811\",\"395.8522407512559\",\"1971-11-12\",\"03:37:15\",\"2000-09-25T22:18:45Z\",\"false\",\"6080071f-4dd1-4ea5-b711-9ad0716e242a\",\"8966\",\"55\",\"f45e58f5-c3b7-11ef-8d19-97ae87be7c54\",\"Tzxsw\"\n" +
181+
"\"1\",\"1709\",\"7By0z5QEXh\",\"652.03955\",\"326.9081263857284\",\"2013-12-17\",\"08:43:09\",\"2010-04-27T07:02:27Z\",\"false\",\"7d511666-2f81-41c4-9d5c-a5fa87f7d1c3\",\"24399\",\"38\",\"f45e8006-c3b7-11ef-8d19-172ff8d0d752\",\"exAbN\"\n";
182+
List<MockFlowFile> results;
183+
184+
// Prepare data for performance test
185+
PutCQLPerformance putCQL = new PutCQLPerformance();
186+
putCQL.init();
187+
putCQL.csvRandomWrite100K();
188+
189+
// sleep before read (2 seconds)
190+
Thread.sleep(2000);
191+
192+
// Read data
193+
for (TestSetupRead setup : setups) {
194+
setup.columnNames="colbigint, colint";
195+
196+
results = runTestParallel(setup, content, 5);
197+
198+
// check amount of read items
199+
for (MockFlowFile result: results) {
200+
assertNotNull(result, String.format("Issue with processing in '%s'", setup.name));
201+
assertEquals(100_000, Long.parseLong(result.getAttribute(CQLAttributes.READ_COUNT)));
202+
}
203+
}
204+
}
205+
206+
@Test
207+
public void testRead100KMultiple() throws Exception {
208+
List<MockFlowFile> results;
209+
210+
// Prepare data for performance test
211+
PutCQLPerformance putCQL = new PutCQLPerformance();
212+
putCQL.init();
213+
putCQL.csvRandomWrite100K();
214+
215+
// sleep before read (2 seconds)
216+
Thread.sleep(2000);
217+
218+
// Read data
219+
for (TestSetupRead setup : setups) {
220+
setup.columnNames="colbigint, colint";
221+
222+
results = runTestParallel(setup, 5);
223+
224+
// check amount of read items
225+
for (MockFlowFile result: results) {
226+
assertNotNull(result, String.format("Issue with processing in '%s'", setup.name));
227+
assertEquals(100_000, Long.parseLong(result.getAttribute(CQLAttributes.READ_COUNT)));
228+
}
229+
}
230+
}
231+
151232
}

0 commit comments

Comments
 (0)