Merge pull request #85 from george0st/change

Tune unit tests

Author: george0st

README.md

@@ -26,11 +26,11 @@ or two other alternative ways:
 
 You can see sample of relevant setting for CQL controllers:
  - [Cassandra setting](./nifi/cql-processor/nifi-cql/src/test/test-cassandra.json)
-   - access with IP addresses
+   - access with IP addresses (name/password)
  - [Scylla setting](./nifi/cql-processor/nifi-cql/src/test/test-scylla.json)
-   - access with IP address (localhost)
+   - access with IP address (without name/password)
  - [AstraDB setting](./nifi/cql-processor/nifi-cql/src/test/test-astra.json)
-   - access with security connection bundle (SCB) and token
+   - access with Security Connection Bundle (SCB) and Token
    - NOTE about SCB: see the file with **'.zip'** suffix
      - How to get the SCB? Go to the https://astra.datastax.com/...,
        Menu Databases/Your database/Region/[Download SCB](./docs/assets/astradb-download-SCB.png)

nifi/cql-processor/docs/todo-list.md

@@ -2,7 +2,6 @@ TO-DO list
 ==========
 
 1. Extend checks for Write/Validate
- - check count write and read
  - generate exception in case of null value for read-validate
 
 

nifi/cql-processor/nifi-cql-processors/src/main/java/org/george0st/processors/cql/PutCQL.java

@@ -83,7 +83,6 @@ public class PutCQL extends AbstractProcessor {
             .name("Table")
             .description("Table and schema name in CQL (expected format '<schema>.<table>').")
             .required(true)
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
             .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
             .build();
 

nifi/cql-processor/nifi-cql-processors/src/test/java/org/george0st/processors/cql/PutCQLBase.java

@@ -125,7 +125,7 @@ protected FlowFile runTestWithProperty(TestSetup setup, String content, Property
 
     private FlowFile coreTest(TestSetup setup, String content, boolean validate){
         try {
-            long finish, start, count;
+            long finish, start, countWrite, countRead;
             FlowFile result;
             boolean ok;
 
@@ -136,14 +136,14 @@ private FlowFile coreTest(TestSetup setup, String content, boolean validate){
             finish = System.currentTimeMillis();
 
             if (ok) {
-                count = Long.parseLong(result.getAttribute(PutCQL.ATTRIBUTE_COUNT));
+                countWrite = Long.parseLong(result.getAttribute(PutCQL.ATTRIBUTE_COUNT));
                 System.out.printf("Source: '%s'; WRITE; '%s': %s (%d ms); Items: %d; Perf: %.1f [calls/sec]%s",
                         result.getAttribute("CQLName"),
                         "FlowFile",
                         ReadableValue.fromMillisecond(finish - start),
                         finish - start,
-                        count,
-                        count / ((finish - start) / 1000.0),
+                        countWrite,
+                        countWrite / ((finish - start) / 1000.0),
                         System.lineSeparator());
 
                 if (validate) {
@@ -153,17 +153,20 @@ private FlowFile coreTest(TestSetup setup, String content, boolean validate){
                     // validate (read value from CSV and from CQL and compare content)
                     try (CqlSession session=testService.getSession()) {
                         start = System.currentTimeMillis();
-                        count = (new CsvCqlValidate(session, setup, CqlTestSchema.primaryKeys)).executeContent(content);
+                        countRead = (new CsvCqlValidate(session, setup, CqlTestSchema.primaryKeys)).executeContent(content);
                         finish = System.currentTimeMillis();
                     }
                     System.out.printf("Source: '%s'; VALIDATE; '%s': %s (%d ms); Items: %d; Perf: %.1f [calls/sec]%s",
                             result.getAttribute("CQLName"),
                             "FlowFile",
                             ReadableValue.fromMillisecond(finish - start),
                             finish - start,
-                            count,
-                            count / ((finish - start) / 1000.0),
+                            countRead,
+                            countRead / ((finish - start) / 1000.0),
                             System.lineSeparator());
+
+                    if (countWrite != countRead)
+                        throw new Exception("The amount of Write and Read operations are different");
                 }
                 return result;
             }

nifi/cql-processor/nifi-cql-processors/src/test/test-astra.json

@@ -10,8 +10,8 @@
   "consistencyLevel": "LOCAL_QUORUM",
 
   "writeConsistencyLevel": "LOCAL_QUORUM",
-  "batchSize": 200,
-  "batchType": "UNLOGGED",
+  "batchSize": 100,
+  "batchType": "LOGGED",
   "table": "prftest.csv2cql_test3",
   "dryRun": false,
 

nifi/cql-processor/nifi-cql-processors/src/test/test-cassandra.json

@@ -11,8 +11,8 @@
   "consistencyLevel": "LOCAL_ONE",
 
   "writeConsistencyLevel": "LOCAL_ONE",
-  "batchSize": 200,
-  "batchType": "UNLOGGED",
+  "batchSize": 100,
+  "batchType": "LOGGED",
   "table": "prftest.csv2cql_test3",
   "dryRun": false,
 

nifi/cql-processor/nifi-cql-processors/src/test/test-scylla.json

@@ -1,5 +1,5 @@
 {
-  "enable": true,
+  "enable": false,
   "name": "scylla",
   "ipAddresses": ["localhost"],
   "port": 9042,
@@ -9,8 +9,8 @@
   "consistencyLevel": "LOCAL_ONE",
 
   "writeConsistencyLevel": "LOCAL_ONE",
-  "batchSize": 200,
-  "batchType": "UNLOGGED",
+  "batchSize": 100,
+  "batchType": "LOGGED",
   "table": "prftest.csv2cql_test3",
   "dryRun": false,
 

nifi/cql-processor/nifi-cql-processors/src/test/test-yugabyte.json

@@ -8,8 +8,8 @@
   "consistencyLevel": "LOCAL_ONE",
 
   "writeConsistencyLevel": "LOCAL_ONE",
-  "batchSize": 200,
-  "batchType": "UNLOGGED",
+  "batchSize": 100,
+  "batchType": "LOGGED",
   "table": "prftest.csv2cql_test3",
   "dryRun": false,
 

nifi/cql-processor/nifi-cql/src/main/java/org/george0st/cql/CQLControllerService.java

@@ -50,8 +50,6 @@ public class CQLControllerService extends AbstractControllerService implements C
                     "(e.g. '192.168.0.1, 192.168.0.2, ...' or 'localhost').")
             .required(false)
             //.defaultValue("localhost")
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
-            //.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
             .addValidator(Validator.VALID)
             .build();
 
@@ -72,7 +70,6 @@ public class CQLControllerService extends AbstractControllerService implements C
                     "NOTE: the 'username' is 'clientId' and 'password' id 'secret', these values are from " +
                     "the file '*-token.json', downloaded from AstraDB web.")
             .required(false)
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
             .addValidator(Validator.VALID)
             .build();
 
@@ -81,19 +78,15 @@ public class CQLControllerService extends AbstractControllerService implements C
             .name("Username")
             .description("Username for the CQL connection.")
             .required(false)
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
             .addValidator(Validator.VALID)
-            //.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
             .build();
 
     public static final PropertyDescriptor PASSWORD = new PropertyDescriptor
             .Builder()
             .name("Password")
             .description("Password for the CQL connection.")
             .required(false)
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
             .addValidator(Validator.VALID)
-            //.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
             .sensitive(true)
             .build();
 
@@ -102,10 +95,7 @@ public class CQLControllerService extends AbstractControllerService implements C
             .name("Local Data Center")
             .description("Name of local data center e.g. 'dc1', 'datacenter1', etc.")
             .required(false)
-            //.defaultValue("datacenter1")
-            //.addValidator(StandardValidators.ATTRIBUTE_KEY_PROPERTY_NAME_VALIDATOR)
             .addValidator(Validator.VALID)
-            //.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
             .build();
 
     public static final PropertyDescriptor CONNECTION_TIMEOUT = new PropertyDescriptor
@@ -206,15 +196,7 @@ protected String getURI(final ConfigurationContext context) {
         //final String ip = context.getProperty(IP_ADDRESSES).evaluateAttributeExpressions().getValue();
         final String ip = context.getProperty(IP_ADDRESSES).getValue();
         final String secureConnectionBundl = context.getProperty(SECURE_CONNECTION_BUNDLE).getValue();
-
-//        final String port = context.getProperty(PORT).evaluateAttributeExpressions().getValue();
-//        final String user = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
-//        if (!ip.contains("@") && user != null && passw != null) {
-//            return uri.replaceFirst("://", "://" + URLEncoder.encode(user, StandardCharsets.UTF_8) + ":" + URLEncoder.encode(passw, StandardCharsets.UTF_8) + "@");
-//        } else {
-//            return uri;
-//        }
-        return ip != null ? ip: secureConnectionBundl;
+        return ip != null ? ip : secureConnectionBundl;
     }
 
     @Override
@@ -230,5 +212,4 @@ public void shutdown() {
     @Override
     public CqlSession getSession() { return cqlAccess != null ? cqlAccess.sessionBuilder.build() : null; }
 
-
 }