From 0cf3c8fe169339ad8f96ab88e98550e40b161023 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Thu, 15 Jun 2023 22:26:42 +0200 Subject: [PATCH] MINIFICPP-1825 Create Properties at compile time Also Relationships, Output Attributes and Dynamic Properties. This only applies to Processor and Controller Service metadata used in generating the manifest and PROCESSORS.md. I did not change Properties and Relationships used later on during runtime, as those are not always const and sometimes they need to own their own strings. There are a few changes in the generated PROCESSORS.md, mainly because the list of allowed values for a Property is no longer alphabetized; now the allowed values are listed in the order they appear in either the property definition or in the corresponding smart enum. Signed-off-by: Gabor Gyimesi This closes #1589 --- LICENSE | 24 ++ NOTICE | 1 + PROCESSORS.md | 163 +++++--- .../AWSCredentialsService.cpp | 46 +-- .../AWSCredentialsService.h | 29 +- extensions/aws/processors/DeleteS3Object.cpp | 6 +- extensions/aws/processors/DeleteS3Object.h | 26 +- extensions/aws/processors/FetchS3Object.cpp | 9 +- extensions/aws/processors/FetchS3Object.h | 34 +- extensions/aws/processors/ListS3.cpp | 18 +- extensions/aws/processors/ListS3.h | 58 ++- extensions/aws/processors/PutS3Object.cpp | 19 +- extensions/aws/processors/PutS3Object.h | 86 +++- extensions/aws/processors/S3Processor.cpp | 11 +- extensions/aws/processors/S3Processor.h | 157 +++++--- .../S3ProcessorStaticDefinitions.cpp | 282 ------------- extensions/aws/s3/S3Wrapper.cpp | 25 +- extensions/aws/s3/S3Wrapper.h | 20 +- .../aws/tests/AWSCredentialsServiceTest.cpp | 6 +- extensions/aws/tests/DeleteS3ObjectTests.cpp | 6 +- extensions/aws/tests/FetchS3ObjectTests.cpp | 4 +- extensions/aws/tests/PutS3ObjectTests.cpp | 22 +- extensions/aws/tests/S3TestsFixture.h | 32 +- .../AzureStorageCredentialsService.cpp | 44 +- .../AzureStorageCredentialsService.h | 39 +- .../AzureBlobStorageProcessorBase.cpp | 14 +- .../AzureBlobStorageProcessorBase.h | 48 ++- .../AzureBlobStorageSingleBlobProcessorBase.h | 10 +- .../AzureDataLakeStorageFileProcessorBase.h | 11 +- .../AzureDataLakeStorageProcessorBase.h | 20 +- .../processors/AzureStorageProcessorBase.cpp | 2 +- .../processors/AzureStorageProcessorBase.h | 8 +- ...AzureStorageProcessorStaticDefinitions.cpp | 291 -------------- .../processors/DeleteAzureBlobStorage.cpp | 6 +- .../azure/processors/DeleteAzureBlobStorage.h | 22 +- .../processors/DeleteAzureDataLakeStorage.cpp | 6 +- .../processors/DeleteAzureDataLakeStorage.h | 8 +- .../processors/FetchAzureBlobStorage.cpp | 6 +- .../azure/processors/FetchAzureBlobStorage.h | 26 +- .../processors/FetchAzureDataLakeStorage.cpp | 6 +- .../processors/FetchAzureDataLakeStorage.h | 33 +- .../azure/processors/ListAzureBlobStorage.cpp | 12 +- .../azure/processors/ListAzureBlobStorage.h | 43 +- .../processors/ListAzureDataLakeStorage.cpp | 20 +- .../processors/ListAzureDataLakeStorage.h | 69 ++-- .../azure/processors/PutAzureBlobStorage.cpp | 8 +- .../azure/processors/PutAzureBlobStorage.h | 24 +- .../processors/PutAzureDataLakeStorage.cpp | 16 +- .../processors/PutAzureDataLakeStorage.h | 60 +-- extensions/bustache/ApplyTemplate.cpp | 7 +- extensions/bustache/ApplyTemplate.h | 12 +- extensions/civetweb/processors/ListenHTTP.cpp | 98 ++--- extensions/civetweb/processors/ListenHTTP.h | 79 +++- extensions/civetweb/tests/ListenHTTPTests.cpp | 62 +-- .../coap/controllerservice/CoapConnector.cpp | 11 +- .../coap/controllerservice/CoapConnector.h | 29 +- extensions/coap/protocols/CoapC2Protocol.h | 2 +- .../coap/tests/CoapC2VerifyHeartbeat.cpp | 2 +- ...sticsearchCredentialsControllerService.cpp | 23 +- ...lasticsearchCredentialsControllerService.h | 26 +- .../elasticsearch/PostElasticsearch.cpp | 56 +-- extensions/elasticsearch/PostElasticsearch.h | 65 ++- .../tests/PostElasticsearchTests.cpp | 26 +- .../ExpressionContextBuilder.h | 2 +- .../ProcessContextExpr.cpp | 28 +- .../expression-language/ProcessContextExpr.h | 33 +- .../tests/ExpressionLanguageTests.cpp | 12 +- .../tests/ProcessContextExprTests.cpp | 56 ++- .../tests/RouteOnAttributeTests.cpp | 12 +- extensions/gcp/GCPAttributes.h | 52 +-- .../GCPCredentialsControllerService.cpp | 33 +- .../GCPCredentialsControllerService.h | 40 +- extensions/gcp/processors/DeleteGCSObject.cpp | 7 +- extensions/gcp/processors/DeleteGCSObject.h | 48 ++- extensions/gcp/processors/FetchGCSObject.cpp | 9 +- extensions/gcp/processors/FetchGCSObject.h | 48 ++- extensions/gcp/processors/GCSProcessor.cpp | 2 +- extensions/gcp/processors/GCSProcessor.h | 35 +- .../GCSProcessorStaticDefinitions.cpp | 267 ------------ extensions/gcp/processors/ListGCSBucket.cpp | 9 +- extensions/gcp/processors/ListGCSBucket.h | 133 +++--- extensions/gcp/processors/PutGCSObject.cpp | 9 +- extensions/gcp/processors/PutGCSObject.h | 171 ++++---- extensions/gcp/tests/DeleteGCSObjectTests.cpp | 40 +- extensions/gcp/tests/FetchGCSObjectTests.cpp | 32 +- .../GCPCredentialsControllerServiceTests.cpp | 23 +- extensions/gcp/tests/ListGCSBucketTests.cpp | 19 +- extensions/gcp/tests/PutGCSObjectTests.cpp | 82 ++-- extensions/gps/GetGPS.cpp | 20 +- extensions/gps/GetGPS.h | 33 +- extensions/http-curl/client/HTTPClient.h | 2 +- .../http-curl/processors/InvokeHTTP.cpp | 162 ++------ extensions/http-curl/processors/InvokeHTTP.h | 219 +++++++--- .../http-curl/sitetosite/HTTPProtocol.h | 2 +- .../http-curl/tests/C2NullConfiguration.cpp | 2 +- .../http-curl/tests/C2PauseResumeTest.cpp | 2 +- .../http-curl/tests/C2VerifyServeResults.cpp | 2 +- extensions/http-curl/tests/HTTPHandlers.h | 7 +- extensions/http-curl/tests/VerifyInvokeHTTP.h | 7 +- .../tests/VerifyInvokeHTTPPostTest.cpp | 2 +- .../http-curl/tests/unit/InvokeHTTPTests.cpp | 35 +- .../jni/ExecuteJavaControllerService.cpp | 44 +- extensions/jni/ExecuteJavaControllerService.h | 12 +- extensions/jni/ExecuteJavaProcessor.cpp | 22 +- extensions/jni/ExecuteJavaProcessor.h | 32 +- extensions/jni/JVMCreator.h | 2 +- extensions/jni/jvm/JavaControllerService.cpp | 31 +- extensions/jni/jvm/JavaControllerService.h | 27 +- extensions/jni/jvm/NarClassLoader.h | 11 +- .../KubernetesControllerService.cpp | 23 +- .../KubernetesControllerService.h | 23 +- .../CollectKubernetesPodMetrics.cpp | 15 +- .../processors/CollectKubernetesPodMetrics.h | 17 +- .../libarchive/ArchiveStreamProvider.cpp | 2 +- extensions/libarchive/BinFiles.cpp | 18 +- extensions/libarchive/BinFiles.h | 61 ++- .../libarchive/BinFilesStaticDefinitions.cpp | 107 ----- extensions/libarchive/CompressContent.cpp | 53 +-- extensions/libarchive/CompressContent.h | 91 +++-- extensions/libarchive/FocusArchiveEntry.cpp | 9 +- extensions/libarchive/FocusArchiveEntry.h | 13 +- extensions/libarchive/ManipulateArchive.cpp | 23 +- extensions/libarchive/ManipulateArchive.h | 37 +- extensions/libarchive/MergeContent.cpp | 24 +- extensions/libarchive/MergeContent.h | 94 +++-- extensions/libarchive/UnfocusArchiveEntry.cpp | 6 +- extensions/libarchive/UnfocusArchiveEntry.h | 7 +- extensions/librdkafka/ConsumeKafka.cpp | 43 +- extensions/librdkafka/ConsumeKafka.h | 206 +++++++--- extensions/librdkafka/KafkaProcessorBase.cpp | 20 +- extensions/librdkafka/KafkaProcessorBase.h | 75 ++-- .../KafkaProcessorStaticDefinitions.cpp | 279 ------------- extensions/librdkafka/PublishKafka.cpp | 32 +- extensions/librdkafka/PublishKafka.h | 178 ++++++-- extensions/lua/LuaScriptExecutor.h | 2 +- ...estExecuteScriptProcessorWithLuaScript.cpp | 20 +- .../mqtt/processors/AbstractMQTTProcessor.cpp | 18 +- .../mqtt/processors/AbstractMQTTProcessor.h | 130 ++++-- ...AbstractMQTTProcessorStaticDefinitions.cpp | 123 ------ extensions/mqtt/processors/ConsumeMQTT.cpp | 30 +- extensions/mqtt/processors/ConsumeMQTT.h | 62 ++- extensions/mqtt/processors/PublishMQTT.cpp | 15 +- extensions/mqtt/processors/PublishMQTT.h | 36 +- extensions/mqtt/tests/ConsumeMQTTTests.cpp | 10 +- extensions/mqtt/tests/PublishMQTTTests.cpp | 2 +- extensions/opc/include/fetchopc.h | 46 ++- extensions/opc/include/opc.h | 18 +- extensions/opc/include/opcbase.h | 39 +- extensions/opc/include/putopc.h | 58 ++- extensions/opc/src/fetchopc.cpp | 18 +- extensions/opc/src/opc.cpp | 8 - extensions/opc/src/opcbase.cpp | 12 +- .../opc/src/opcbaseStaticDefinitions.cpp | 149 ------- extensions/opc/src/putopc.cpp | 27 +- extensions/opencv/CaptureRTSPFrame.cpp | 45 +-- extensions/opencv/CaptureRTSPFrame.h | 39 +- extensions/opencv/MotionDetector.cpp | 45 +-- extensions/opencv/MotionDetector.h | 50 ++- .../opencv/tests/CaptureRTSPFrameTest.cpp | 16 +- .../SourceInitiatedSubscriptionListener.cpp | 93 +---- .../SourceInitiatedSubscriptionListener.h | 124 +++++- extensions/pcap/CapturePacket.cpp | 24 +- extensions/pcap/CapturePacket.h | 37 +- extensions/pdh/PerformanceDataMonitor.cpp | 46 +-- extensions/pdh/PerformanceDataMonitor.h | 41 +- .../pdh/tests/PerformanceDataMonitorTests.cpp | 6 +- .../procfs/processors/ProcFsMonitor.cpp | 39 +- extensions/procfs/processors/ProcFsMonitor.h | 66 +-- .../procfs/tests/ProcFsMonitorTests.cpp | 18 +- extensions/python/ExecutePythonProcessor.cpp | 38 +- extensions/python/ExecutePythonProcessor.h | 40 +- extensions/python/PythonScriptExecutor.h | 2 +- .../tests/ExecutePythonProcessorTests.cpp | 23 +- ...ExecuteScriptProcessorWithPythonScript.cpp | 14 +- .../rocksdb-repos/DatabaseContentRepository.h | 2 +- extensions/rocksdb-repos/FlowFileRepository.h | 2 +- .../rocksdb-repos/ProvenanceRepository.h | 2 +- .../controllers/RocksDbStateStorage.cpp | 27 +- .../controllers/RocksDbStateStorage.h | 29 +- extensions/script/ExecuteScript.cpp | 26 +- extensions/script/ExecuteScript.h | 47 ++- .../script/tests/ExecuteScriptTests.cpp | 8 +- .../sensors/GetEnvironmentalSensors.cpp | 31 +- extensions/sensors/GetEnvironmentalSensors.h | 11 +- extensions/sensors/GetMovementSensors.cpp | 34 +- extensions/sensors/GetMovementSensors.h | 11 +- extensions/sftp/processors/FetchSFTP.cpp | 14 +- extensions/sftp/processors/FetchSFTP.h | 79 +++- extensions/sftp/processors/ListSFTP.cpp | 37 +- extensions/sftp/processors/ListSFTP.h | 145 +++++-- extensions/sftp/processors/PutSFTP.cpp | 22 +- extensions/sftp/processors/PutSFTP.h | 144 +++++-- .../sftp/processors/SFTPProcessorBase.cpp | 8 +- .../sftp/processors/SFTPProcessorBase.h | 118 ++++-- .../SFTPProcessorStaticDefinitions.cpp | 325 --------------- extensions/sftp/tests/FetchSFTPTests.cpp | 16 +- extensions/sftp/tests/ListSFTPTests.cpp | 5 +- extensions/sftp/tests/PutSFTPTests.cpp | 44 +- extensions/splunk/PutSplunkHTTP.cpp | 6 +- extensions/splunk/PutSplunkHTTP.h | 42 +- .../splunk/QuerySplunkIndexingStatus.cpp | 8 +- extensions/splunk/QuerySplunkIndexingStatus.h | 52 ++- extensions/splunk/SplunkHECProcessor.cpp | 12 +- extensions/splunk/SplunkHECProcessor.h | 40 +- .../SplunkHECProcessorStaticDefinitions.cpp | 114 ------ .../splunk/tests/PutSplunkHTTPTests.cpp | 41 +- .../tests/QuerySplunkIndexingStatusTests.cpp | 26 +- extensions/sql/processors/ExecuteSQL.cpp | 15 +- extensions/sql/processors/ExecuteSQL.h | 21 +- extensions/sql/processors/FlowFileSource.cpp | 43 +- extensions/sql/processors/FlowFileSource.h | 60 +-- extensions/sql/processors/PutSQL.cpp | 7 +- extensions/sql/processors/PutSQL.h | 21 +- .../sql/processors/QueryDatabaseTable.cpp | 17 +- .../sql/processors/QueryDatabaseTable.h | 57 ++- extensions/sql/processors/SQLProcessor.cpp | 2 +- extensions/sql/processors/SQLProcessor.h | 10 +- .../SQLProcessorStaticDefinitions.cpp | 119 ------ extensions/sql/services/DatabaseService.cpp | 4 +- extensions/sql/services/DatabaseService.h | 9 +- extensions/sql/services/ODBCConnector.cpp | 3 + extensions/sql/services/ODBCConnector.h | 2 +- .../controllers/PersistentMapStateStorage.cpp | 27 +- .../controllers/PersistentMapStateStorage.h | 29 +- .../controllers/VolatileMapStateStorage.cpp | 8 +- .../controllers/VolatileMapStateStorage.h | 8 +- .../processors/AppendHostInfo.cpp | 37 +- .../processors/AppendHostInfo.h | 32 +- .../processors/AttributesToJSON.cpp | 56 +-- .../processors/AttributesToJSON.h | 63 ++- .../processors/DefragmentText.cpp | 40 +- .../processors/DefragmentText.h | 58 ++- .../processors/ExecuteProcess.cpp | 40 +- .../processors/ExecuteProcess.h | 39 +- .../processors/ExtractText.cpp | 64 +-- .../processors/ExtractText.h | 67 +++- .../processors/FetchFile.cpp | 82 +--- .../processors/FetchFile.h | 100 +++-- .../processors/GenerateFlowFile.cpp | 43 +- .../processors/GenerateFlowFile.h | 48 ++- .../processors/GetFile.cpp | 61 +-- .../standard-processors/processors/GetFile.h | 82 +++- .../standard-processors/processors/GetTCP.cpp | 67 +--- .../standard-processors/processors/GetTCP.h | 73 +++- .../processors/HashContent.cpp | 17 +- .../processors/HashContent.h | 31 +- .../processors/ListFile.cpp | 89 +--- .../standard-processors/processors/ListFile.h | 117 ++++-- .../processors/ListenSyslog.cpp | 82 +--- .../processors/ListenSyslog.h | 143 ++++--- .../processors/ListenTCP.cpp | 52 +-- .../processors/ListenTCP.h | 62 ++- .../processors/ListenUDP.cpp | 39 +- .../processors/ListenUDP.h | 49 ++- .../processors/LogAttribute.cpp | 47 +-- .../processors/LogAttribute.h | 58 ++- .../processors/NetworkListenerProcessor.cpp | 10 +- .../processors/NetworkListenerProcessor.h | 8 +- .../processors/PutFile.cpp | 46 +-- .../standard-processors/processors/PutFile.h | 74 ++-- .../standard-processors/processors/PutTCP.cpp | 65 +-- .../standard-processors/processors/PutTCP.h | 85 +++- .../standard-processors/processors/PutUDP.cpp | 21 +- .../standard-processors/processors/PutUDP.h | 24 +- .../processors/ReplaceText.cpp | 76 +--- .../processors/ReplaceText.h | 61 ++- .../processors/RetryFlowFile.cpp | 74 +--- .../processors/RetryFlowFile.h | 112 ++++-- .../processors/RouteOnAttribute.cpp | 13 +- .../processors/RouteOnAttribute.h | 9 +- .../processors/RouteText.cpp | 155 ++----- .../processors/RouteText.h | 156 ++++--- .../processors/TailFile.cpp | 117 +----- .../standard-processors/processors/TailFile.h | 101 ++++- .../processors/UpdateAttribute.cpp | 23 +- .../processors/UpdateAttribute.h | 10 +- .../integration/SecureSocketGetTCPTest.cpp | 2 +- .../tests/unit/AppendHostInfoTests.cpp | 4 +- .../tests/unit/AttributesToJSONTests.cpp | 36 +- .../tests/unit/DefragmentTextTests.cpp | 79 ++-- .../tests/unit/ExecuteProcessTests.cpp | 14 +- .../tests/unit/ExtractTextTests.cpp | 38 +- .../tests/unit/FetchFileTests.cpp | 6 +- .../tests/unit/GenerateFlowFileTests.cpp | 44 +- .../tests/unit/GetFileTests.cpp | 10 +- .../tests/unit/GetTCPTests.cpp | 8 +- .../tests/unit/HashContentTest.cpp | 30 +- .../tests/unit/ListFileTests.cpp | 24 +- .../tests/unit/ListenSyslogTests.cpp | 6 +- .../tests/unit/ListenTcpTests.cpp | 34 +- .../tests/unit/ManifestTests.cpp | 4 +- .../tests/unit/ProcessorTests.cpp | 26 +- .../tests/unit/PutFileTests.cpp | 52 +-- .../tests/unit/PutTCPTests.cpp | 12 +- .../tests/unit/PutUDPTests.cpp | 2 +- .../tests/unit/ReplaceTextTests.cpp | 134 +++---- .../tests/unit/RetryFlowFileTests.cpp | 40 +- .../tests/unit/TailFileTests.cpp | 348 ++++++++-------- .../tests/unit/UpdateAttributeTests.cpp | 4 +- .../tests/unit/YamlConfigurationTests.cpp | 63 +-- extensions/systemd/ConsumeJournald.cpp | 47 +-- extensions/systemd/ConsumeJournald.h | 58 ++- extensions/tensorflow/TFApplyGraph.cpp | 33 +- extensions/tensorflow/TFApplyGraph.h | 45 ++- .../tensorflow/TFConvertImageToTensor.cpp | 78 +--- .../tensorflow/TFConvertImageToTensor.h | 69 ++-- extensions/tensorflow/TFExtractTopLabels.cpp | 4 +- extensions/tensorflow/TFExtractTopLabels.h | 4 +- .../test-processors/KamikazeProcessor.cpp | 19 +- .../test-processors/KamikazeProcessor.h | 27 +- .../LogOnDestructionProcessor.h | 4 +- extensions/usb-camera/GetUSBCamera.cpp | 26 +- extensions/usb-camera/GetUSBCamera.h | 48 ++- .../CollectorInitiatedSubscription.cpp | 108 +---- .../CollectorInitiatedSubscription.h | 108 ++++- .../ConsumeWindowsEventLog.cpp | 177 ++------ .../ConsumeWindowsEventLog.h | 160 ++++++-- extensions/windows-event-log/TailEventLog.cpp | 13 +- extensions/windows-event-log/TailEventLog.h | 23 +- .../windows-event-log/tests/CWELTestUtils.h | 10 +- .../tests/ConsumeWindowsEventLogTests.cpp | 62 +-- libminifi/include/ForwardingNode.h | 6 +- libminifi/include/RemoteProcessorGroupPort.h | 41 +- libminifi/include/agent/agent_docs.h | 29 +- .../controllers/LinuxPowerManagementService.h | 49 ++- .../controllers/NetworkPrioritizerService.h | 43 +- .../include/controllers/SSLContextService.h | 111 +++-- .../UpdatePolicyControllerService.h | 32 +- libminifi/include/core/CachedValueValidator.h | 24 +- .../include/core/ConfigurableComponent.h | 24 +- libminifi/include/core/Connectable.h | 3 +- libminifi/include/core/Core.h | 63 ++- libminifi/include/core/DynamicProperty.h | 40 +- libminifi/include/core/FlowFile.h | 7 +- .../include/core/OutputAttributeDefinition.h | 55 +++ libminifi/include/core/ProcessContext.h | 73 ++-- .../include/core/ProcessContextBuilder.h | 2 +- libminifi/include/core/Processor.h | 6 +- libminifi/include/core/Property.h | 40 +- libminifi/include/core/PropertyBuilder.h | 225 ----------- libminifi/include/core/PropertyDefinition.h | 75 ++++ .../include/core/PropertyDefinitionBuilder.h | 97 +++++ libminifi/include/core/PropertyType.h | 357 +++++++++++++++++ libminifi/include/core/PropertyValidation.h | 379 ------------------ libminifi/include/core/PropertyValue.h | 32 +- libminifi/include/core/Relationship.h | 11 +- .../include/core/RelationshipDefinition.h | 25 +- .../{OutputAttribute.h => ValidationResult.h} | 34 +- .../core/controller/ControllerServiceNode.h | 11 +- .../core/flow/StructuredConnectionParser.h | 3 +- libminifi/include/core/state/Value.h | 3 + .../core/state/nodes/AgentInformation.h | 7 +- .../core/state/nodes/SupportedOperations.h | 3 +- libminifi/include/utils/ArrayUtils.h | 33 ++ libminifi/include/utils/Enum.h | 99 ++--- .../include/utils/ProcessorConfigUtils.h | 39 +- libminifi/include/utils/StringUtils.h | 10 +- libminifi/include/utils/net/Ssl.h | 2 +- .../utils/tls/WindowsCertStoreLocation.h | 53 +-- libminifi/src/Configuration.cpp | 270 ++++++------- libminifi/src/ForwardingNode.cpp | 4 +- libminifi/src/RemoteProcessorGroupPort.cpp | 28 +- libminifi/src/agent/JsonSchema.cpp | 5 +- .../LinuxPowerManagementService.cpp | 37 +- .../controllers/NetworkPrioritizerService.cpp | 30 +- .../src/controllers/SSLContextService.cpp | 93 +---- .../UpdatePolicyControllerService.cpp | 26 +- .../controllers/keyvalue/AutoPersistor.cpp | 1 - libminifi/src/core/CachedValueValidator.cpp | 41 ++ libminifi/src/core/ConfigurableComponent.cpp | 23 +- libminifi/src/core/Connectable.cpp | 9 +- libminifi/src/core/Property.cpp | 44 +- libminifi/src/core/PropertyType.cpp | 55 +++ .../src/core/flow/StructuredConfiguration.cpp | 17 +- .../core/flow/StructuredConnectionParser.cpp | 8 +- .../core/state/nodes/SupportedOperations.cpp | 9 +- libminifi/src/properties/Properties.cpp | 12 +- libminifi/src/utils/ProcessorConfigUtils.cpp | 33 +- libminifi/src/utils/net/Ssl.cpp | 2 +- libminifi/src/utils/tls/CertificateUtils.cpp | 2 +- .../utils/tls/WindowsCertStoreLocation.cpp | 87 ---- libminifi/test/DummyProcessor.h | 4 +- .../test/ReadFromFlowFileTestProcessor.cpp | 15 +- .../test/ReadFromFlowFileTestProcessor.h | 10 +- libminifi/test/StatefulProcessor.h | 4 +- libminifi/test/TestBase.cpp | 46 ++- libminifi/test/TestBase.h | 11 +- libminifi/test/Utils.h | 2 +- .../test/WriteToFlowFileTestProcessor.cpp | 6 +- libminifi/test/WriteToFlowFileTestProcessor.h | 10 +- .../archive-tests/CompressContentTests.cpp | 28 +- .../test/archive-tests/FocusArchiveTests.cpp | 14 +- .../archive-tests/ManipulateArchiveTests.cpp | 76 ++-- .../test/archive-tests/MergeFileTests.cpp | 3 +- .../AzureBlobStorageTestsFixture.h | 12 +- .../AzureDataLakeStorageTestsFixture.h | 20 +- .../DeleteAzureBlobStorageTests.cpp | 72 ++-- .../DeleteAzureDataLakeStorageTests.cpp | 36 +- .../FetchAzureBlobStorageTests.cpp | 74 ++-- .../FetchAzureDataLakeStorageTests.cpp | 46 +-- .../azure-tests/ListAzureBlobStorageTests.cpp | 18 +- .../ListAzureDataLakeStorageTests.cpp | 36 +- .../azure-tests/PutAzureBlobStorageTests.cpp | 68 ++-- .../PutAzureDataLakeStorageTests.cpp | 54 +-- .../bustache-tests/ApplyTemplateTests.cpp | 12 +- libminifi/test/flow-tests/CustomProcessors.h | 33 +- libminifi/test/gps-tests/GPSTests.cpp | 4 +- .../OnScheduleErrorHandlingTests.cpp | 2 +- libminifi/test/pcap-tests/PcapTest.cpp | 4 +- libminifi/test/rocksdb-tests/SwapTests.cpp | 16 +- libminifi/test/sql-tests/ExecuteSQLTests.cpp | 6 +- libminifi/test/sql-tests/FlowFileMatcher.h | 8 +- .../sql-tests/QueryDatabaseTableTests.cpp | 46 +-- libminifi/test/sql-tests/SQLTestPlan.h | 5 +- .../test/sql-tests/mocks/MockODBCService.h | 2 +- .../test/tensorflow-tests/TensorFlowTests.cpp | 20 +- libminifi/test/unit/ArrayUtilsTests.cpp | 63 +++ .../test/unit/ComponentManifestTests.cpp | 27 +- libminifi/test/unit/CoreTests.cpp | 49 +++ libminifi/test/unit/EnumTests.cpp | 29 +- libminifi/test/unit/IdTests.cpp | 2 +- libminifi/test/unit/MockClasses.h | 13 +- .../test/unit/ProcessorConfigUtilsTests.cpp | 13 +- .../test/unit/PropertyValidationTests.cpp | 227 ++++++----- libminifi/test/unit/UpdatePolicyTests.cpp | 2 +- .../unit/WindowsCertStoreLocationTests.cpp | 25 +- minifi_main/AgentDocs.cpp | 22 +- nanofi/include/cxx/CallbackProcessor.h | 10 +- nanofi/src/cxx/CallbackProcessor.cpp | 17 +- 429 files changed, 8841 insertions(+), 10071 deletions(-) delete mode 100644 extensions/aws/processors/S3ProcessorStaticDefinitions.cpp delete mode 100644 extensions/azure/processors/AzureStorageProcessorStaticDefinitions.cpp delete mode 100644 extensions/gcp/processors/GCSProcessorStaticDefinitions.cpp delete mode 100644 extensions/libarchive/BinFilesStaticDefinitions.cpp delete mode 100644 extensions/librdkafka/KafkaProcessorStaticDefinitions.cpp delete mode 100644 extensions/mqtt/processors/AbstractMQTTProcessorStaticDefinitions.cpp delete mode 100644 extensions/opc/src/opcbaseStaticDefinitions.cpp delete mode 100644 extensions/sftp/processors/SFTPProcessorStaticDefinitions.cpp delete mode 100644 extensions/splunk/SplunkHECProcessorStaticDefinitions.cpp delete mode 100644 extensions/sql/processors/SQLProcessorStaticDefinitions.cpp create mode 100644 libminifi/include/core/OutputAttributeDefinition.h delete mode 100644 libminifi/include/core/PropertyBuilder.h create mode 100644 libminifi/include/core/PropertyDefinition.h create mode 100644 libminifi/include/core/PropertyDefinitionBuilder.h create mode 100644 libminifi/include/core/PropertyType.h delete mode 100644 libminifi/include/core/PropertyValidation.h rename extensions/sql/services/DatabaseServiceStaticDefinitions.cpp => libminifi/include/core/RelationshipDefinition.h (50%) rename libminifi/include/core/{OutputAttribute.h => ValidationResult.h} (53%) create mode 100644 libminifi/src/core/CachedValueValidator.cpp create mode 100644 libminifi/src/core/PropertyType.cpp delete mode 100644 libminifi/src/utils/tls/WindowsCertStoreLocation.cpp create mode 100644 libminifi/test/unit/ArrayUtilsTests.cpp create mode 100644 libminifi/test/unit/CoreTests.cpp diff --git a/LICENSE b/LICENSE index dace74cfef..4e101b30e1 100644 --- a/LICENSE +++ b/LICENSE @@ -3368,3 +3368,27 @@ This product bundles 'CodeCoverage.cmake' which is available under a BSD 3-Clau ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +This product contains code snippets from https://bitwizeshift.github.io, which is available under an MIT license. + + The MIT License (MIT) + + Copyright (c) 2020 Matthew Rodusek + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/NOTICE b/NOTICE index ae7f8946c6..40a554a2c4 100644 --- a/NOTICE +++ b/NOTICE @@ -74,6 +74,7 @@ This software includes third party software subject to the following copyrights: - Zstandard - Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - LZ4 Library - Copyright (c) 2011-2020, Yann Collet - OpenSSL - Copyright (c) 1998-2022 The OpenSSL Project, Copyright (c) 1995-1998 Eric A. Young, Tim J. Hudson. All rights reserved. +- bitwizeshift.github.io - Copyright (c) 2020 Matthew Rodusek The licenses for these third party components are included in LICENSE.txt diff --git a/PROCESSORS.md b/PROCESSORS.md index 1fa6cdfec0..c6cb499511 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -48,8 +48,10 @@ limitations under the License. - [FetchSFTP](#fetchsftp) - [FocusArchiveEntry](#focusarchiveentry) - [GenerateFlowFile](#generateflowfile) +- [GetEnvironmentalSensors](#getenvironmentalsensors) - [GetFile](#getfile) - [GetGPS](#getgps) +- [GetMovementSensors](#getmovementsensors) - [GetTCP](#gettcp) - [GetUSBCamera](#getusbcamera) - [HashContent](#hashcontent) @@ -112,7 +114,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Network Interface Filter | | | A regular expression to filter ip addresses based on the name of the network interface | | Hostname Attribute | source.hostname | | Flowfile attribute used to record the agent's hostname | | IP Attribute | source.ipv4 | | Flowfile attribute used to record the agent's IP addresses in a comma separated list | -| Refresh Policy | On schedule | On every trigger
On schedule | When to recalculate the host info | +| Refresh Policy | On schedule | On schedule
On every trigger | When to recalculate the host info | ### Relationships @@ -314,7 +316,7 @@ In the list below, the names of required properties appear in bold. Any other pr |--------------------|-------------------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Mode | compress | compress
decompress | Indicates whether the processor should compress content or decompress content. | | Compression Level | 1 | | The compression level to use; this is valid only when using GZIP compression. | -| Compression Format | use mime.type attribute | bzip2
gzip
lzma
use mime.type attribute
xz-lzma2 | The compression format to use. | +| Compression Format | use mime.type attribute | gzip
lzma
xz-lzma2
bzip2
use mime.type attribute | The compression format to use. | | Update Filename | false | | Determines if filename extension need to be updated | | Encapsulate in TAR | true | | If true, on compression the FlowFile is added to a TAR archive and then compressed, and on decompression a compressed, TAR-encapsulated FlowFile is expected.
If false, on compression the content of the FlowFile simply gets compressed, and on decompression a simple compressed content is expected.
true is the behaviour compatible with older MiNiFi C++ versions, false is the behaviour compatible with NiFi. | | Batch Size | 1 | | Maximum number of FlowFiles processed in a single session | @@ -342,7 +344,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Batch Size** | 1000 | | The maximum number of entries processed in a single execution. | | **Payload Format** | Syslog | Raw
Syslog | Configures flow file content formatting. Raw: only the message. Syslog: similar to syslog or journalctl output. | | **Include Timestamp** | true | | Include message timestamp in the 'timestamp' attribute. | -| **Journal Type** | System | Both
System
User | Type of journal to consume. | +| **Journal Type** | System | User
System
Both | Type of journal to consume. | | **Process Old Messages** | false | | Process events created before the first usage (schedule) of the processor instance. | | **Timestamp Format** | %x %X %Z | | Format string to use when creating the timestamp attribute or writing messages in the syslog format. ISO/ISO 8601/ISO8601 are equivalent to "%FT%T%Ez". See https://howardhinnant.github.io/date/date.html#to_stream_formatting for all flags. | @@ -366,9 +368,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |------------------------------|----------------|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | SSL Context Service | | | SSL Context Service Name | -| **Security Protocol** | plaintext | plaintext
sasl_plaintext
sasl_ssl
ssl | Protocol used to communicate with brokers. Corresponds to Kafka's 'security.protocol' property. | +| **Security Protocol** | plaintext | plaintext
ssl
sasl_plaintext
sasl_ssl | Protocol used to communicate with brokers. Corresponds to Kafka's 'security.protocol' property. | | Kerberos Service Name | | | Kerberos Service Name | -| Kerberos Principal | | | Keberos Principal | +| Kerberos Principal | | | Kerberos Principal | | Kerberos Keytab Path | | | The path to the location on the local filesystem where the kerberos keytab is located. Read permission on the file is required. | | **SASL Mechanism** | GSSAPI | GSSAPI
PLAIN | The SASL mechanism to use for authentication. Corresponds to Kafka's 'sasl.mechanism' property. | | Username | | | The username when the SASL Mechanism is sasl_plaintext | @@ -379,11 +381,11 @@ In the list below, the names of required properties appear in bold. Any other pr | **Honor Transactions** | true | | Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If false, the Processor will use an "isolation level" of read_uncomitted. This means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions. If this value is true, MiNiFi will not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the consumer must wait for the producer to finish its entire transaction instead of pulling as the messages become available. | | **Group ID** | | | A Group ID is used to identify consumers that are within the same consumer group. Corresponds to Kafka's 'group.id' property.
**Supports Expression Language: true** | | **Offset Reset** | latest | earliest
latest
none | Allows you to manage the condition when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that data has been deleted). Corresponds to Kafka's 'auto.offset.reset' property. | -| **Key Attribute Encoding** | UTF-8 | Hex
UTF-8 | FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the attribute should be encoded. | +| **Key Attribute Encoding** | UTF-8 | UTF-8
Hex | FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the attribute should be encoded. | | Message Demarcator | | | Since KafkaConsumer receives messages in batches, you have an option to output FlowFiles which contains all Kafka messages in a single batch for a given topic and partition and this property allows you to provide a string (interpreted as UTF-8) to use for demarcating apart multiple Kafka messages. This is an optional property and if not provided each Kafka message received will result in a single FlowFile which time it is triggered.
**Supports Expression Language: true** | -| Message Header Encoding | UTF-8 | Hex
UTF-8 | Any message header that is found on a Kafka message will be added to the outbound FlowFile as an attribute. This property indicates the Character Encoding to use for deserializing the headers. | +| Message Header Encoding | UTF-8 | UTF-8
Hex | Any message header that is found on a Kafka message will be added to the outbound FlowFile as an attribute. This property indicates the Character Encoding to use for deserializing the headers. | | Headers To Add As Attributes | | | A comma separated list to match against all message headers. Any message header whose name matches an item from the list will be added to the FlowFile as an Attribute. If not specified, no Header values will be added as FlowFile attributes. The behaviour on when multiple headers of the same name are present is set using the Duplicate Header Handling attribute. | -| Duplicate Header Handling | Keep Latest | Comma-separated Merge
Keep First
Keep Latest | For headers to be added as attributes, this option specifies how to handle cases where multiple headers are present with the same key. For example in case of receiving these two headers: "Accept: text/html" and "Accept: application/xml" and we want to attach the value of "Accept" as a FlowFile attribute:
- "Keep First" attaches: "Accept -> text/html"
- "Keep Latest" attaches: "Accept -> application/xml"
- "Comma-separated Merge" attaches: "Accept -> text/html, application/xml"
| +| Duplicate Header Handling | Keep Latest | Keep First
Keep Latest
Comma-separated Merge | For headers to be added as attributes, this option specifies how to handle cases where multiple headers are present with the same key. For example in case of receiving these two headers: "Accept: text/html" and "Accept: application/xml" and we want to attach the value of "Accept" as a FlowFile attribute:
- "Keep First" attaches: "Accept -> text/html"
- "Keep Latest" attaches: "Accept -> application/xml"
- "Comma-separated Merge" attaches: "Accept -> text/html, application/xml"
| | Max Poll Records | 10000 | | Specifies the maximum number of records Kafka should return when polling each time the processor is triggered. | | **Max Poll Time** | 4 seconds | | Specifies the maximum amount of time the consumer can use for polling data from the brokers. Polling is a blocking operation, so the upper limit of this value is specified in 4 seconds. | | Session Timeout | 60 seconds | | Client group session and failure detection timeout. The consumer sends periodic heartbeats to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the broker configuration properties group.min.session.timeout.ms and group.max.session.timeout.ms. | @@ -409,7 +411,7 @@ In the list below, the names of required properties appear in bold. Any other pr |-----------------------------|---------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| | **Broker URI** | | | The URI to use to connect to the MQTT broker | | Client ID | | | MQTT client ID to use. WARNING: Must not be empty when using MQTT 3.1.0! | -| MQTT Version | 3.x AUTO | 3.1.0
3.1.1
3.x AUTO
5.0 | The MQTT specification version when connecting to the broker. | +| MQTT Version | 3.x AUTO | 3.x AUTO
3.1.0
3.1.1
5.0 | The MQTT specification version when connecting to the broker. | | **Topic** | | | The topic to subscribe to. | | Clean Session | true | | Whether to start afresh rather than remembering previous subscriptions. If true, then make broker forget subscriptions after disconnected. MQTT 3.x only. | | Clean Start | true | | Whether to start afresh rather than remembering previous subscriptions. MQTT 5.x only. | @@ -442,10 +444,10 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Description | -|---------------|---------------------------| -| _mqtt.broker_ | URI of the sending broker | -| _mqtt.topic_ | Topic of the message | +| Attribute | Relationship | Description | +|-------------|--------------|---------------------------| +| mqtt.broker | | URI of the sending broker | +| mqtt.topic | | Topic of the message | ## ConsumeWindowsEventLog @@ -460,7 +462,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Channel** | System | | The Windows Event Log Channel to listen to. In order to process logs from a log file use the format 'SavedLog:\'.
**Supports Expression Language: true** | +| **Channel** | System | | The Windows Event Log Channel to listen to. In order to process logs from a log file use the format 'SavedLog:\'.
**Supports Expression Language: true** | | **Query** | * | | XPath Query to filter events. (See https://msdn.microsoft.com/en-us/library/windows/desktop/dd996910(v=vs.85).aspx for examples.)
**Supports Expression Language: true** | | **Max Buffer Size** | 1 MB | | The individual Event Log XMLs are rendered to a buffer. This specifies the maximum size in bytes that the buffer will be allowed to grow to. (Limiting the maximum size of an individual Event XML.) | | **Inactive Duration To Reconnect** | 10 min | | If no new event logs are processed for the specified time period, this processor will try reconnecting to recover from a state where any further messages cannot be consumed. Such situation can happen if Windows Event Log service is restarted, or ERROR_EVT_QUERY_RESULT_STALE (15011) is returned. Setting no duration, e.g. '0 ms' disables auto-reconnection. | @@ -469,12 +471,12 @@ In the list below, the names of required properties appear in bold. Any other pr | Resolve Metadata in Attributes | true | | If true, any metadata that is resolved ( such as IDs or keyword metadata ) will be placed into attributes, otherwise it will be replaced in the XML or text output | | Event Header Delimiter | | | If set, the chosen delimiter will be used in the Event output header. Otherwise, a colon followed by spaces will be used. | | Event Header | LOG_NAME=Log Name, SOURCE = Source, TIME_CREATED = Date,EVENT_RECORDID=Record ID,EVENTID = Event ID,TASK_CATEGORY = Task Category,LEVEL = Level,KEYWORDS = Keywords,USER = User,COMPUTER = Computer, EVENT_TYPE = EventType | | Comma seperated list of key/value pairs with the following keys LOG_NAME, SOURCE, TIME_CREATED,EVENT_RECORDID,EVENTID,TASK_CATEGORY,LEVEL,KEYWORDS,USER,COMPUTER, and EVENT_TYPE. Eliminating fields will remove them from the header. | -| **Output Format** | Both | Both
JSON
Plaintext
XML | Set the output format type. In case 'Both' is selected the processor generates two flow files for every event captured in format XML and Plaintext | -| **JSON Format** | Simple | Simple
Raw
Flattened | Set the json format type. Only applicable if Output Format is set to 'JSON' | +| **Output Format** | Both | XML
Both
Plaintext
JSON | Set the output format type. In case 'Both' is selected the processor generates two flow files for every event captured in format XML and Plaintext | +| **JSON Format** | Simple | Raw
Simple
Flattened | Set the json format type. Only applicable if Output Format is set to 'JSON' | | Batch Commit Size | 1000 | | Maximum number of Events to consume and create to Flow Files from before committing. | | State Directory | CWELState | | DEPRECATED. Only use it for state migration from the state file, supplying the legacy state directory. | | **Process Old Events** | false | | This property defines if old events (which are created before first time server is started) should be processed. | -| Cache SID Lookups | true | true
false | Determines whether SID to name lookups are cached in memory | +| Cache SID Lookups | true | | Determines whether SID to name lookups are cached in memory | ### Relationships @@ -529,7 +531,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.
**Supports Expression Language: true** | | **Use Managed Identity Credentials** | false | | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | | Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | -| **Delete Snapshots Option** | None | Delete Snapshots Only
Include Snapshots
None | Specifies the snapshot deletion options to be used when deleting a blob. None: Deletes the blob only. Include Snapshots: Delete the blob and its snapshots. Delete Snapshots Only: Delete only the blob's snapshots. | +| **Delete Snapshots Option** | None | None
Include Snapshots
Delete Snapshots Only | Specifies the snapshot deletion options to be used when deleting a blob. None: Deletes the blob only. Include Snapshots: Delete the blob and its snapshots. Delete Snapshots Only: Delete only the blob's snapshots. | ### Relationships @@ -600,7 +602,6 @@ In the list below, the names of required properties appear in bold. Any other pr | gcs.error.domain | failure | The domain of the error occurred during operation. | - ## DeleteS3Object ### Description @@ -618,7 +619,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Secret Key | | | AWS account secret key
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | | AWS Credentials Provider service | | | The name of the AWS Credentials Provider controller service that is used to obtain AWS credentials. | -| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-west-1
us-west-2 | AWS Region | +| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-isob-east-1
us-iso-west-1
us-west-1
us-west-2 | AWS Region | | **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client | | Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints.
**Supports Expression Language: true** | | Proxy Host | | | Proxy host name or IP
**Supports Expression Language: true** | @@ -859,11 +860,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |--------------------------------------|---------------|-----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | File to Fetch | | | The fully-qualified filename of the file to fetch from the file system. If not defined the default ${absolute.path}/${filename} path is used.
**Supports Expression Language: true** | -| **Completion Strategy** | None | Delete File
Move File
None | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | +| **Completion Strategy** | None | None
Move File
Delete File | Specifies what to do with the original file on the file system once it has been pulled into MiNiFi | | Move Destination Directory | | | The directory to move the original file to once it has been fetched from the file system. This property is ignored unless the Completion Strategy is set to "Move File". If the directory does not exist, it will be created.
**Supports Expression Language: true** | -| **Move Conflict Strategy** | Rename | Fail
Keep Existing
Rename
Replace File | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | -| **Log level when file not found** | ERROR | DEBUG
ERROR
INFO
OFF
TRACE
WARN | Log level to use in case the file does not exist when the processor is triggered | -| **Log level when permission denied** | ERROR | DEBUG
ERROR
INFO
OFF
TRACE
WARN | Log level to use in case agent does not have sufficient permissions to read the file | +| **Move Conflict Strategy** | Rename | Rename
Replace File
Keep Existing
Fail | If Completion Strategy is set to Move File and a file already exists in the destination directory with the same name, this property specifies how that naming conflict should be resolved | +| **Log level when file not found** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
OFF | Log level to use in case the file does not exist when the processor is triggered | +| **Log level when permission denied** | ERROR | TRACE
DEBUG
INFO
WARN
ERROR
OFF | Log level to use in case agent does not have sufficient permissions to read the file | ### Relationships @@ -930,11 +931,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Certificate path | | | Path to the DER-encoded cert file | | Key path | | | Path to the DER-encoded key file | | Trusted server certificate path | | | Path to the DER-encoded trusted server certificate | -| **Node ID type** | | Int
Path
String | Specifies the type of the provided node ID | +| **Node ID type** | | Path
Int
String | Specifies the type of the provided node ID | | **Node ID** | | | Specifies the ID of the root node to traverse | | Namespace index | 0 | | The index of the namespace. Used only if node ID type is not path. | | Max depth | 0 | | Specifiec the max depth of browsing. 0 means unlimited. | -| **Lazy mode** | Off | Off
On | Only creates flowfiles from nodes with new timestamp from the server. | +| **Lazy mode** | Off | On
Off | Only creates flowfiles from nodes with new timestamp from the server. | ### Relationships @@ -961,7 +962,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Secret Key | | | AWS account secret key
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | | AWS Credentials Provider service | | | The name of the AWS Credentials Provider controller service that is used to obtain AWS credentials. | -| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-west-1
us-west-2 | AWS Region | +| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-isob-east-1
us-iso-west-1
us-west-1
us-west-2 | AWS Region | | **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client | | Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints.
**Supports Expression Language: true** | | Proxy Host | | | Proxy host name or IP
**Supports Expression Language: true** | @@ -1010,7 +1011,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Http Proxy Username | | | Http Proxy Username
**Supports Expression Language: true** | | Http Proxy Password | | | Http Proxy Password
**Supports Expression Language: true** | | **Remote File** | | | The fully qualified filename on the remote system
**Supports Expression Language: true** | -| **Completion Strategy** | None | Delete File
Move File
None | Specifies what to do with the original file on the server once it has been pulled into NiFi. If the Completion Strategy fails, a warning will be logged but the data will still be transferred. | +| **Completion Strategy** | None | None
Move File
Delete File | Specifies what to do with the original file on the server once it has been pulled into NiFi. If the Completion Strategy fails, a warning will be logged but the data will still be transferred. | | Move Destination Directory | | | The directory on the remote server to move the original file to once it has been ingested into NiFi. This property is ignored unless the Completion Strategy is set to 'Move File'. The specified directory must already exist on the remote system if 'Create Directory' is disabled, or the rename will fail.
**Supports Expression Language: true** | | **Create Directory** | false | | Specifies whether or not the remote directory should be created if it does not exist. | | Disable Directory Listing | false | | Control how 'Move Destination Directory' is created when 'Completion Strategy' is 'Move File' and 'Create Directory' is enabled. If set to 'true', directory listing is not performed prior to create missing directories. By default, this processor executes a directory listing command to see target directory existence before creating missing directories. However, there are situations that you might need to disable the directory listing such as the following. Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, then an error is returned because the directory already exists. | @@ -1072,6 +1073,26 @@ In the list below, the names of required properties appear in bold. Any other pr | success | success operational on the flow record | +## GetEnvironmentalSensors + +### Description + +Provides sensor information from known sensors to include humidity and pressure data + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|---------------------------------| +| success | All files are routed to success | + + ## GetFile ### Description @@ -1126,6 +1147,26 @@ In the list below, the names of required properties appear in bold. Any other pr | success | All files are routed to success | +## GetMovementSensors + +### Description + +Defines a processor that is able to retrieve sensor information from a class of known servo sensors + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|---------------------------------| +| success | All files are routed to success | + + ## GetTCP ### Description @@ -1246,7 +1287,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Put Response Body in Attribute | | | If set, the response body received back will be put into an attribute of the original FlowFile instead of a separate FlowFile. The attribute key to put to is determined by evaluating value of this property. | | Always Output Response | false | | Will force a response FlowFile to be generated and routed to the 'Response' relationship regardless of what the server status code received is | | Penalize on "No Retry" | false | | Enabling this property will penalize FlowFiles that are routed to the "No Retry" relationship. | -| **Invalid HTTP Header Field Handling Strategy** | transform | drop
fail
transform | Indicates what should happen when an attribute's name is not a valid HTTP header field name. Options: transform - invalid characters are replaced, fail - flow file is transferred to failure, drop - drops invalid attributes from HTTP message | +| **Invalid HTTP Header Field Handling Strategy** | transform | fail
transform
drop | Indicates what should happen when an attribute's name is not a valid HTTP header field name. Options: transform - invalid characters are replaced, fail - flow file is transferred to failure, drop - drops invalid attributes from HTTP message | ### Relationships @@ -1342,7 +1383,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Authorized DN Pattern | .* | | A Regular Expression to apply against the Distinguished Name of incoming connections. If the Pattern does not match the DN, the connection will be refused. | | SSL Certificate | | | File containing PEM-formatted file including TLS/SSL certificate and key | | SSL Certificate Authority | | | File containing trusted PEM-formatted certificates | -| SSL Verify Peer | no | no
yes | Whether or not to verify the client's certificate (yes/no) | +| SSL Verify Peer | no | yes
no | Whether or not to verify the client's certificate (yes/no) | | SSL Minimum Version | TLS1.2 | TLS1.2 | Minimum TLS/SSL version allowed (TLS1.2) | | HTTP Headers to receive as Attributes (Regex) | | | Specifies the Regular Expression that determines the names of HTTP Headers that should be passed along as FlowFile attributes | | Batch Size | 20000 | | Maximum number of buffered requests to be processed in a single batch. If set to zero all buffered requests are processed. | @@ -1373,7 +1414,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Parse Messages | false | | Indicates if the processor should parse the Syslog messages. If set to false, each outgoing FlowFile will only contain the sender, protocol, and port, and no additional attributes. | | Max Size of Message Queue | 10000 | | Maximum number of Syslog messages allowed to be buffered before processing them when the processor is triggered. If the buffer is full, the message is ignored. If set to zero the buffer is unlimited. | | SSL Context Service | | | The Controller Service to use in order to obtain an SSL Context. If this property is set, messages will be received over a secure connection. This Property is only considered if the Property has a value of "TCP". | -| Client Auth | NONE | NONE
REQUIRED
WANT | The client authentication policy to use for the SSL Context. Only used if an SSL Context Service is provided. | +| Client Auth | NONE | NONE
WANT
REQUIRED | The client authentication policy to use for the SSL Context. Only used if an SSL Context Service is provided. | ### Relationships @@ -1419,7 +1460,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Max Batch Size** | 500 | | The maximum number of messages to process at a time. | | **Max Size of Message Queue** | 10000 | | Maximum number of messages allowed to be buffered before processing them when the processor is triggered. If the buffer is full, the message is ignored. If set to zero the buffer is unlimited. | | SSL Context Service | | | The Controller Service to use in order to obtain an SSL Context. If this property is set, messages will be received over a secure connection. | -| Client Auth | NONE | NONE
REQUIRED
WANT | The client authentication policy to use for the SSL Context. Only used if an SSL Context Service is provided. | +| Client Auth | NONE | NONE
WANT
REQUIRED | The client authentication policy to use for the SSL Context. Only used if an SSL Context Service is provided. | ### Relationships @@ -1576,7 +1617,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Secret Key | | | AWS account secret key
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | | AWS Credentials Provider service | | | The name of the AWS Credentials Provider controller service that is used to obtain AWS credentials. | -| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-west-1
us-west-2 | AWS Region | +| **Region** | us-west-2 | af-south-1
ap-east-1
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-southeast-1
ap-southeast-2
ap-southeast-3
ca-central-1
cn-north-1
cn-northwest-1
eu-central-1
eu-north-1
eu-south-1
eu-west-1
eu-west-2
eu-west-3
me-central-1
me-south-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-isob-east-1
us-iso-west-1
us-west-1
us-west-2 | AWS Region | | **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client | | Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints.
**Supports Expression Language: true** | | Proxy Host | | | Proxy host name or IP
**Supports Expression Language: true** | @@ -1627,16 +1668,16 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port of the proxy server
**Supports Expression Language: true** | | Http Proxy Username | | | Http Proxy Username
**Supports Expression Language: true** | | Http Proxy Password | | | Http Proxy Password
**Supports Expression Language: true** | -| **Listing Strategy** | Tracking Timestamps | Tracking Entities
Tracking Timestamps | Specify how to determine new/updated entities. See each strategy descriptions for detail. | +| **Listing Strategy** | Tracking Timestamps | Tracking Timestamps
Tracking Entities | Specify how to determine new/updated entities. See each strategy descriptions for detail. | | Remote Path | | | The fully qualified filename on the remote system
**Supports Expression Language: true** | | **Search Recursively** | false | | If true, will pull files from arbitrarily nested subdirectories; otherwise, will not traverse subdirectories | | **Follow symlink** | false | | If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories | | File Filter Regex | | | Provides a Java Regular Expression for filtering Filenames; if a filter is supplied, only files whose names match that Regular Expression will be fetched | | Path Filter Regex | | | When Search Recursively is true, then only subdirectories whose path matches the given Regular Expression will be scanned | | **Ignore Dotted Files** | true | | If true, files whose names begin with a dot (".") will be ignored | -| **Target System Timestamp Precision** | Auto Detect | Auto Detect
Milliseconds
Minutes
Seconds | Specify timestamp precision at the target system. Since this processor uses timestamp of entities to decide which should be listed, it is crucial to use the right timestamp precision. | +| **Target System Timestamp Precision** | Auto Detect | Auto Detect
Milliseconds
Seconds
Minutes | Specify timestamp precision at the target system. Since this processor uses timestamp of entities to decide which should be listed, it is crucial to use the right timestamp precision. | | Entity Tracking Time Window | | | Specify how long this processor should track already-listed entities. 'Tracking Entities' strategy can pick any entity whose timestamp is inside the specified time window. For example, if set to '30 minutes', any entity having timestamp in recent 30 minutes will be the listing target when this processor runs. A listed entity is considered 'new/updated' and a FlowFile is emitted if one of following condition meets: 1. does not exist in the already-listed entities, 2. has newer timestamp than the cached entity, 3. has different size than the cached entity. If a cached entity's timestamp becomes older than specified time window, that entity will be removed from the cached already-listed entities. Used by 'Tracking Entities' strategy. | -| Entity Tracking Initial Listing Target | All Available | All Available
Tracking Time Window | Specify how initial listing should be handled. Used by 'Tracking Entities' strategy. | +| Entity Tracking Initial Listing Target | All Available | Tracking Time Window
All Available | Specify how initial listing should be handled. Used by 'Tracking Entities' strategy. | | **Minimum File Age** | 0 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | | Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | | **Minimum File Size** | 0 B | | The minimum size that a file must be in order to be pulled | @@ -1661,10 +1702,10 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------------|---------------|---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Log Level | | debug
error
info
trace
warn | The Log Level to use when logging the Attributes | +| Log Level | | info
trace
error
warn
debug | The Log Level to use when logging the Attributes | | Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | -| Log Payload | false | | If true, the FlowFile's payload will be logged, in addition to its attributes.otherwise, just the Attributes will be logged | +| Log Payload | false | | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | | Hexencode Payload | false | | If true, the FlowFile's payload will be logged in a hexencoded format | | Maximum Payload Line Length | 0 | | The logged payload will be broken into lines this long. 0 means no newlines will be added. | | Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | @@ -1722,15 +1763,15 @@ In the list below, the names of required properties appear in bold. Any other pr | Maximum number of Bins | 100 | | Specifies the maximum number of bins that can be held in memory at any one time | | Max Bin Age | | | The maximum age of a Bin that will trigger a Bin to be complete. Expected format is