From 914289a5a89f03406dba8be2147121e04e20ac4a Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Tue, 8 Aug 2023 13:31:59 +0200 Subject: [PATCH] MINIFICPP-2158 Show true|false as allowed values of boolean properties in PROCESSORS.md Closes #1625 Signed-off-by: Marton Szasz --- PROCESSORS.md | 185 +++++++++++++++++++------------------- minifi_main/AgentDocs.cpp | 17 ++-- 2 files changed, 103 insertions(+), 99 deletions(-) diff --git a/PROCESSORS.md b/PROCESSORS.md index 2211f4ef71..286eab5236 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -159,8 +159,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Attributes List | | | Comma separated list of attributes to be included in the resulting JSON. If this value is left empty then all existing Attributes will be included. This list of attributes is case sensitive. If an attribute specified in the list is not found it will be be emitted to the resulting JSON with an empty string or NULL value. | | Attributes Regular Expression | | | Regular expression that will be evaluated against the flow file attributes to select the matching attributes. Both the matching attributes and the selected attributes from the Attributes List property will be written in the resulting JSON. | | **Destination** | flowfile-attribute | flowfile-attribute
flowfile-content | Control if JSON value is written as a new flowfile attribute 'JSONAttributes' or written in the flowfile content. Writing to flowfile content will overwrite any existing flowfile content. | -| **Include Core Attributes** | true | | Determines if the FlowFile core attributes which are contained in every FlowFile should be included in the final JSON value generated. | -| **Null Value** | false | | If true a non existing selected attribute will be NULL in the resulting JSON. If false an empty string will be placed in the JSON. | +| **Include Core Attributes** | true | true
false | Determines if the FlowFile core attributes which are contained in every FlowFile should be included in the final JSON value generated. | +| **Null Value** | false | true
false | If true a non existing selected attribute will be NULL in the resulting JSON. If false an empty string will be placed in the JSON. | ### Relationships @@ -212,7 +212,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Batch Size | 50 | | The number of packets to combine within a given PCAP | | Network Controllers | .* | | Regular expression of the network controller(s) to which we will attach | | Base Directory | /tmp/ | | Scratch directory for PCAP files | -| Capture Bluetooth | false | | True indicates that we support bluetooth interfaces | +| Capture Bluetooth | false | true
false | True indicates that we support bluetooth interfaces | ### Relationships @@ -312,14 +312,14 @@ Compresses or decompresses the contents of FlowFiles using a user-specified comp In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|------------------------|-------------------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Mode** | compress | compress
decompress | Indicates whether the processor should compress content or decompress content. | -| Compression Level | 1 | | The compression level to use; this is valid only when using GZIP compression. | -| **Compression Format** | use mime.type attribute | gzip
lzma
xz-lzma2
bzip2
use mime.type attribute | The compression format to use. | -| Update Filename | false | | Determines if filename extension need to be updated | -| Encapsulate in TAR | true | | If true, on compression the FlowFile is added to a TAR archive and then compressed, and on decompression a compressed, TAR-encapsulated FlowFile is expected.
If false, on compression the content of the FlowFile simply gets compressed, and on decompression a simple compressed content is expected.
true is the behaviour compatible with older MiNiFi C++ versions, false is the behaviour compatible with NiFi. | -| Batch Size | 1 | | Maximum number of FlowFiles processed in a single session | +| Name | Default Value | Allowable Values | Description | +|-----------------------|-------------------------|------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Mode** | compress | compress
decompress | Indicates whether the processor should compress content or decompress content. | +| **Compression Level** | 1 | | The compression level to use; this is valid only when using GZIP compression. | +| Compression Format | use mime.type attribute | gzip
lzma
xz-lzma2
bzip2
use mime.type attribute | The compression format to use. | +| Update Filename | false | true
false | Determines if filename extension need to be updated | +| Encapsulate in TAR | true | true
false | If true, on compression the FlowFile is added to a TAR archive and then compressed, and on decompression a compressed, TAR-encapsulated FlowFile is expected.
If false, on compression the content of the FlowFile simply gets compressed, and on decompression a simple compressed content is expected.
true is the behaviour compatible with older MiNiFi C++ versions, false is the behaviour compatible with NiFi. | +| Batch Size | 1 | | Maximum number of FlowFiles processed in a single session | ### Relationships @@ -343,9 +343,9 @@ In the list below, the names of required properties appear in bold. Any other pr |--------------------------|---------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Batch Size** | 1000 | | The maximum number of entries processed in a single execution. | | **Payload Format** | Syslog | Raw
Syslog | Configures flow file content formatting. Raw: only the message. Syslog: similar to syslog or journalctl output. | -| **Include Timestamp** | true | | Include message timestamp in the 'timestamp' attribute. | +| **Include Timestamp** | true | true
false | Include message timestamp in the 'timestamp' attribute. | | **Journal Type** | System | User
System
Both | Type of journal to consume. | -| **Process Old Messages** | false | | Process events created before the first usage (schedule) of the processor instance. | +| **Process Old Messages** | false | true
false | Process events created before the first usage (schedule) of the processor instance. | | **Timestamp Format** | %x %X %Z | | Format string to use when creating the timestamp attribute or writing messages in the syslog format. ISO/ISO 8601/ISO8601 are equivalent to "%FT%T%Ez". See https://howardhinnant.github.io/date/date.html#to_stream_formatting for all flags. | ### Relationships @@ -378,7 +378,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Kafka Brokers** | localhost:9092 | | A comma-separated list of known Kafka Brokers in the format :.
**Supports Expression Language: true** | | **Topic Names** | | | The name of the Kafka Topic(s) to pull from. Multiple topic names are supported as a comma separated list.
**Supports Expression Language: true** | | **Topic Name Format** | Names | Names
Patterns | Specifies whether the Topic(s) provided are a comma separated list of names or a single regular expression. Using regular expressions does not automatically discover Kafka topics created after the processor started. | -| **Honor Transactions** | true | | Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If false, the Processor will use an "isolation level" of read_uncomitted. This means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions. If this value is true, MiNiFi will not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the consumer must wait for the producer to finish its entire transaction instead of pulling as the messages become available. | +| **Honor Transactions** | true | true
false | Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If false, the Processor will use an "isolation level" of read_uncomitted. This means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions. If this value is true, MiNiFi will not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the consumer must wait for the producer to finish its entire transaction instead of pulling as the messages become available. | | **Group ID** | | | A Group ID is used to identify consumers that are within the same consumer group. Corresponds to Kafka's 'group.id' property.
**Supports Expression Language: true** | | **Offset Reset** | latest | earliest
latest
none | Allows you to manage the condition when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that data has been deleted). Corresponds to Kafka's 'auto.offset.reset' property. | | **Key Attribute Encoding** | UTF-8 | UTF-8
Hex | FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the attribute should be encoded. | @@ -413,8 +413,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Client ID | | | MQTT client ID to use. WARNING: Must not be empty when using MQTT 3.1.0! | | **MQTT Version** | 3.x AUTO | 3.x AUTO
3.1.0
3.1.1
5.0 | The MQTT specification version when connecting to the broker. | | **Topic** | | | The topic to subscribe to. | -| Clean Session | true | | Whether to start afresh rather than remembering previous subscriptions. If true, then make broker forget subscriptions after disconnected. MQTT 3.x only. | -| Clean Start | true | | Whether to start afresh rather than remembering previous subscriptions. MQTT 5.x only. | +| Clean Session | true | true
false | Whether to start afresh rather than remembering previous subscriptions. If true, then make broker forget subscriptions after disconnected. MQTT 3.x only. | +| Clean Start | true | true
false | Whether to start afresh rather than remembering previous subscriptions. MQTT 5.x only. | | Session Expiry Interval | 0 s | | Time to delete session on broker after client is disconnected. MQTT 5.x only. | | Queue Max Message | 1000 | | Maximum number of messages allowed on the received MQTT queue | | Attribute From Content Type | | | Name of FlowFile attribute to be filled from content type of received message. MQTT 5.x only. | @@ -425,8 +425,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Keep Alive Interval | 60 sec | | Defines the maximum time interval between messages sent or received | | Last Will Topic | | | The topic to send the client's Last Will to. If the Last Will topic is not set then a Last Will will not be sent | | Last Will Message | | | The message to send as the client's Last Will. If the Last Will Message is empty, Last Will will be deleted from the broker | -| Last Will QoS | 0 | 0
1
2 | The Quality of Service (QoS) to send the last will with. | -| Last Will Retain | false | | Whether to retain the client's Last Will | +| **Last Will QoS** | 0 | 0
1
2 | The Quality of Service (QoS) to send the last will with. | +| Last Will Retain | false | true
false | Whether to retain the client's Last Will | | Last Will Content Type | | | Content type of the client's Last Will. MQTT 5.x only. | | Username | | | Username to use when connecting to the broker | | Password | | | Password to use when connecting to the broker | @@ -467,16 +467,16 @@ In the list below, the names of required properties appear in bold. Any other pr | **Max Buffer Size** | 1 MB | | The individual Event Log XMLs are rendered to a buffer. This specifies the maximum size in bytes that the buffer will be allowed to grow to. (Limiting the maximum size of an individual Event XML.) | | **Inactive Duration To Reconnect** | 10 min | | If no new event logs are processed for the specified time period, this processor will try reconnecting to recover from a state where any further messages cannot be consumed. Such situation can happen if Windows Event Log service is restarted, or ERROR_EVT_QUERY_RESULT_STALE (15011) is returned. Setting no duration, e.g. '0 ms' disables auto-reconnection. | | Identifier Match Regex | .*Sid | | Regular Expression to match Subject Identifier Fields. These will be placed into the attributes of the FlowFile | -| Apply Identifier Function | true | | If true it will resolve SIDs matched in the 'Identifier Match Regex' to the DOMAIN\USERNAME associated with that ID | -| Resolve Metadata in Attributes | true | | If true, any metadata that is resolved ( such as IDs or keyword metadata ) will be placed into attributes, otherwise it will be replaced in the XML or text output | +| Apply Identifier Function | true | true
false | If true it will resolve SIDs matched in the 'Identifier Match Regex' to the DOMAIN\USERNAME associated with that ID | +| Resolve Metadata in Attributes | true | true
false | If true, any metadata that is resolved ( such as IDs or keyword metadata ) will be placed into attributes, otherwise it will be replaced in the XML or text output | | Event Header Delimiter | | | If set, the chosen delimiter will be used in the Event output header. Otherwise, a colon followed by spaces will be used. | | Event Header | LOG_NAME=Log Name, SOURCE = Source, TIME_CREATED = Date,EVENT_RECORDID=Record ID,EVENTID = Event ID,TASK_CATEGORY = Task Category,LEVEL = Level,KEYWORDS = Keywords,USER = User,COMPUTER = Computer, EVENT_TYPE = EventType | | Comma seperated list of key/value pairs with the following keys LOG_NAME, SOURCE, TIME_CREATED,EVENT_RECORDID,EVENTID,TASK_CATEGORY,LEVEL,KEYWORDS,USER,COMPUTER, and EVENT_TYPE. Eliminating fields will remove them from the header. | | **Output Format** | Both | XML
Both
Plaintext
JSON | Set the output format type. In case 'Both' is selected the processor generates two flow files for every event captured in format XML and Plaintext | | **JSON Format** | Simple | Raw
Simple
Flattened | Set the json format type. Only applicable if Output Format is set to 'JSON' | | Batch Commit Size | 1000 | | Maximum number of Events to consume and create to Flow Files from before committing. | | State Directory | CWELState | | DEPRECATED. Only use it for state migration from the state file, supplying the legacy state directory. | -| **Process Old Events** | false | | This property defines if old events (which are created before first time server is started) should be processed. | -| Cache SID Lookups | true | | Determines whether SID to name lookups are cached in memory | +| **Process Old Events** | false | true
false | This property defines if old events (which are created before first time server is started) should be processed. | +| Cache SID Lookups | true | true
false | Determines whether SID to name lookups are cached in memory | ### Relationships @@ -529,7 +529,7 @@ In the list below, the names of required properties appear in bold. Any other pr | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.
**Supports Expression Language: true** | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).
**Supports Expression Language: true** | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.
**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | +| **Use Managed Identity Credentials** | false | true
false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | | Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | | **Delete Snapshots Option** | None | None
Include Snapshots
Delete Snapshots Only | Specifies the snapshot deletion options to be used when deleting a blob. None: Deletes the blob only. Include Snapshots: Delete the blob and its snapshots. Delete Snapshots Only: Delete only the blob's snapshots. | @@ -626,7 +626,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Supports Expression Language: true** | -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Version | | | The Version of the Object to delete
**Supports Expression Language: true** | @@ -676,7 +676,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Command Arguments | | | The arguments to supply to the executable delimited by white space. White space can be escaped by enclosing it in double-quotes. | | Working Directory | | | The directory to use as the current working directory when executing the command | | Batch Duration | 0 sec | | If the process is expected to be long-running and produce textual output, a batch duration can be specified. | -| Redirect Error Stream | false | | If true will redirect any error stream output of the process to the output stream. | +| Redirect Error Stream | false | true
false | If true will redirect any error stream output of the process to the output stream. | ### Relationships @@ -700,7 +700,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Script File | | | Path to script file to execute. Only one of Script File or Script Body may be used | | Script Body | | | Script to execute. Only one of Script File or Script Body may be used | | Module Directory | | | Comma-separated list of paths to files and/or directories which contain modules required by the script | -| **Reload on Script Change** | true | | If true and Script File property is used, then script file will be reloaded if it has changed, otherwise the first loaded version will be used at all times. | +| **Reload on Script Change** | true | true
false | If true and Script File property is used, then script file will be reloaded if it has changed, otherwise the first loaded version will be used at all times. | ### Relationships @@ -774,11 +774,11 @@ In the list below, the names of required properties appear in bold. Any other pr |----------------------------------|---------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Attribute | | | Attribute to set from content | | Size Limit | 2097152 | | Maximum number of bytes to read into the attribute. 0 for no limit. Default is 2MB. | -| Regex Mode | false | | Set this to extract parts of flowfile content using regular experssions in dynamic properties | -| Include Capture Group 0 | true | | Indicates that Capture Group 0 should be included as an attribute. Capture Group 0 represents the entirety of the regular expression match, is typically not used, and could have considerable length. | -| Enable Case-insensitive Matching | false | | Indicates that two characters match even if they are in a different case. | +| Regex Mode | false | true
false | Set this to extract parts of flowfile content using regular experssions in dynamic properties | +| Include Capture Group 0 | true | true
false | Indicates that Capture Group 0 should be included as an attribute. Capture Group 0 represents the entirety of the regular expression match, is typically not used, and could have considerable length. | +| Enable Case-insensitive Matching | false | true
false | Indicates that two characters match even if they are in a different case. | | Maximum Capture Group Length | 1024 | | Specifies the maximum number of characters a given capture group value can have. Any characters beyond the max will be truncated. | -| Enable repeating capture group | false | | f set to true, every string matching the capture groups will be extracted. Otherwise, if the Regular Expression matches more than once, only the first match will be extracted. | +| Enable repeating capture group | false | true
false | f set to true, every string matching the capture groups will be extracted. Otherwise, if the Regular Expression matches more than once, only the first match will be extracted. | ### Relationships @@ -806,7 +806,7 @@ In the list below, the names of required properties appear in bold. Any other pr | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.
**Supports Expression Language: true** | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).
**Supports Expression Language: true** | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.
**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | +| **Use Managed Identity Credentials** | false | true
false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | | Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | | Range Start | | | The byte position at which to start reading from the blob. An empty value or a value of zero will start reading at the beginning of the blob.
**Supports Expression Language: true** | | Range Length | | | The number of bytes to download from the blob, starting from the Range Start. An empty value or a value that extends beyond the end of the blob will read to the end of the blob.
**Supports Expression Language: true** | @@ -969,10 +969,10 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Supports Expression Language: true** | -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Version | | | The Version of the Object to download
**Supports Expression Language: true** | -| **Requester Pays** | false | | If true, indicates that the requester consents to pay any charges associated with retrieving objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. | +| **Requester Pays** | false | true
false | If true, indicates that the requester consents to pay any charges associated with retrieving objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. | ### Relationships @@ -1000,11 +1000,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Password | | | Password for the user account
**Supports Expression Language: true** | | Private Key Path | | | The fully qualified path to the Private Key file
**Supports Expression Language: true** | | Private Key Passphrase | | | Password for the private key
**Supports Expression Language: true** | -| **Strict Host Key Checking** | false | | Indicates whether or not strict enforcement of hosts keys should be applied | +| **Strict Host Key Checking** | false | true
false | Indicates whether or not strict enforcement of hosts keys should be applied | | Host Key File | | | If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used | | **Connection Timeout** | 30 sec | | Amount of time to wait before timing out while creating a connection | | **Data Timeout** | 30 sec | | When transferring a file between the local and remote system, this value specifies how long is allowed to elapse without any data being transferred between systems | -| **Send Keep Alive On Timeout** | true | | Indicates whether or not to send a single Keep Alive message when SSH socket times out | +| **Send Keep Alive On Timeout** | true | true
false | Indicates whether or not to send a single Keep Alive message when SSH socket times out | | Proxy Type | DIRECT | DIRECT
HTTP
SOCKS | Specifies the Proxy Configuration Controller Service to proxy network requests. If set, it supersedes proxy settings configured per component. Supported proxies: HTTP + AuthN, SOCKS + AuthN | | Proxy Host | | | The fully qualified hostname or IP address of the proxy server
**Supports Expression Language: true** | | Proxy Port | | | The port of the proxy server
**Supports Expression Language: true** | @@ -1013,9 +1013,9 @@ In the list below, the names of required properties appear in bold. Any other pr | **Remote File** | | | The fully qualified filename on the remote system
**Supports Expression Language: true** | | **Completion Strategy** | None | None
Move File
Delete File | Specifies what to do with the original file on the server once it has been pulled into NiFi. If the Completion Strategy fails, a warning will be logged but the data will still be transferred. | | Move Destination Directory | | | The directory on the remote server to move the original file to once it has been ingested into NiFi. This property is ignored unless the Completion Strategy is set to 'Move File'. The specified directory must already exist on the remote system if 'Create Directory' is disabled, or the rename will fail.
**Supports Expression Language: true** | -| **Create Directory** | false | | Specifies whether or not the remote directory should be created if it does not exist. | -| Disable Directory Listing | false | | Control how 'Move Destination Directory' is created when 'Completion Strategy' is 'Move File' and 'Create Directory' is enabled. If set to 'true', directory listing is not performed prior to create missing directories. By default, this processor executes a directory listing command to see target directory existence before creating missing directories. However, there are situations that you might need to disable the directory listing such as the following. Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, then an error is returned because the directory already exists. | -| **Use Compression** | false | | Indicates whether or not ZLIB compression should be used when transferring files | +| **Create Directory** | false | true
false | Specifies whether or not the remote directory should be created if it does not exist. | +| Disable Directory Listing | false | true
false | Control how 'Move Destination Directory' is created when 'Completion Strategy' is 'Move File' and 'Create Directory' is enabled. If set to 'true', directory listing is not performed prior to create missing directories. By default, this processor executes a directory listing command to see target directory existence before creating missing directories. However, there are situations that you might need to disable the directory listing such as the following. Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, then an error is returned because the directory already exists. | +| **Use Compression** | false | true
false | Indicates whether or not ZLIB compression should be used when transferring files | ### Relationships @@ -1063,7 +1063,7 @@ In the list below, the names of required properties appear in bold. Any other pr | File Size | 1 kB | | The size of the file that will be used | | Batch Size | 1 | | The number of FlowFiles to be transferred in each invocation | | Data Format | Binary | Text
Binary | Specifies whether the data should be Text or Binary | -| Unique FlowFiles | true | | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles | +| Unique FlowFiles | true | true
false | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles | | Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | ### Relationships @@ -1106,13 +1106,13 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Input Directory** | | | The input directory from which to pull files
**Supports Expression Language: true** | -| Recurse Subdirectories | true | | Indicates whether or not to pull files from subdirectories | -| Keep Source File | false | | If true, the file is not deleted after it has been copied to the Content Repository | +| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | +| Keep Source File | false | true
false | If true, the file is not deleted after it has been copied to the Content Repository | | Minimum File Age | 0 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | | Maximum File Age | 0 sec | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | | Minimum File Size | 0 B | | The minimum size that a file can be in order to be pulled | | Maximum File Size | 0 B | | The maximum size that a file can be in order to be pulled | -| Ignore Hidden Files | true | | Indicates whether or not hidden files should be ignored | +| Ignore Hidden Files | true | true
false | Indicates whether or not hidden files should be ignored | | Polling Interval | 0 sec | | Indicates how long to wait before performing a directory listing | | Batch Size | 10 | | The maximum number of files to pull in each iteration | | File Filter | .* | | Only files whose names match the given regular expression will be picked up | @@ -1202,7 +1202,6 @@ In the list below, the names of required properties appear in bold. Any other pr | source.endpoint | success, partial | The address of the source endpoint the message came from | - ## GetUSBCamera ### Description @@ -1271,8 +1270,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Remote URL | | | Remote URL which will be connected to, including scheme, host, port, path.
**Supports Expression Language: true** | | Connection Timeout | 5 s | | Max wait time for connection to remote service | | Read Timeout | 15 s | | Max wait time for response from remote service | -| Include Date Header | true | | Include an RFC-2616 Date header in the request. | -| Follow Redirects | true | | Follow HTTP redirects issued by remote server. | +| Include Date Header | true | true
false | Include an RFC-2616 Date header in the request. | +| Follow Redirects | true | true
false | Follow HTTP redirects issued by remote server. | | Attributes to Send | | | Regular expression that defines which attributes to send as HTTP headers in the request. If not defined, no attributes are sent as headers. | | SSL Context Service | | | The SSL Context Service used to provide client certificate information for TLS/SSL (https) connections. | | Proxy Host | | | The fully qualified hostname or IP address of the proxy server | @@ -1280,13 +1279,13 @@ In the list below, the names of required properties appear in bold. Any other pr | invokehttp-proxy-username | | | Username to set when authenticating against proxy | | invokehttp-proxy-password | | | Password to set when authenticating against proxy | | Content-type | application/octet-stream | | The Content-Type to specify for when content is being transmitted through a PUT, POST or PATCH. In the case of an empty value after evaluating an expression language expression, Content-Type defaults to | -| send-message-body | true | | DEPRECATED. Only kept for backwards compatibility, no functionality is included. | -| Send Message Body | true | | If true, sends the HTTP message body on POST/PUT/PATCH requests (default). If false, suppresses the message body and content-type header for these requests. | -| Use Chunked Encoding | false | | When POST'ing, PUT'ing or PATCH'ing content set this property to true in order to not pass the 'Content-length' header and instead send 'Transfer-Encoding' with a value of 'chunked'. This will enable the data transfer mechanism which was introduced in HTTP 1.1 to pass data of unknown lengths in chunks. | -| Disable Peer Verification | false | | Disables peer verification for the SSL session | +| send-message-body | true | true
false | DEPRECATED. Only kept for backwards compatibility, no functionality is included. | +| Send Message Body | true | true
false | If true, sends the HTTP message body on POST/PUT/PATCH requests (default). If false, suppresses the message body and content-type header for these requests. | +| Use Chunked Encoding | false | true
false | When POST'ing, PUT'ing or PATCH'ing content set this property to true in order to not pass the 'Content-length' header and instead send 'Transfer-Encoding' with a value of 'chunked'. This will enable the data transfer mechanism which was introduced in HTTP 1.1 to pass data of unknown lengths in chunks. | +| Disable Peer Verification | false | true
false | Disables peer verification for the SSL session | | Put Response Body in Attribute | | | If set, the response body received back will be put into an attribute of the original FlowFile instead of a separate FlowFile. The attribute key to put to is determined by evaluating value of this property. | -| Always Output Response | false | | Will force a response FlowFile to be generated and routed to the 'Response' relationship regardless of what the server status code received is | -| Penalize on "No Retry" | false | | Enabling this property will penalize FlowFiles that are routed to the "No Retry" relationship. | +| Always Output Response | false | true
false | Will force a response FlowFile to be generated and routed to the 'Response' relationship regardless of what the server status code received is | +| Penalize on "No Retry" | false | true
false | Enabling this property will penalize FlowFiles that are routed to the "No Retry" relationship. | | **Invalid HTTP Header Field Handling Strategy** | transform | fail
transform
drop | Indicates what should happen when an attribute's name is not a valid HTTP header field name. Options: transform - invalid characters are replaced, fail - flow file is transferred to failure, drop - drops invalid attributes from HTTP message | ### Relationships @@ -1328,7 +1327,7 @@ In the list below, the names of required properties appear in bold. Any other pr | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.
**Supports Expression Language: true** | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).
**Supports Expression Language: true** | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.
**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | +| **Use Managed Identity Credentials** | false | true
false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | | **Listing Strategy** | timestamps | none
timestamps | Specify how to determine new/updated entities. If 'timestamps' is selected it tracks the latest timestamp of listed entity to determine new/updated entities. If 'none' is selected it lists an entity without any tracking, the same entity will be listed each time on executing this processor. | | Prefix | | | Search prefix for listing
**Supports Expression Language: true** | @@ -1354,7 +1353,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | | **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** | | Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** | -| **Recurse Subdirectories** | true | | Indicates whether to list files from subdirectories of the directory | +| **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory | | File Filter | | | Only files whose names match the given regular expression will be listed | | Path Filter | | | When 'Recurse Subdirectories' is true, then only subdirectories whose paths match the given regular expression will be scanned | | Listing Strategy | timestamps | none
timestamps | Specify how to determine new/updated entities. If 'timestamps' is selected it tracks the latest timestamp of listed entity to determine new/updated entities. If 'none' is selected it lists an entity without any tracking, the same entity will be listed each time on executing this processor. | @@ -1411,7 +1410,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Listening Port** | 514 | | The port for Syslog communication. (Well-known ports (0-1023) require root access) | | **Protocol** | UDP | TCP
UDP | The protocol for Syslog communication. | | Max Batch Size | 500 | | The maximum number of Syslog events to process at a time. | -| Parse Messages | false | | Indicates if the processor should parse the Syslog messages. If set to false, each outgoing FlowFile will only contain the sender, protocol, and port, and no additional attributes. | +| Parse Messages | false | true
false | Indicates if the processor should parse the Syslog messages. If set to false, each outgoing FlowFile will only contain the sender, protocol, and port, and no additional attributes. | | Max Size of Message Queue | 10000 | | Maximum number of Syslog messages allowed to be buffered before processing them when the processor is triggered. If the buffer is full, the message is ignored. If set to zero the buffer is unlimited. | | SSL Context Service | | | The Controller Service to use in order to obtain an SSL Context. If this property is set, messages will be received over a secure connection. This Property is only considered if the Property has a value of "TCP". | | Client Auth | NONE | NONE
WANT
REQUIRED | The client authentication policy to use for the SSL Context. Only used if an SSL Context Service is provided. | @@ -1519,14 +1518,14 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Input Directory** | | | The input directory from which files to pull files | -| **Recurse Subdirectories** | true | | Indicates whether to list files from subdirectories of the directory | +| **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory | | File Filter | | | Only files whose names match the given regular expression will be picked up | | Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | | **Minimum File Age** | 0 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | | Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | | **Minimum File Size** | 0 B | | The minimum size that a file must be in order to be pulled | | Maximum File Size | | | The maximum size that a file can be in order to be pulled | -| **Ignore Hidden Files** | true | | Indicates whether or not hidden files should be ignored | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | ### Relationships @@ -1564,7 +1563,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Number of retries** | 6 | | How many retry attempts should be made before routing to the failure relationship. | | Endpoint Override URL | | | Overrides the default Google Cloud Storage endpoints
**Supports Expression Language: true** | | **Bucket** | | | Bucket of the object.
**Supports Expression Language: true** | -| List all versions | false | | Set this option to `true` to get all the previous versions separately. | +| List all versions | false | true
false | Set this option to `true` to get all the previous versions separately. | ### Relationships @@ -1624,14 +1623,14 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Supports Expression Language: true** | -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | | Delimiter | | | The string used to delimit directories within the bucket. Please consult the AWS documentation for the correct use of this field. | | Prefix | | | The prefix used to filter the object list. In most cases, it should end with a forward slash ('/'). | -| **Use Versions** | false | | Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned. | +| **Use Versions** | false | true
false | Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned. | | **Minimum Object Age** | 0 sec | | The minimum age that an S3 object must be in order to be considered; any object younger than this amount of time (according to last modification date) will be ignored. | -| **Write Object Tags** | false | | If set to 'true', the tags associated with the S3 object will be written as FlowFile attributes. | -| **Write User Metadata** | false | | If set to 'true', the user defined metadata associated with the S3 object will be added to FlowFile attributes/records. | -| **Requester Pays** | false | | If true, indicates that the requester consents to pay any charges associated with listing the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. Note that this setting is only used if Write User Metadata is true. | +| **Write Object Tags** | false | true
false | If set to 'true', the tags associated with the S3 object will be written as FlowFile attributes. | +| **Write User Metadata** | false | true
false | If set to 'true', the user defined metadata associated with the S3 object will be added to FlowFile attributes/records. | +| **Requester Pays** | false | true
false | If true, indicates that the requester consents to pay any charges associated with listing the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. Note that this setting is only used if Write User Metadata is true. | ### Relationships @@ -1658,11 +1657,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Password | | | Password for the user account
**Supports Expression Language: true** | | Private Key Path | | | The fully qualified path to the Private Key file
**Supports Expression Language: true** | | Private Key Passphrase | | | Password for the private key
**Supports Expression Language: true** | -| **Strict Host Key Checking** | false | | Indicates whether or not strict enforcement of hosts keys should be applied | +| **Strict Host Key Checking** | false | true
false | Indicates whether or not strict enforcement of hosts keys should be applied | | Host Key File | | | If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used | | **Connection Timeout** | 30 sec | | Amount of time to wait before timing out while creating a connection | | **Data Timeout** | 30 sec | | When transferring a file between the local and remote system, this value specifies how long is allowed to elapse without any data being transferred between systems | -| **Send Keep Alive On Timeout** | true | | Indicates whether or not to send a single Keep Alive message when SSH socket times out | +| **Send Keep Alive On Timeout** | true | true
false | Indicates whether or not to send a single Keep Alive message when SSH socket times out | | Proxy Type | DIRECT | DIRECT
HTTP
SOCKS | Specifies the Proxy Configuration Controller Service to proxy network requests. If set, it supersedes proxy settings configured per component. Supported proxies: HTTP + AuthN, SOCKS + AuthN | | Proxy Host | | | The fully qualified hostname or IP address of the proxy server
**Supports Expression Language: true** | | Proxy Port | | | The port of the proxy server
**Supports Expression Language: true** | @@ -1670,11 +1669,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Http Proxy Password | | | Http Proxy Password
**Supports Expression Language: true** | | **Listing Strategy** | Tracking Timestamps | Tracking Timestamps
Tracking Entities | Specify how to determine new/updated entities. See each strategy descriptions for detail. | | Remote Path | | | The fully qualified filename on the remote system
**Supports Expression Language: true** | -| **Search Recursively** | false | | If true, will pull files from arbitrarily nested subdirectories; otherwise, will not traverse subdirectories | -| **Follow symlink** | false | | If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories | +| **Search Recursively** | false | true
false | If true, will pull files from arbitrarily nested subdirectories; otherwise, will not traverse subdirectories | +| **Follow symlink** | false | true
false | If true, will pull even symbolic files and also nested symbolic subdirectories; otherwise, will not read symbolic files and will not traverse symbolic link subdirectories | | File Filter Regex | | | Provides a Java Regular Expression for filtering Filenames; if a filter is supplied, only files whose names match that Regular Expression will be fetched | | Path Filter Regex | | | When Search Recursively is true, then only subdirectories whose path matches the given Regular Expression will be scanned | -| **Ignore Dotted Files** | true | | If true, files whose names begin with a dot (".") will be ignored | +| **Ignore Dotted Files** | true | true
false | If true, files whose names begin with a dot (".") will be ignored | | **Target System Timestamp Precision** | Auto Detect | Auto Detect
Milliseconds
Seconds
Minutes | Specify timestamp precision at the target system. Since this processor uses timestamp of entities to decide which should be listed, it is crucial to use the right timestamp precision. | | Entity Tracking Time Window | | | Specify how long this processor should track already-listed entities. 'Tracking Entities' strategy can pick any entity whose timestamp is inside the specified time window. For example, if set to '30 minutes', any entity having timestamp in recent 30 minutes will be the listing target when this processor runs. A listed entity is considered 'new/updated' and a FlowFile is emitted if one of following condition meets: 1. does not exist in the already-listed entities, 2. has newer timestamp than the cached entity, 3. has different size than the cached entity. If a cached entity's timestamp becomes older than specified time window, that entity will be removed from the cached already-listed entities. Used by 'Tracking Entities' strategy. | | Entity Tracking Initial Listing Target | All Available | Tracking Time Window
All Available | Specify how initial listing should be handled. Used by 'Tracking Entities' strategy. | @@ -1705,8 +1704,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Log Level | | info
trace
error
warn
debug | The Log Level to use when logging the Attributes | | Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | -| Log Payload | false | | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | -| Hexencode Payload | false | | If true, the FlowFile's payload will be logged in a hexencoded format | +| Log Payload | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | +| Hexencode Payload | false | true
false | If true, the FlowFile's payload will be logged in a hexencoded format | | Maximum Payload Line Length | 0 | | The logged payload will be broken into lines this long. 0 means no newlines will be added. | | Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | | FlowFiles To Log | 1 | | Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously. | @@ -1767,7 +1766,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Merge Format | Binary Concatenation | Binary Concatenation
TAR
ZIP
FlowFile Stream, v3 | Merge Format | | Correlation Attribute Name | | | Correlation Attribute Name | | Delimiter Strategy | Filename | Filename
Text | Determines if Header, Footer, and Demarcator should point to files | -| Keep Path | false | | If using the Zip or Tar Merge Format, specifies whether or not the FlowFiles' paths should be included in their entry | +| Keep Path | false | true
false | If using the Zip or Tar Merge Format, specifies whether or not the FlowFiles' paths should be included in their entry | | Header File | | | Filename specifying the header to use | | Footer File | | | Filename specifying the footer to use | | Demarcator File | | | Filename specifying the demarcator to use | @@ -2028,7 +2027,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Kafka Key | | | The key to use for the message. If not specified, the UUID of the flow file is used as the message key.
**Supports Expression Language: true** | | Message Key Field | | | DEPRECATED, does not work -- use Kafka Key instead | | Debug contexts | | | A comma-separated list of debug contexts to enable.Including: generic, broker, topic, metadata, feature, queue, msg, protocol, cgrp, security, fetch, interceptor, plugin, consumer, admin, eos, all | -| Fail empty flow files | true | | Keep backwards compatibility with <=0.7.0 bug which caused flow files with empty content to not be published to Kafka and forwarded to failure. The old behavior is deprecated. Use connections to drop empty flow files! | +| Fail empty flow files | true | true
false | Keep backwards compatibility with <=0.7.0 bug which caused flow files with empty content to not be published to Kafka and forwarded to failure. The old behavior is deprecated. Use connections to drop empty flow files! | ### Relationships @@ -2052,9 +2051,9 @@ In the list below, the names of required properties appear in bold. Any other pr |-------------------------|---------------|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| | **Broker URI** | | | The URI to use to connect to the MQTT broker | | Client ID | | | MQTT client ID to use. WARNING: Must not be empty when using MQTT 3.1.0! | -| MQTT Version | 3.x AUTO | 3.x AUTO
3.1.0
3.1.1
5.0 | The MQTT specification version when connecting to the broker. | +| **MQTT Version** | 3.x AUTO | 3.x AUTO
3.1.0
3.1.1
5.0 | The MQTT specification version when connecting to the broker. | | **Topic** | | | The topic to publish to.
**Supports Expression Language: true** | -| Retain | false | | Retain published message in broker | +| Retain | false | true
false | Retain published message in broker | | Message Expiry Interval | | | Time while message is valid and will be forwarded by broker. MQTT 5.x only. | | Content Type | | | Content type of the message. MQTT 5.x only.
**Supports Expression Language: true** | | **Quality of Service** | 0 | 0
1
2 | The Quality of Service (QoS) of messages. | @@ -2063,7 +2062,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Last Will Topic | | | The topic to send the client's Last Will to. If the Last Will topic is not set then a Last Will will not be sent | | Last Will Message | | | The message to send as the client's Last Will. If the Last Will Message is empty, Last Will will be deleted from the broker | | **Last Will QoS** | 0 | 0
1
2 | The Quality of Service (QoS) to send the last will with. | -| Last Will Retain | false | | Whether to retain the client's Last Will | +| Last Will Retain | false | true
false | Whether to retain the client's Last Will | | Last Will Content Type | | | Content type of the client's Last Will. MQTT 5.x only. | | Username | | | Username to use when connecting to the broker | | Password | | | Password to use when connecting to the broker | @@ -2100,9 +2099,9 @@ In the list below, the names of required properties appear in bold. Any other pr | SAS Token | | | Shared Access Signature token. Specify either SAS Token (recommended) or Storage Account Key together with Storage Account Name if Managed Identity is not used.
**Supports Expression Language: true** | | Common Storage Account Endpoint Suffix | | | Storage accounts in public Azure always use a common FQDN suffix. Override this endpoint suffix with a different suffix in certain circumstances (like Azure Stack or non-public Azure regions).
**Supports Expression Language: true** | | Connection String | | | Connection string used to connect to Azure Storage service. This overrides all other set credential properties if Managed Identity is not used.
**Supports Expression Language: true** | -| **Use Managed Identity Credentials** | false | | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | +| **Use Managed Identity Credentials** | false | true
false | If true Managed Identity credentials will be used together with the Storage Account Name for authentication. | | Blob | | | The filename of the blob. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | -| **Create Container** | false | | Specifies whether to check if the container exists and to automatically create it if it does not. Permission to list containers is required. If false, this check is not made, but the Put operation will fail if the container does not exist. | +| **Create Container** | false | true
false | Specifies whether to check if the container exists and to automatically create it if it does not. Permission to list containers is required. If false, this check is not made, but the Put operation will fail if the container does not exist. | ### Relationships @@ -2187,7 +2186,7 @@ In the list below, the names of required properties appear in bold. Any other pr | CRC32C Checksum | | | CRC32C Checksum (encoded in Base64, big-Endian order) of the file for server-side validation.
**Supports Expression Language: true** | | Server Side Encryption Key | | | An AES256 Encryption Key (encoded in base64) for server-side encryption of the object.
**Supports Expression Language: true** | | Object ACL | | authenticatedRead
bucketOwnerFullControl
bucketOwnerRead
private
projectPrivate
publicRead
publicReadWrite | Access Control to be attached to the object uploaded. Not providing this will revert to bucket defaults. | -| Overwrite Object | true | | If false, the upload to GCS will succeed only if the object does not exist. | +| Overwrite Object | true | true
false | If false, the upload to GCS will succeed only if the object does not exist. | ### Relationships @@ -2286,7 +2285,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Supports Expression Language: true** | -| **Use Default Credentials** | false | | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Content Type | application/octet-stream | | Sets the Content-Type HTTP header indicating the type of content stored in the associated object. The value of this header is a standard MIME type. If no content type is provided the default content type "application/octet-stream" will be used.
**Supports Expression Language: true** | | **Storage Class** | Standard | Standard
ReducedRedundancy
StandardIA
OnezoneIA
IntelligentTiering
Glacier
DeepArchive | AWS S3 Storage Class | @@ -2296,7 +2295,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Read ACL User List | | | A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to read the Access Control List for an object.
**Supports Expression Language: true** | | Write ACL User List | | | A comma-separated list of Amazon User ID's or E-mail addresses that specifies who should have permissions to change the Access Control List for an object.
**Supports Expression Language: true** | | Canned ACL | | | Amazon Canned ACL for an object. Allowed values: BucketOwnerFullControl, BucketOwnerRead, AuthenticatedRead, PublicReadWrite, PublicRead, Private, AwsExecRead; will be ignored if any other ACL/permission property is specified.
**Supports Expression Language: true** | -| **Use Path Style Access** | false | | Path-style access can be enforced by setting this property to true. Set it to true if your endpoint does not support virtual-hosted-style requests, only path-style requests. | +| **Use Path Style Access** | false | true
false | Path-style access can be enforced by setting this property to true. Set it to true if your endpoint does not support virtual-hosted-style requests, only path-style requests. | | **Multipart Threshold** | 5 GB | | Specifies the file size threshold for switch from the PutS3Object API to the PutS3MultipartUpload API. Flow files bigger than this limit will be sent using the multipart process. The valid range is 5MB to 5GB. | | **Multipart Part Size** | 5 GB | | Specifies the part size for use when the PutS3Multipart Upload API is used. Flow files will be broken into chunks of this size for the upload process, but the last part sent can be smaller since it is not padded. The valid range is 5MB to 5GB. | | **Multipart Upload AgeOff Interval** | 60 min | | Specifies the interval at which existing multipart uploads in AWS S3 will be evaluated for ageoff. When processor is triggered it will initiate the ageoff evaluation if this interval has been exceeded. | @@ -2328,29 +2327,29 @@ In the list below, the names of required properties appear in bold. Any other pr | Password | | | Password for the user account
**Supports Expression Language: true** | | Private Key Path | | | The fully qualified path to the Private Key file
**Supports Expression Language: true** | | Private Key Passphrase | | | Password for the private key
**Supports Expression Language: true** | -| **Strict Host Key Checking** | false | | Indicates whether or not strict enforcement of hosts keys should be applied | +| **Strict Host Key Checking** | false | true
false | Indicates whether or not strict enforcement of hosts keys should be applied | | Host Key File | | | If supplied, the given file will be used as the Host Key; otherwise, no use host key file will be used | | **Connection Timeout** | 30 sec | | Amount of time to wait before timing out while creating a connection | | **Data Timeout** | 30 sec | | When transferring a file between the local and remote system, this value specifies how long is allowed to elapse without any data being transferred between systems | -| **Send Keep Alive On Timeout** | true | | Indicates whether or not to send a single Keep Alive message when SSH socket times out | +| **Send Keep Alive On Timeout** | true | true
false | Indicates whether or not to send a single Keep Alive message when SSH socket times out | | Proxy Type | DIRECT | DIRECT
HTTP
SOCKS | Specifies the Proxy Configuration Controller Service to proxy network requests. If set, it supersedes proxy settings configured per component. Supported proxies: HTTP + AuthN, SOCKS + AuthN | | Proxy Host | | | The fully qualified hostname or IP address of the proxy server
**Supports Expression Language: true** | | Proxy Port | | | The port of the proxy server
**Supports Expression Language: true** | | Http Proxy Username | | | Http Proxy Username
**Supports Expression Language: true** | | Http Proxy Password | | | Http Proxy Password
**Supports Expression Language: true** | | Remote Path | | | The path on the remote system from which to pull or push files
**Supports Expression Language: true** | -| **Create Directory** | false | | Specifies whether or not the remote directory should be created if it does not exist. | -| Disable Directory Listing | false | | If set to 'true', directory listing is not performed prior to create missing directories. By default, this processor executes a directory listing command to see target directory existence before creating missing directories. However, there are situations that you might need to disable the directory listing such as the following. Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, then an error is returned because the directory already exists. | +| **Create Directory** | false | true
false | Specifies whether or not the remote directory should be created if it does not exist. | +| Disable Directory Listing | false | true
false | If set to 'true', directory listing is not performed prior to create missing directories. By default, this processor executes a directory listing command to see target directory existence before creating missing directories. However, there are situations that you might need to disable the directory listing such as the following. Directory listing might fail with some permission setups (e.g. chmod 100) on a directory. Also, if any other SFTP client created the directory after this processor performed a listing and before a directory creation request by this processor is finished, then an error is returned because the directory already exists. | | **Batch Size** | 500 | | The maximum number of FlowFiles to send in a single connection | | **Conflict Resolution** | NONE | REPLACE
IGNORE
RENAME
REJECT
FAIL
NONE | Determines how to handle the problem of filename collisions | -| Reject Zero-Byte Files | true | | Determines whether or not Zero-byte files should be rejected without attempting to transfer | -| Dot Rename | true | | If true, then the filename of the sent file is prepended with a "." and then renamed back to the original once the file is completely sent. Otherwise, there is no rename. This property is ignored if the Temporary Filename property is set. | +| Reject Zero-Byte Files | true | true
false | Determines whether or not Zero-byte files should be rejected without attempting to transfer | +| Dot Rename | true | true
false | If true, then the filename of the sent file is prepended with a "." and then renamed back to the original once the file is completely sent. Otherwise, there is no rename. This property is ignored if the Temporary Filename property is set. | | Temporary Filename | | | If set, the filename of the sent file will be equal to the value specified during the transfer and after successful completion will be renamed to the original filename. If this value is set, the Dot Rename property is ignored.
**Supports Expression Language: true** | | Last Modified Time | | | The lastModifiedTime to assign to the file after transferring it. If not set, the lastModifiedTime will not be changed. Format must be yyyy-MM-dd'T'HH:mm:ssZ. You may also use expression language such as ${file.lastModifiedTime}. If the value is invalid, the processor will not be invalid but will fail to change lastModifiedTime of the file.
**Supports Expression Language: true** | | Permissions | | | The permissions to assign to the file after transferring it. Format must be either UNIX rwxrwxrwx with a - in place of denied permissions (e.g. rw-r--r--) or an octal number (e.g. 644). If not set, the permissions will not be changed. You may also use expression language such as ${file.permissions}. If the value is invalid, the processor will not be invalid but will fail to change permissions of the file.
**Supports Expression Language: true** | | Remote Owner | | | Integer value representing the User ID to set on the file after transferring it. If not set, the owner will not be set. You may also use expression language such as ${file.owner}. If the value is invalid, the processor will not be invalid but will fail to change the owner of the file.
**Supports Expression Language: true** | | Remote Group | | | Integer value representing the Group ID to set on the file after transferring it. If not set, the group will not be set. You may also use expression language such as ${file.group}. If the value is invalid, the processor will not be invalid but will fail to change the group of the file.
**Supports Expression Language: true** | -| **Use Compression** | false | | Indicates whether or not ZLIB compression should be used when transferring files | +| **Use Compression** | false | true
false | Indicates whether or not ZLIB compression should be used when transferring files | ### Relationships @@ -2446,7 +2445,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Port** | | | The port or service on the destination.
**Supports Expression Language: true** | | **Idle Connection Expiration** | 15 seconds | | The amount of time a connection should be held open without being used before closing the connection. A value of 0 seconds will disable this feature.
**Supports Expression Language: true** | | **Timeout** | 15 seconds | | The timeout for connecting to and communicating with the destination.
**Supports Expression Language: true** | -| **Connection Per FlowFile** | false | | Specifies whether to send each FlowFile's content on an individual connection. | +| **Connection Per FlowFile** | false | true
false | Specifies whether to send each FlowFile's content on an individual connection. | | Outgoing Message Delimiter | | | Specifies the delimiter to use when sending messages out over the same TCP stream. The delimiter is appended to each FlowFile message that is transmitted over the stream so that the receiver can determine when one message ends and the next message begins. Users should ensure that the FlowFile content does not contain the delimiter character to avoid errors.
**Supports Expression Language: true** | | SSL Context Service | | | The Controller Service to use in order to obtain an SSL Context. If this property is set, messages will be sent over a secure connection. | | Max Size of Socket Send Buffer | | | The maximum size of the socket send buffer that should be used. This is a suggestion to the Operating System to indicate how big the socket buffer should be. | @@ -2608,8 +2607,8 @@ In the list below, the names of required properties appear in bold. Any other pr |-------------------------------------|------------------|-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Retry Attribute** | flowfile.retries | | The name of the attribute that contains the current retry count for the FlowFile.WARNING: If the name matches an attribute already on the FlowFile that does not contain a numerical value, the processor will either overwrite that attribute with '1' or fail based on configuration.
**Supports Expression Language: true** | | **Maximum Retries** | 3 | | The maximum number of times a FlowFile can be retried before being passed to the 'retries_exceeded' relationship.
**Supports Expression Language: true** | -| **Penalize Retries** | true | | If set to 'true', this Processor will penalize input FlowFiles before passing them to the 'retry' relationship. This does not apply to the 'retries_exceeded' relationship. | -| **Fail on Non-numerical Overwrite** | false | | If the FlowFile already has the attribute defined in 'Retry Attribute' that is *not* a number, fail the FlowFile instead of resetting that value to '1' | +| **Penalize Retries** | true | true
false | If set to 'true', this Processor will penalize input FlowFiles before passing them to the 'retry' relationship. This does not apply to the 'retries_exceeded' relationship. | +| **Fail on Non-numerical Overwrite** | false | true
false | If the FlowFile already has the attribute defined in 'Retry Attribute' that is *not* a number, fail the FlowFile instead of resetting that value to '1' | | **Reuse Mode** | Fail on Reuse | Fail on Reuse
Warn on Reuse
Reset Reuse | Defines how the Processor behaves if the retry FlowFile has a different retry UUID than the instance that received the FlowFile. This generally means that the attribute was not reset after being successfully retried by a previous instance of this processor. | ### Dynamic Properties @@ -2672,8 +2671,8 @@ In the list below, the names of required properties appear in bold. Any other pr |----------------------------------------|-----------------|-----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Routing Strategy** | Dynamic Routing | Dynamic Routing
Route On All
Route On Any | Specifies how to determine which Relationship(s) to use when evaluating the segments of incoming text against the 'Matching Strategy' and user-defined properties. 'Dynamic Routing' routes to all the matching dynamic relationships (or 'unmatched' if none matches). 'Route On All' routes to 'matched' iff all dynamic relationships match. 'Route On Any' routes to 'matched' iff any of the dynamic relationships match. | | **Matching Strategy** | | Starts With
Ends With
Contains
Equals
Matches Regex
Contains Regex
Satisfies Expression | Specifies how to evaluate each segment of incoming text against the user-defined properties. Possible values are: 'Starts With', 'Ends With', 'Contains', 'Equals', 'Matches Regex', 'Contains Regex', 'Satisfies Expression'. | -| **Ignore Leading/Trailing Whitespace** | true | | Indicates whether or not the whitespace at the beginning and end should be ignored when evaluating a segment. | -| **Ignore Case** | false | | If true, capitalization will not be taken into account when comparing values. E.g., matching against 'HELLO' or 'hello' will have the same result. This property is ignored if the 'Matching Strategy' is set to 'Satisfies Expression'. | +| **Ignore Leading/Trailing Whitespace** | true | true
false | Indicates whether or not the whitespace at the beginning and end should be ignored when evaluating a segment. | +| **Ignore Case** | false | true
false | If true, capitalization will not be taken into account when comparing values. E.g., matching against 'HELLO' or 'hello' will have the same result. This property is ignored if the 'Matching Strategy' is set to 'Satisfies Expression'. | | Grouping Regular Expression | | | Specifies a Regular Expression to evaluate against each segment to determine which Group it should be placed in. The Regular Expression must have at least one Capturing Group that defines the segment's Group. If multiple Capturing Groups exist in the Regular Expression, the values from all Capturing Groups will be joined together with ", ". Two segments will not be placed into the same FlowFile unless they both have the same value for the Group (or neither matches the Regular Expression). For example, to group together all lines in a CSV File by the first column, we can set this value to "(.*?),.*" (and use "Per Line" segmentation). Two segments that have the same Group but different Relationships will never be placed into the same FlowFile. | | Grouping Fallback Value | | | If the 'Grouping Regular Expression' is specified and the matching fails, this value will be considered the group of the segment. | | **Segmentation Strategy** | Per Line | Full Text
Per Line | Specifies what portions of the FlowFile content constitutes a single segment to be processed. 'Full Text' considers the whole content as a single segment, 'Per Line' considers each line of the content as a separate segment | @@ -2717,7 +2716,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Subscriptions Base Path** | /wsman/subscriptions | | The URI path that will be used as the base for endpoints serving individual subscriptions. | | **SSL Certificate** | | | File containing PEM-formatted file including TLS/SSL certificate and key. The root CA of the certificate must be the CA set in SSL Certificate Authority. | | **SSL Certificate Authority** | | | File containing the PEM-formatted CA that is the root CA for both this server's certificate and the event source clients' certificates. | -| SSL Verify Peer | true | | Whether or not to verify the client's certificate | +| SSL Verify Peer | true | true
false | Whether or not to verify the client's certificate | | **XPath XML Query** | @@ -2778,7 +2777,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Input Delimiter | \n | | Specifies the character that should be used for delimiting the data being tailedfrom the incoming file. If none is specified, data will be ingested as it becomes available. | | **tail-mode** | Single file | Single file
Multiple file | Specifies the tail file mode. In 'Single file' mode only a single file will be watched. In 'Multiple file' mode a regex may be used. Note that in multiple file mode we will still continue to watch for rollover on the initial set of watched files. The Regex used to locate multiple files will be run during the schedule phrase. Note that if rotated files are matched by the regex, those files will be tailed. | | tail-base-directory | | | Base directory used to look for files to tail. This property is required when using Multiple file mode. Can contain expression language placeholders if Attribute Provider Service is set.
**Supports Expression Language: true** | -| Recursive lookup | false | | When using Multiple file mode, this property determines whether files are tailed in child directories of the Base Directory or not. | +| Recursive lookup | false | true
false | When using Multiple file mode, this property determines whether files are tailed in child directories of the Base Directory or not. | | Lookup frequency | 10 min | | When using Multiple file mode, this property specifies the minimum duration the processor will wait between looking for new files to tail in the Base Directory. | | Rolling Filename Pattern | ${filename}.* | | If the file to tail "rolls over" as would be the case with log files, this filename pattern will be used to identify files that have rolled over so MiNiFi can read the remaining of the rolled-over file and then continue with the new log file. This pattern supports the wildcard characters * and ?, it also supports the notation ${filename} to specify a pattern based on the name of the file (without extension), and will assume that the files that have rolled over live in the same directory as the file being tailed. | | **Initial Start Position** | Beginning of File | Beginning of Time
Beginning of File
Current Time | When the Processor first begins to tail data, this property specifies where the Processor should begin reading data. Once data has been ingested from a file, the Processor will continue from the last point from which it has received data.
Beginning of Time: Start with the oldest data that matches the Rolling Filename Pattern and then begin reading from the File to Tail.
Beginning of File: Start with the beginning of the File to Tail. Do not ingest any data that has already been rolled over.
Current Time: Start with the data at the end of the File to Tail. Do not ingest any data that has already been rolled over or any data in the File to Tail that has already been written. | diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index 570375af8c..bd1c6a8d9b 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -50,11 +50,16 @@ std::string formatName(std::string_view name_view, bool is_required) { } } -std::string formatAllowableValues(const std::vector& values) { - return values - | ranges::views::transform([](const auto& value) { return value.to_string(); }) - | ranges::views::join(std::string_view{"
"}) - | ranges::to(); +std::string formatAllowedValues(const minifi::core::Property& property) { + if (&property.getValidator() == &minifi::core::StandardPropertyTypes::BOOLEAN_TYPE) { + return "true
false"; + } else { + const auto allowed_values = property.getAllowedValues(); + return allowed_values + | ranges::views::transform([](const auto &value) { return value.to_string(); }) + | ranges::views::join(std::string_view{"
"}) + | ranges::to(); + } } std::string formatDescription(std::string_view description_view, bool supports_expression_language = false) { @@ -104,7 +109,7 @@ void AgentDocs::generate(const std::filesystem::path& docsdir, std::ostream &gen properties.addRow({ formatName(prop.getName(), prop.getRequired()), prop.getDefaultValue().to_string(), - formatAllowableValues(prop.getAllowedValues()), + formatAllowedValues(prop), formatDescription(prop.getDescription(), prop.supportsExpressionLanguage())}); } outfile << properties.toString() << '\n';