MINIFICPP-1825 Create Properties at compile time

Also Relationships, Output Attributes and Dynamic Properties. This only applies to Processor and Controller Service metadata used in generating the manifest and PROCESSORS.md. I did not change Properties and Relationships used later on during runtime, as those are not always const and sometimes they need to own their own strings. There are a few changes in the generated PROCESSORS.md, mainly because the list of allowed values for a Property is no longer alphabetized; now the allowed values are listed in the order they appear in either the property definition or in the corresponding smart enum. Signed-off-by: Gabor Gyimesi <[email protected]> This closes #1589
apache · Jul 19, 2023 · 0cf3c8f · 0cf3c8f
1 parent 26d68d9
commit 0cf3c8f
Show file tree

Hide file tree

Showing 429 changed files with 8,841 additions and 10,071 deletions.
diff --git a/LICENSE b/LICENSE
@@ -3368,3 +3368,27 @@ This product bundles 'CodeCoverage.cmake' which is  available under a BSD 3-Clau
     ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This product contains code snippets from https://bitwizeshift.github.io, which is available under an MIT license.
+
+    The MIT License (MIT)
+
+    Copyright (c) 2020 Matthew Rodusek
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
diff --git a/NOTICE b/NOTICE
@@ -74,6 +74,7 @@ This software includes third party software subject to the following copyrights:
 - Zstandard - Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
 - LZ4 Library - Copyright (c) 2011-2020, Yann Collet
 - OpenSSL - Copyright (c) 1998-2022 The OpenSSL Project, Copyright (c) 1995-1998 Eric A. Young, Tim J. Hudson. All rights reserved.
+- bitwizeshift.github.io - Copyright (c) 2020 Matthew Rodusek
 
 The licenses for these third party components are included in LICENSE.txt
 

diff --git a/PROCESSORS.md b/PROCESSORS.md
diff --git a/extensions/aws/controllerservices/AWSCredentialsService.cpp b/extensions/aws/controllerservices/AWSCredentialsService.cpp
@@ -18,54 +18,27 @@
 
 #include "AWSCredentialsService.h"
 
-#include "core/PropertyBuilder.h"
 #include "core/Resource.h"
 
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace aws {
-namespace controllers {
-
-const core::Property AWSCredentialsService::UseDefaultCredentials(
-    core::PropertyBuilder::createProperty("Use Default Credentials")
-    ->withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.")
-    ->withDefaultValue<bool>(false)
-    ->isRequired(true)
-    ->build());
-
-const core::Property AWSCredentialsService::AccessKey(
-    core::PropertyBuilder::createProperty("Access Key")->withDescription("Specifies the AWS Access Key.")
-    ->build());
-
-const core::Property AWSCredentialsService::SecretKey(
-    core::PropertyBuilder::createProperty("Secret Key")
-    ->withDescription("Specifies the AWS Secret Key.")
-    ->build());
-
-const core::Property AWSCredentialsService::CredentialsFile(
-  core::PropertyBuilder::createProperty("Credentials File")
-    ->withDescription("Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey")
-    ->build());
+namespace org::apache::nifi::minifi::aws::controllers {
 
 void AWSCredentialsService::initialize() {
-  setSupportedProperties(properties());
+  setSupportedProperties(Properties);
 }
 
 void AWSCredentialsService::onEnable() {
   std::string value;
-  if (getProperty(AccessKey.getName(), value)) {
+  if (getProperty(AccessKey, value)) {
     aws_credentials_provider_.setAccessKey(value);
   }
-  if (getProperty(SecretKey.getName(), value)) {
+  if (getProperty(SecretKey, value)) {
     aws_credentials_provider_.setSecretKey(value);
   }
-  if (getProperty(CredentialsFile.getName(), value)) {
+  if (getProperty(CredentialsFile, value)) {
     aws_credentials_provider_.setCredentialsFile(value);
   }
   bool use_default_credentials = false;
-  if (getProperty(UseDefaultCredentials.getName(), use_default_credentials)) {
+  if (getProperty(UseDefaultCredentials, use_default_credentials)) {
     aws_credentials_provider_.setUseDefaultCredentials(use_default_credentials);
   }
 }
@@ -79,9 +52,4 @@ std::optional<Aws::Auth::AWSCredentials> AWSCredentialsService::getAWSCredential
 
 REGISTER_RESOURCE(AWSCredentialsService, ControllerService);
 
-}  // namespace controllers
-}  // namespace aws
-}  // namespace minifi
-}  // namespace nifi
-}  // namespace apache
-}  // namespace org
+}  // namespace org::apache::nifi::minifi::aws::controllers
diff --git a/extensions/aws/controllerservices/AWSCredentialsService.h b/extensions/aws/controllerservices/AWSCredentialsService.h
@@ -28,6 +28,9 @@
 #include "utils/AWSInitializer.h"
 #include "core/controller/ControllerService.h"
 #include "core/logging/LoggerConfiguration.h"
+#include "core/PropertyDefinition.h"
+#include "core/PropertyDefinitionBuilder.h"
+#include "core/PropertyType.h"
 #include "AWSCredentialsProvider.h"
 
 class AWSCredentialsServiceTestAccessor;
@@ -46,18 +49,28 @@ class AWSCredentialsService : public core::controller::ControllerService {
 
   EXTENSIONAPI static constexpr const char* Description = "AWS Credentials Management Service";
 
-  EXTENSIONAPI static const core::Property UseDefaultCredentials;
-  EXTENSIONAPI static const core::Property AccessKey;
-  EXTENSIONAPI static const core::Property SecretKey;
-  EXTENSIONAPI static const core::Property CredentialsFile;
-  static auto properties() {
-    return std::array{
+  EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials")
+      .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.")
+      .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE)
+      .withDefaultValue("false")
+      .isRequired(true)
+      .build();
+  EXTENSIONAPI static constexpr auto AccessKey = core::PropertyDefinitionBuilder<>::createProperty("Access Key")
+      .withDescription("Specifies the AWS Access Key.")
+      .build();
+  EXTENSIONAPI static constexpr auto SecretKey = core::PropertyDefinitionBuilder<>::createProperty("Secret Key")
+      .withDescription("Specifies the AWS Secret Key.")
+      .build();
+  EXTENSIONAPI static constexpr auto CredentialsFile = core::PropertyDefinitionBuilder<>::createProperty("Credentials File")
+      .withDescription("Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey")
+      .build();
+  EXTENSIONAPI static constexpr auto Properties = std::array<core::PropertyReference, 4>{
       UseDefaultCredentials,
       AccessKey,
       SecretKey,
       CredentialsFile
-    };
-  }
+  };
+
 
   EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false;
   ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES

diff --git a/extensions/aws/processors/DeleteS3Object.cpp b/extensions/aws/processors/DeleteS3Object.cpp
@@ -27,8 +27,8 @@
 namespace org::apache::nifi::minifi::aws::processors {
 
 void DeleteS3Object::initialize() {
-  setSupportedProperties(properties());
-  setSupportedRelationships(relationships());
+  setSupportedProperties(Properties);
+  setSupportedRelationships(Relationships);
 }
 
 std::optional<aws::s3::DeleteObjectRequestParameters> DeleteS3Object::buildDeleteS3RequestParams(
@@ -81,4 +81,6 @@ void DeleteS3Object::onTrigger(const std::shared_ptr<core::ProcessContext> &cont
   }
 }
 
+REGISTER_RESOURCE(DeleteS3Object, Processor);
+
 }  // namespace org::apache::nifi::minifi::aws::processors
diff --git a/extensions/aws/processors/DeleteS3Object.h b/extensions/aws/processors/DeleteS3Object.h
@@ -23,10 +23,13 @@
 #include <string>
 #include <utility>
 
+#include "core/PropertyDefinition.h"
+#include "core/PropertyDefinitionBuilder.h"
 #include "S3Processor.h"
 #include "utils/ArrayUtils.h"
 #include "utils/GeneralUtils.h"
 
+
 template<typename T>
 class S3TestsFixture;
 
@@ -36,18 +39,23 @@ class DeleteS3Object : public S3Processor {
  public:
   EXTENSIONAPI static constexpr const char* Description = "Deletes FlowFiles on an Amazon S3 Bucket. If attempting to delete a file that does not exist, FlowFile is routed to success.";
 
-  static const core::Property ObjectKey;
-  static const core::Property Version;
-  static auto properties() {
-    return minifi::utils::array_cat(S3Processor::properties(), std::array{
+  EXTENSIONAPI static constexpr auto ObjectKey = core::PropertyDefinitionBuilder<>::createProperty("Object Key")
+    .withDescription("The key of the S3 object. If none is given the filename attribute will be used by default.")
+    .supportsExpressionLanguage(true)
+    .build();
+  EXTENSIONAPI static constexpr auto Version = core::PropertyDefinitionBuilder<>::createProperty("Version")
+    .withDescription("The Version of the Object to delete")
+    .supportsExpressionLanguage(true)
+    .build();
+  EXTENSIONAPI static constexpr auto Properties = minifi::utils::array_cat(S3Processor::Properties, std::array<core::PropertyReference, 2>{
       ObjectKey,
       Version
-    });
-  }
+  });
+
 
-  EXTENSIONAPI static const core::Relationship Success;
-  EXTENSIONAPI static const core::Relationship Failure;
-  static auto relationships() { return std::array{Success, Failure}; }
+  EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", "FlowFiles are routed to success relationship"};
+  EXTENSIONAPI static constexpr auto Failure = core::RelationshipDefinition{"failure", "FlowFiles are routed to failure relationship"};
+  EXTENSIONAPI static constexpr auto Relationships = std::array{Success, Failure};
 
   EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
   EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false;

diff --git a/extensions/aws/processors/FetchS3Object.cpp b/extensions/aws/processors/FetchS3Object.cpp
@@ -24,19 +24,20 @@
 
 #include "core/ProcessContext.h"
 #include "core/ProcessSession.h"
+#include "core/Resource.h"
 #include "utils/OptionalUtils.h"
 
 namespace org::apache::nifi::minifi::aws::processors {
 
 void FetchS3Object::initialize() {
-  setSupportedProperties(properties());
-  setSupportedRelationships(relationships());
+  setSupportedProperties(Properties);
+  setSupportedRelationships(Relationships);
 }
 
 void FetchS3Object::onSchedule(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) {
   S3Processor::onSchedule(context, sessionFactory);
 
-  context->getProperty(RequesterPays.getName(), requester_pays_);
+  context->getProperty(RequesterPays, requester_pays_);
   logger_->log_debug("FetchS3Object: RequesterPays [%s]", requester_pays_ ? "true" : "false");
 }
 
@@ -113,4 +114,6 @@ void FetchS3Object::onTrigger(const std::shared_ptr<core::ProcessContext> &conte
   }
 }
 
+REGISTER_RESOURCE(FetchS3Object, Processor);
+
 }  // namespace org::apache::nifi::minifi::aws::processors
diff --git a/extensions/aws/processors/FetchS3Object.h b/extensions/aws/processors/FetchS3Object.h
@@ -24,6 +24,9 @@
 #include <utility>
 #include <vector>
 
+#include "core/PropertyDefinition.h"
+#include "core/PropertyDefinitionBuilder.h"
+#include "core/PropertyType.h"
 #include "io/StreamPipe.h"
 #include "S3Processor.h"
 #include "utils/ArrayUtils.h"
@@ -38,20 +41,31 @@ class FetchS3Object : public S3Processor {
  public:
   EXTENSIONAPI static constexpr const char* Description = "This Processor retrieves the contents of an S3 Object and writes it to the content of a FlowFile.";
 
-  static const core::Property ObjectKey;
-  static const core::Property Version;
-  static const core::Property RequesterPays;
-  static auto properties() {
-    return minifi::utils::array_cat(S3Processor::properties(), std::array{
+  EXTENSIONAPI static constexpr auto ObjectKey = core::PropertyDefinitionBuilder<>::createProperty("Object Key")
+      .withDescription("The key of the S3 object. If none is given the filename attribute will be used by default.")
+      .supportsExpressionLanguage(true)
+      .build();
+  EXTENSIONAPI static constexpr auto Version = core::PropertyDefinitionBuilder<>::createProperty("Version")
+      .withDescription("The Version of the Object to download")
+      .supportsExpressionLanguage(true)
+      .build();
+  EXTENSIONAPI static constexpr auto RequesterPays = core::PropertyDefinitionBuilder<>::createProperty("Requester Pays")
+      .isRequired(true)
+      .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE)
+      .withDefaultValue("false")
+      .withDescription("If true, indicates that the requester consents to pay any charges associated with retrieving "
+          "objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'.")
+      .build();
+  EXTENSIONAPI static constexpr auto Properties = minifi::utils::array_cat(S3Processor::Properties, std::array<core::PropertyReference, 3>{
       ObjectKey,
       Version,
       RequesterPays
-    });
-  }
+  });
+
 
-  EXTENSIONAPI static const core::Relationship Success;
-  EXTENSIONAPI static const core::Relationship Failure;
-  static auto relationships() { return std::array{Success, Failure}; }
+  EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", "FlowFiles are routed to success relationship"};
+  EXTENSIONAPI static constexpr auto Failure = core::RelationshipDefinition{"failure", "FlowFiles are routed to failure relationship"};
+  EXTENSIONAPI static constexpr auto Relationships = std::array{Success, Failure};
 
   EXTENSIONAPI static constexpr bool SupportsDynamicProperties = true;
   EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false;

diff --git a/extensions/aws/processors/ListS3.cpp b/extensions/aws/processors/ListS3.cpp
@@ -28,8 +28,8 @@
 namespace org::apache::nifi::minifi::aws::processors {
 
 void ListS3::initialize() {
-  setSupportedProperties(properties());
-  setSupportedRelationships(relationships());
+  setSupportedProperties(Properties);
+  setSupportedRelationships(Relationships);
 }
 
 void ListS3::onSchedule(const std::shared_ptr<core::ProcessContext> &context, const std::shared_ptr<core::ProcessSessionFactory> &sessionFactory) {
@@ -51,13 +51,13 @@ void ListS3::onSchedule(const std::shared_ptr<core::ProcessContext> &context, co
   list_request_params_->setClientConfig(common_properties->proxy, common_properties->endpoint_override_url);
   list_request_params_->bucket = common_properties->bucket;
 
-  context->getProperty(Delimiter.getName(), list_request_params_->delimiter);
+  context->getProperty(Delimiter, list_request_params_->delimiter);
   logger_->log_debug("ListS3: Delimiter [%s]", list_request_params_->delimiter);
 
-  context->getProperty(Prefix.getName(), list_request_params_->prefix);
+  context->getProperty(Prefix, list_request_params_->prefix);
   logger_->log_debug("ListS3: Prefix [%s]", list_request_params_->prefix);
 
-  context->getProperty(UseVersions.getName(), list_request_params_->use_versions);
+  context->getProperty(UseVersions, list_request_params_->use_versions);
   logger_->log_debug("ListS3: UseVersions [%s]", list_request_params_->use_versions ? "true" : "false");
 
   if (auto minimum_object_age = context->getProperty<core::TimePeriodValue>(MinimumObjectAge)) {
@@ -67,13 +67,13 @@ void ListS3::onSchedule(const std::shared_ptr<core::ProcessContext> &context, co
   }
   logger_->log_debug("S3Processor: Minimum Object Age [%llud]", list_request_params_->min_object_age);
 
-  context->getProperty(WriteObjectTags.getName(), write_object_tags_);
+  context->getProperty(WriteObjectTags, write_object_tags_);
   logger_->log_debug("ListS3: WriteObjectTags [%s]", write_object_tags_ ? "true" : "false");
 
-  context->getProperty(WriteUserMetadata.getName(), write_user_metadata_);
+  context->getProperty(WriteUserMetadata, write_user_metadata_);
   logger_->log_debug("ListS3: WriteUserMetadata [%s]", write_user_metadata_ ? "true" : "false");
 
-  context->getProperty(RequesterPays.getName(), requester_pays_);
+  context->getProperty(RequesterPays, requester_pays_);
   logger_->log_debug("ListS3: RequesterPays [%s]", requester_pays_ ? "true" : "false");
 }
 
@@ -176,4 +176,6 @@ void ListS3::onTrigger(const std::shared_ptr<core::ProcessContext> &context, con
   }
 }
 
+REGISTER_RESOURCE(ListS3, Processor);
+
 }  // namespace org::apache::nifi::minifi::aws::processors