diff --git a/METRICS.md b/METRICS.md index ee73a00186..d6d63481af 100644 --- a/METRICS.md +++ b/METRICS.md @@ -108,6 +108,15 @@ An agent identifier should also be defined to identify which agent the metric is nifi.metrics.publisher.agent.identifier=Agent1 +### Configure Prometheus metrics publisher with SSL + +The communication between MiNiFi and Prometheus can be encrypted using SSL. This can be achieved by adding the SSL certificate path (a single file containing both the MiNiFi certificate and the MiNiFi SSL key) and optionally adding the root CA path if Prometheus uses a self-signed certificate, to the minifi.properties file. Here is an example with the SSL properties: + + # in minifi.properties + + nifi.metrics.publisher.PrometheusMetricsPublisher.certificate=/tmp/certs/prometheus-publisher/minifi-cpp.crt + nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate=/tmp/certs/prometheus-publisher/root-ca.pem + ## System Metrics The following section defines the currently available metrics to be published by the MiNiFi C++ agent. diff --git a/docker/test/integration/cluster/ContainerStore.py b/docker/test/integration/cluster/ContainerStore.py index b0bb0449a3..fefbcc51d8 100644 --- a/docker/test/integration/cluster/ContainerStore.py +++ b/docker/test/integration/cluster/ContainerStore.py @@ -242,6 +242,15 @@ def acquire_container(self, context, container_name: str, engine='minifi-cpp', c network=self.network, image_store=self.image_store, command=command)) + elif engine == "prometheus-ssl": + return self.containers.setdefault(container_name, + PrometheusContainer(feature_context=feature_context, + name=container_name, + vols=self.vols, + network=self.network, + image_store=self.image_store, + command=command, + ssl=True)) elif engine == "minifi-c2-server": return self.containers.setdefault(container_name, MinifiC2ServerContainer(feature_context=feature_context, @@ -313,6 +322,9 @@ def set_ssl_context_properties_in_minifi(self): def enable_prometheus_in_minifi(self): self.minifi_options.enable_prometheus = True + def enable_prometheus_with_ssl_in_minifi(self): + self.minifi_options.enable_prometheus_with_ssl = True + def enable_sql_in_minifi(self): self.minifi_options.enable_sql = True diff --git a/docker/test/integration/cluster/DockerTestCluster.py b/docker/test/integration/cluster/DockerTestCluster.py index 6d4ad4dc56..5b4575a9f1 100644 --- a/docker/test/integration/cluster/DockerTestCluster.py +++ b/docker/test/integration/cluster/DockerTestCluster.py @@ -87,6 +87,9 @@ def set_ssl_context_properties_in_minifi(self): def enable_prometheus_in_minifi(self): self.container_store.enable_prometheus_in_minifi() + def enable_prometheus_with_ssl_in_minifi(self): + self.container_store.enable_prometheus_with_ssl_in_minifi() + def enable_sql_in_minifi(self): self.container_store.enable_sql_in_minifi() diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index 3ce398f7ec..6a0bc83011 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -31,6 +31,7 @@ def __init__(self): self.enable_c2_with_ssl = False self.enable_provenance = False self.enable_prometheus = False + self.enable_prometheus_with_ssl = False self.enable_sql = False self.config_format = "json" self.use_flow_config_from_url = False @@ -118,20 +119,24 @@ def _create_properties(self): if not self.options.enable_provenance: f.write("nifi.provenance.repository.class.name=NoOpRepository\n") - if self.options.enable_prometheus or self.options.enable_log_metrics_publisher: - classes = [] - if self.options.enable_prometheus: - f.write("nifi.metrics.publisher.agent.identifier=Agent1\n") - f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.port=9936\n") - f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.metrics=RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus\n") - classes.append("PrometheusMetricsPublisher") + metrics_publisher_classes = [] + if self.options.enable_prometheus or self.options.enable_prometheus_with_ssl: + f.write("nifi.metrics.publisher.agent.identifier=Agent1\n") + f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.port=9936\n") + f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.metrics=RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus\n") + metrics_publisher_classes.append("PrometheusMetricsPublisher") - if self.options.enable_log_metrics_publisher: - f.write("nifi.metrics.publisher.LogMetricsPublisher.metrics=RepositoryMetrics\n") - f.write("nifi.metrics.publisher.LogMetricsPublisher.logging.interval=1s\n") - classes.append("LogMetricsPublisher") + if self.options.enable_prometheus_with_ssl: + f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.certificate=/tmp/resources/minifi_merged_cert.crt\n") + f.write("nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate=/tmp/resources/root_ca.crt\n") - f.write("nifi.metrics.publisher.class=" + ",".join(classes) + "\n") + if self.options.enable_log_metrics_publisher: + f.write("nifi.metrics.publisher.LogMetricsPublisher.metrics=RepositoryMetrics\n") + f.write("nifi.metrics.publisher.LogMetricsPublisher.logging.interval=1s\n") + metrics_publisher_classes.append("LogMetricsPublisher") + + if metrics_publisher_classes: + f.write("nifi.metrics.publisher.class=" + ",".join(metrics_publisher_classes) + "\n") if self.options.use_flow_config_from_url: f.write(f"nifi.c2.flow.url=http://minifi-c2-server-{self.feature_context.id}:10090/c2/config?class=minifi-test-class\n") diff --git a/docker/test/integration/cluster/containers/PrometheusContainer.py b/docker/test/integration/cluster/containers/PrometheusContainer.py index a5f1fc430d..bc1da33cb9 100644 --- a/docker/test/integration/cluster/containers/PrometheusContainer.py +++ b/docker/test/integration/cluster/containers/PrometheusContainer.py @@ -16,12 +16,42 @@ import os import tempfile import docker.types + from .Container import Container +from OpenSSL import crypto +from ssl_utils.SSL_cert_utils import make_cert_without_extended_usage class PrometheusContainer(Container): - def __init__(self, feature_context, name, vols, network, image_store, command=None): - super().__init__(feature_context, name, 'prometheus', vols, network, image_store, command) + def __init__(self, feature_context, name, vols, network, image_store, command=None, ssl=False): + engine = "prometheus-ssl" if ssl else "prometheus" + super().__init__(feature_context, name, engine, vols, network, image_store, command) + self.ssl = ssl + extra_ssl_settings = "" + if ssl: + prometheus_cert, prometheus_key = make_cert_without_extended_usage(f"prometheus-{feature_context.id}", feature_context.root_ca_cert, feature_context.root_ca_key) + + self.root_ca_file = tempfile.NamedTemporaryFile(delete=False) + self.root_ca_file.write(crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=feature_context.root_ca_cert)) + self.root_ca_file.close() + os.chmod(self.root_ca_file.name, 0o644) + + self.prometheus_cert_file = tempfile.NamedTemporaryFile(delete=False) + self.prometheus_cert_file.write(crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=prometheus_cert)) + self.prometheus_cert_file.close() + os.chmod(self.prometheus_cert_file.name, 0o644) + + self.prometheus_key_file = tempfile.NamedTemporaryFile(delete=False) + self.prometheus_key_file.write(crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=prometheus_key)) + self.prometheus_key_file.close() + os.chmod(self.prometheus_key_file.name, 0o644) + + extra_ssl_settings = """ + scheme: https + tls_config: + ca_file: /etc/prometheus/certs/root-ca.pem +""" + prometheus_yml_content = """ global: scrape_interval: 2s @@ -30,7 +60,9 @@ def __init__(self, feature_context, name, vols, network, image_store, command=No - job_name: "minifi" static_configs: - targets: ["minifi-cpp-flow-{feature_id}:9936"] -""".format(feature_id=self.feature_context.id) +{extra_ssl_settings} +""".format(feature_id=self.feature_context.id, extra_ssl_settings=extra_ssl_settings) + self.yaml_file = tempfile.NamedTemporaryFile(delete=False) self.yaml_file.write(prometheus_yml_content.encode()) self.yaml_file.close() @@ -45,15 +77,24 @@ def deploy(self): logging.info('Creating and running Prometheus docker container...') + mounts = [docker.types.Mount( + type='bind', + source=self.yaml_file.name, + target='/etc/prometheus/prometheus.yml' + )] + + if self.ssl: + mounts.append(docker.types.Mount( + type='bind', + source=self.root_ca_file.name, + target='/etc/prometheus/certs/root-ca.pem' + )) + self.client.containers.run( image="prom/prometheus:v2.35.0", detach=True, name=self.name, network=self.network.name, ports={'9090/tcp': 9090}, - mounts=[docker.types.Mount( - type='bind', - source=self.yaml_file.name, - target='/etc/prometheus/prometheus.yml' - )], + mounts=mounts, entrypoint=self.command) diff --git a/docker/test/integration/features/MiNiFi_integration_test_driver.py b/docker/test/integration/features/MiNiFi_integration_test_driver.py index cf26a1c213..66d3f54abc 100644 --- a/docker/test/integration/features/MiNiFi_integration_test_driver.py +++ b/docker/test/integration/features/MiNiFi_integration_test_driver.py @@ -22,7 +22,7 @@ from pydoc import locate -from ssl_utils.SSL_cert_utils import make_ca, make_client_cert +from ssl_utils.SSL_cert_utils import make_ca, make_cert_without_extended_usage from minifi.core.InputPort import InputPort from cluster.DockerTestCluster import DockerTestCluster @@ -53,9 +53,9 @@ def __init__(self, context, feature_id: str): self.cluster.set_directory_bindings(self.docker_directory_bindings.get_directory_bindings(self.feature_id), self.docker_directory_bindings.get_data_directories(self.feature_id)) self.root_ca_cert, self.root_ca_key = make_ca("root CA") - minifi_client_cert, minifi_client_key = make_client_cert(common_name=f"minifi-cpp-flow-{self.feature_id}", - ca_cert=self.root_ca_cert, - ca_key=self.root_ca_key) + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=f"minifi-cpp-flow-{self.feature_id}", + ca_cert=self.root_ca_cert, + ca_key=self.root_ca_key) self.put_test_resource('root_ca.crt', OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, cert=self.root_ca_cert)) @@ -67,6 +67,12 @@ def __init__(self, context, feature_id: str): OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, pkey=minifi_client_key)) + self.put_test_resource('minifi_merged_cert.crt', + OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM, + cert=minifi_client_cert) + + OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM, + pkey=minifi_client_key)) + def get_container_name_with_postfix(self, container_name: str): return self.cluster.container_store.get_container_name_with_postfix(container_name) @@ -347,6 +353,9 @@ def set_ssl_context_properties_in_minifi(self): def enable_prometheus_in_minifi(self): self.cluster.enable_prometheus_in_minifi() + def enable_prometheus_with_ssl_in_minifi(self): + self.cluster.enable_prometheus_with_ssl_in_minifi() + def enable_splunk_hec_ssl(self, container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem): self.cluster.enable_splunk_hec_ssl(container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem) diff --git a/docker/test/integration/features/prometheus.feature b/docker/test/integration/features/prometheus.feature index 3140b72da4..0b54ccd26b 100644 --- a/docker/test/integration/features/prometheus.feature +++ b/docker/test/integration/features/prometheus.feature @@ -35,6 +35,22 @@ Feature: MiNiFi can publish metrics to Prometheus server And "DeviceInfoNode" is published to the Prometheus server in less than 60 seconds And "AgentStatus" is published to the Prometheus server in less than 60 seconds + Scenario: Published metrics are scraped by Prometheus server through SSL connection + Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/input" + And a file with the content "test" is present in "/tmp/input" + And a PutFile processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFile1 processor is connected to the PutFile + And Prometheus with SSL is enabled in MiNiFi + And a Prometheus server is set up with SSL + When all instances start up + Then "RepositoryMetrics" is published to the Prometheus server in less than 60 seconds + And "QueueMetrics" is published to the Prometheus server in less than 60 seconds + And "GetFileMetrics" processor metric is published to the Prometheus server in less than 60 seconds for "GetFile1" processor + And "PutFileMetrics" processor metric is published to the Prometheus server in less than 60 seconds for "PutFile" processor + And "FlowInformation" is published to the Prometheus server in less than 60 seconds + And "DeviceInfoNode" is published to the Prometheus server in less than 60 seconds + And "AgentStatus" is published to the Prometheus server in less than 60 seconds + Scenario: Multiple GetFile metrics are reported by Prometheus Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/input" And a GetFile processor with the name "GetFile2" and the "Input Directory" property set to "/tmp/input" diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 20a7b5fa90..13ef0e0267 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -353,6 +353,11 @@ def step_impl(context): context.test.enable_log_metrics_publisher_in_minifi() +@given("Prometheus with SSL is enabled in MiNiFi") +def step_impl(context): + context.test.enable_prometheus_with_ssl_in_minifi() + + # HTTP proxy setup @given("the http proxy server is set up") @given("a http proxy server is set up accordingly") @@ -989,6 +994,11 @@ def step_impl(context): context.test.acquire_container(context=context, name="prometheus", engine="prometheus") +@given("a Prometheus server is set up with SSL") +def step_impl(context): + context.test.acquire_container(context=context, name="prometheus", engine="prometheus-ssl") + + @then("\"{metric_class}\" are published to the Prometheus server in less than {timeout_seconds:d} seconds") @then("\"{metric_class}\" is published to the Prometheus server in less than {timeout_seconds:d} seconds") def step_impl(context, metric_class, timeout_seconds): diff --git a/docker/test/integration/ssl_utils/SSL_cert_utils.py b/docker/test/integration/ssl_utils/SSL_cert_utils.py index 11921d0dc2..113071e484 100644 --- a/docker/test/integration/ssl_utils/SSL_cert_utils.py +++ b/docker/test/integration/ssl_utils/SSL_cert_utils.py @@ -132,6 +132,10 @@ def _make_cert(common_name, ca_cert, ca_key, extended_key_usage=None): if extended_key_usage: extensions.append(crypto.X509Extension(b"extendedKeyUsage", False, extended_key_usage)) + cert.add_extensions([ + crypto.X509Extension(b"subjectAltName", False, b"DNS.1:" + common_name.encode()) + ]) + cert.add_extensions(extensions) cert.set_issuer(ca_cert.get_subject()) diff --git a/extensions/prometheus/PrometheusExposerWrapper.cpp b/extensions/prometheus/PrometheusExposerWrapper.cpp index 48ee9b71f3..4d05e8eef0 100644 --- a/extensions/prometheus/PrometheusExposerWrapper.cpp +++ b/extensions/prometheus/PrometheusExposerWrapper.cpp @@ -20,9 +20,27 @@ namespace org::apache::nifi::minifi::extensions::prometheus { -PrometheusExposerWrapper::PrometheusExposerWrapper(uint32_t port) - : exposer_(std::to_string(port)) { - logger_->log_info("Started Prometheus metrics publisher on port %" PRIu32, port); +PrometheusExposerWrapper::PrometheusExposerWrapper(const PrometheusExposerConfig& config) + : exposer_(parseExposerConfig(config)) { + logger_->log_info("Started Prometheus metrics publisher on port %" PRIu32 "%s", config.port, config.certificate ? " with TLS enabled" : ""); +} + +std::vector PrometheusExposerWrapper::parseExposerConfig(const PrometheusExposerConfig& config) { + std::vector result; + result.push_back("listening_ports"); + if (config.certificate) { + result.push_back(std::to_string(config.port) + "s"); + result.push_back("ssl_certificate"); + result.push_back(*config.certificate); + } else { + result.push_back(std::to_string(config.port)); + } + + if (config.ca_certificate) { + result.push_back("ssl_ca_file"); + result.push_back(*config.ca_certificate); + } + return result; } void PrometheusExposerWrapper::registerMetric(const std::shared_ptr& metric) { diff --git a/extensions/prometheus/PrometheusExposerWrapper.h b/extensions/prometheus/PrometheusExposerWrapper.h index 2cbe992fad..b77bea3a0d 100644 --- a/extensions/prometheus/PrometheusExposerWrapper.h +++ b/extensions/prometheus/PrometheusExposerWrapper.h @@ -17,21 +17,32 @@ #pragma once #include +#include +#include #include "MetricsExposer.h" #include "prometheus/exposer.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" +#include "controllers/SSLContextService.h" namespace org::apache::nifi::minifi::extensions::prometheus { +struct PrometheusExposerConfig { + uint32_t port; + std::optional certificate; + std::optional ca_certificate; +}; + class PrometheusExposerWrapper : public MetricsExposer { public: - explicit PrometheusExposerWrapper(uint32_t port); + explicit PrometheusExposerWrapper(const PrometheusExposerConfig& config); void registerMetric(const std::shared_ptr& metric) override; void removeMetric(const std::shared_ptr& metric) override; private: + static std::vector parseExposerConfig(const PrometheusExposerConfig& config); + ::prometheus::Exposer exposer_; std::shared_ptr logger_{core::logging::LoggerFactory::getLogger()}; }; diff --git a/extensions/prometheus/PrometheusMetricsPublisher.cpp b/extensions/prometheus/PrometheusMetricsPublisher.cpp index f745249a35..d9052afb14 100644 --- a/extensions/prometheus/PrometheusMetricsPublisher.cpp +++ b/extensions/prometheus/PrometheusMetricsPublisher.cpp @@ -17,11 +17,11 @@ #include "PrometheusMetricsPublisher.h" #include +#include #include "core/Resource.h" #include "utils/StringUtils.h" #include "utils/OsUtils.h" -#include "PrometheusExposerWrapper.h" #include "utils/Id.h" namespace org::apache::nifi::minifi::extensions::prometheus { @@ -33,18 +33,32 @@ PrometheusMetricsPublisher::PrometheusMetricsPublisher(const std::string &name, void PrometheusMetricsPublisher::initialize(const std::shared_ptr& configuration, const std::shared_ptr& response_node_loader) { state::MetricsPublisher::initialize(configuration, response_node_loader); if (!exposer_) { - exposer_ = std::make_unique(readPort()); + exposer_ = std::make_unique(readExposerConfig()); } loadAgentIdentifier(); } -uint32_t PrometheusMetricsPublisher::readPort() { +PrometheusExposerConfig PrometheusMetricsPublisher::readExposerConfig() const { gsl_Expects(configuration_); + PrometheusExposerConfig config; if (auto port = configuration_->get(Configuration::nifi_metrics_publisher_prometheus_metrics_publisher_port)) { - return std::stoul(*port); + try { + config.port = std::stoul(*port); + } catch(const std::exception&) { + throw Exception(GENERAL_EXCEPTION, "Port configured for Prometheus metrics publisher is invalid: '" + *port + "'"); + } + } else { + throw Exception(GENERAL_EXCEPTION, "Port not configured for Prometheus metrics publisher!"); } - throw Exception(GENERAL_EXCEPTION, "Port not configured for Prometheus metrics publisher!"); + if (auto cert = configuration_->get(Configuration::nifi_metrics_publisher_prometheus_metrics_publisher_certificate)) { + config.certificate = *cert; + } + + if (auto ca_cert = configuration_->get(Configuration::nifi_metrics_publisher_prometheus_metrics_publisher_ca_certificate)) { + config.ca_certificate = *ca_cert; + } + return config; } void PrometheusMetricsPublisher::clearMetricNodes() { @@ -67,7 +81,7 @@ void PrometheusMetricsPublisher::loadMetricNodes() { } } -std::vector PrometheusMetricsPublisher::getMetricNodes() { +std::vector PrometheusMetricsPublisher::getMetricNodes() const { gsl_Expects(response_node_loader_ && configuration_); std::vector nodes; auto metric_classes_str = configuration_->get(minifi::Configuration::nifi_metrics_publisher_prometheus_metrics_publisher_metrics); diff --git a/extensions/prometheus/PrometheusMetricsPublisher.h b/extensions/prometheus/PrometheusMetricsPublisher.h index 131502f4a0..3dd5f81082 100644 --- a/extensions/prometheus/PrometheusMetricsPublisher.h +++ b/extensions/prometheus/PrometheusMetricsPublisher.h @@ -27,7 +27,7 @@ #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" -#include "MetricsExposer.h" +#include "PrometheusExposerWrapper.h" namespace org::apache::nifi::minifi::extensions::prometheus { @@ -42,8 +42,8 @@ class PrometheusMetricsPublisher : public state::MetricsPublisher { void loadMetricNodes() override; private: - uint32_t readPort(); - std::vector getMetricNodes(); + PrometheusExposerConfig readExposerConfig() const; + std::vector getMetricNodes() const; void loadAgentIdentifier(); std::mutex registered_metrics_mutex_; diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h index 3e82955103..14b0af4b91 100644 --- a/libminifi/include/properties/Configuration.h +++ b/libminifi/include/properties/Configuration.h @@ -188,6 +188,8 @@ class Configuration : public Properties { static constexpr const char *nifi_metrics_publisher_log_metrics_logging_interval = "nifi.metrics.publisher.LogMetricsPublisher.logging.interval"; static constexpr const char *nifi_metrics_publisher_log_metrics_log_level = "nifi.metrics.publisher.LogMetricsPublisher.log.level"; static constexpr const char *nifi_metrics_publisher_metrics = "nifi.metrics.publisher.metrics"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_certificate = "nifi.metrics.publisher.PrometheusMetricsPublisher.certificate"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_ca_certificate = "nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate"; // Controller socket options static constexpr const char *controller_socket_enable = "controller.socket.enable"; diff --git a/libminifi/src/Configuration.cpp b/libminifi/src/Configuration.cpp index e5a593b8a5..93f691ca19 100644 --- a/libminifi/src/Configuration.cpp +++ b/libminifi/src/Configuration.cpp @@ -150,6 +150,8 @@ const std::unordered_map