Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MINIFICPP-2460 Add tests for S2S tests using SSL #1869

Closed
wants to merge 1 commit into from
Closed

MINIFICPP-2460 Add tests for S2S tests using SSL #1869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions docker/test/integration/cluster/ContainerStore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .containers.MinifiContainer import MinifiOptions
from .containers.MinifiContainer import MinifiContainer
from .containers.NifiContainer import NifiContainer
from .containers.NifiContainer import NiFiOptions
from .containers.ZookeeperContainer import ZookeeperContainer
from .containers.KafkaBrokerContainer import KafkaBrokerContainer
from .containers.S3ServerContainer import S3ServerContainer
Expand Down Expand Up @@ -52,6 +53,7 @@ def __init__(self, network, image_store, kubernetes_proxy, feature_id):
self.network = network
self.image_store = image_store
self.kubernetes_proxy = kubernetes_proxy
self.nifi_options = NiFiOptions()

def get_container_name_with_postfix(self, container_name: str):
if not container_name.endswith(self.feature_id):
Expand Down Expand Up @@ -83,13 +85,14 @@ def acquire_container(self, context, container_name: str, engine='minifi-cpp', c
logging.info('Container name was not provided; using generated name \'%s\'', container_name)

feature_context = FeatureContext(feature_id=context.feature_id,
root_ca_cert=context.test.root_ca_cert,
root_ca_key=context.test.root_ca_key)
root_ca_cert=context.root_ca_cert,
root_ca_key=context.root_ca_key)

if engine == 'nifi':
return self.containers.setdefault(container_name,
NifiContainer(feature_context=feature_context,
config_dir=self.data_directories["nifi_config_dir"],
options=self.nifi_options,
name=container_name,
vols=self.vols,
network=self.network,
Expand Down Expand Up @@ -405,3 +408,6 @@ def enable_ssl_in_grafana_loki(self):

def enable_multi_tenancy_in_grafana_loki(self):
self.grafana_loki_options.enable_multi_tenancy = True

def enable_ssl_in_nifi(self):
self.nifi_options.use_ssl = True
3 changes: 3 additions & 0 deletions docker/test/integration/cluster/DockerTestCluster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,3 +418,6 @@ def wait_for_lines_on_grafana_loki(self, lines: List[str], timeout_seconds: int,

def set_value_on_plc_with_modbus(self, container_name, modbus_cmd):
return self.modbus_checker.set_value_on_plc_with_modbus(container_name, modbus_cmd)

def enable_ssl_in_nifi(self):
self.container_store.enable_ssl_in_nifi()
121 changes: 89 additions & 32 deletions docker/test/integration/cluster/DockerTestDirectoryBindings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import logging
import os
import shutil
import hashlib
import subprocess
import OpenSSL.crypto
from ssl_utils.SSL_cert_utils import make_self_signed_cert, make_cert_without_extended_usage, make_server_cert


class DockerTestDirectoryBindings:
Expand Down Expand Up @@ -59,39 +60,39 @@ def create_new_data_directories(self):
shutil.copytree(test_dir + "/resources/minifi", self.data_directories[self.feature_id]["minifi_config_dir"], dirs_exist_ok=True)
shutil.copytree(test_dir + "/resources/minifi-controller", self.data_directories[self.feature_id]["resources_dir"] + "/minifi-controller")

def get_data_directories(self, feature_id):
return self.data_directories[feature_id]
def get_data_directories(self):
return self.data_directories[self.feature_id]

def docker_path_to_local_path(self, feature_id, docker_path):
def docker_path_to_local_path(self, docker_path):
# Docker paths are currently hard-coded
if docker_path == "/tmp/input":
return self.data_directories[feature_id]["input_dir"]
return self.data_directories[self.feature_id]["input_dir"]
if docker_path == "/tmp/output":
return self.data_directories[feature_id]["output_dir"]
return self.data_directories[self.feature_id]["output_dir"]
if docker_path == "/tmp/resources":
return self.data_directories[feature_id]["resources_dir"]
return self.data_directories[self.feature_id]["resources_dir"]
# Might be worth reworking these
if docker_path == "/tmp/output/success":
self.create_directory(self.data_directories[feature_id]["output_dir"] + "/success")
return self.data_directories[feature_id]["output_dir"] + "/success"
self.create_directory(self.data_directories[self.feature_id]["output_dir"] + "/success")
return self.data_directories[self.feature_id]["output_dir"] + "/success"
if docker_path == "/tmp/output/failure":
self.create_directory(self.data_directories[feature_id]["output_dir"] + "/failure")
return self.data_directories[feature_id]["output_dir"] + "/failure"
self.create_directory(self.data_directories[self.feature_id]["output_dir"] + "/failure")
return self.data_directories[self.feature_id]["output_dir"] + "/failure"
raise Exception("Docker directory \"%s\" has no preset bindings." % docker_path)

def get_directory_bindings(self, feature_id):
def get_directory_bindings(self):
"""
Performs a standard container flow deployment with the addition
of volumes supporting test input/output directories.
"""
vols = {}
vols[self.data_directories[feature_id]["input_dir"]] = {"bind": "/tmp/input", "mode": "rw"}
vols[self.data_directories[feature_id]["output_dir"]] = {"bind": "/tmp/output", "mode": "rw"}
vols[self.data_directories[feature_id]["resources_dir"]] = {"bind": "/tmp/resources", "mode": "rw"}
vols[self.data_directories[feature_id]["system_certs_dir"]] = {"bind": "/usr/local/share/certs", "mode": "rw"}
vols[self.data_directories[feature_id]["minifi_config_dir"]] = {"bind": "/tmp/minifi_config", "mode": "rw"}
vols[self.data_directories[feature_id]["nifi_config_dir"]] = {"bind": "/tmp/nifi_config", "mode": "rw"}
vols[self.data_directories[feature_id]["kubernetes_config_dir"]] = {"bind": "/tmp/kubernetes_config", "mode": "rw"}
vols[self.data_directories[self.feature_id]["input_dir"]] = {"bind": "/tmp/input", "mode": "rw"}
vols[self.data_directories[self.feature_id]["output_dir"]] = {"bind": "/tmp/output", "mode": "rw"}
vols[self.data_directories[self.feature_id]["resources_dir"]] = {"bind": "/tmp/resources", "mode": "rw"}
vols[self.data_directories[self.feature_id]["system_certs_dir"]] = {"bind": "/usr/local/share/certs", "mode": "rw"}
vols[self.data_directories[self.feature_id]["minifi_config_dir"]] = {"bind": "/tmp/minifi_config", "mode": "rw"}
vols[self.data_directories[self.feature_id]["nifi_config_dir"]] = {"bind": "/tmp/nifi_config", "mode": "rw"}
vols[self.data_directories[self.feature_id]["kubernetes_config_dir"]] = {"bind": "/tmp/kubernetes_config", "mode": "rw"}
return vols

@staticmethod
Expand Down Expand Up @@ -120,21 +121,24 @@ def put_file_contents(file_abs_path, contents):
test_input_file.write(contents)
os.chmod(file_abs_path, 0o0777)

def put_test_resource(self, feature_id, file_name, contents):
def put_test_resource(self, file_name, contents):
"""
Creates a resource file in the test resource dir and writes
the given content to it.
"""

file_abs_path = os.path.join(self.data_directories[feature_id]["resources_dir"], file_name)
file_abs_path = os.path.join(self.data_directories[self.feature_id]["resources_dir"], file_name)
self.put_file_contents(file_abs_path, contents)

def put_test_input(self, feature_id, file_name, contents):
file_abs_path = os.path.join(self.data_directories[feature_id]["input_dir"], file_name)
def get_test_resource_path(self, file_name):
return os.path.join(self.data_directories[self.feature_id]["resources_dir"], file_name)

def put_test_input(self, file_name, contents):
file_abs_path = os.path.join(self.data_directories[self.feature_id]["input_dir"], file_name)
self.put_file_contents(file_abs_path, contents)

def put_file_to_docker_path(self, feature_id, path, file_name, contents):
file_abs_path = os.path.join(self.docker_path_to_local_path(feature_id, path), file_name)
def put_file_to_docker_path(self, path, file_name, contents):
file_abs_path = os.path.join(self.docker_path_to_local_path(path), file_name)
self.put_file_contents(file_abs_path, contents)

@staticmethod
Expand All @@ -146,14 +150,67 @@ def generate_md5_hash(file_path):

return md5_hash.hexdigest()

def put_random_file_to_docker_path(self, test_id: str, path: str, file_name: str, file_size: int):
file_abs_path = os.path.join(self.docker_path_to_local_path(test_id, path), file_name)
def put_random_file_to_docker_path(self, path: str, file_name: str, file_size: int):
file_abs_path = os.path.join(self.docker_path_to_local_path(path), file_name)
with open(file_abs_path, 'wb') as test_input_file:
test_input_file.write(os.urandom(file_size))
os.chmod(file_abs_path, 0o0777)
return self.generate_md5_hash(file_abs_path)

def rm_out_child(self, feature_id, dir):
child = os.path.join(self.data_directories[feature_id]["output_dir"], dir)
logging.info('Removing %s from output folder', child)
shutil.rmtree(child)
def create_cert_files(self):
self.root_ca_cert, self.root_ca_key = make_self_signed_cert("root CA")

minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=f"minifi-cpp-flow-{self.feature_id}",
ca_cert=self.root_ca_cert,
ca_key=self.root_ca_key)
minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{self.feature_id}",
ca_cert=self.root_ca_cert,
ca_key=self.root_ca_key)
self_signed_server_cert, self_signed_server_key = make_self_signed_cert(f"server-{self.feature_id}")

self.put_test_resource('root_ca.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=self.root_ca_cert))
self.put_test_resource("system_certs_dir/ca-root-nss.crt",
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=self.root_ca_cert))
self.put_test_resource('minifi_client.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=minifi_client_cert))
self.put_test_resource('minifi_client.key',
OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
pkey=minifi_client_key))
self.put_test_resource('minifi_server.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=minifi_server_cert)
+ OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
pkey=minifi_server_key))
self.put_test_resource('self_signed_server.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=self_signed_server_cert)
+ OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
pkey=self_signed_server_key))
self.put_test_resource('minifi_merged_cert.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=minifi_client_cert)
+ OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
pkey=minifi_client_key))
nifi_client_cert, nifi_client_key = make_server_cert(common_name=f"nifi-{self.feature_id}",
ca_cert=self.root_ca_cert,
ca_key=self.root_ca_key)
self.put_test_resource('nifi_client.crt',
OpenSSL.crypto.dump_certificate(type=OpenSSL.crypto.FILETYPE_PEM,
cert=nifi_client_cert))
self.put_test_resource('nifi_client.key',
OpenSSL.crypto.dump_privatekey(type=OpenSSL.crypto.FILETYPE_PEM,
pkey=nifi_client_key))
base = os.path.dirname(self.get_test_resource_path('nifi_client.key'))
test_dir = os.environ['TEST_DIRECTORY'] # Based on DockerVerify.sh
cmd = [
os.path.join(test_dir, "convert_cert_to_jks.sh"),
base,
os.path.join(base, "nifi_client.key"),
os.path.join(base, "nifi_client.crt"),
os.path.join(base, "root_ca.crt"),
]
subprocess.run(cmd, check=True)
5 changes: 2 additions & 3 deletions docker/test/integration/cluster/containers/MinifiContainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,8 @@ def _create_properties(self):

if self.options.set_ssl_context_properties:
f.write("nifi.remote.input.secure=true\n")
f.write("nifi.security.client.certificate=/tmp/resources/minifi-cpp-flow.crt\n")
f.write("nifi.security.client.private.key=/tmp/resources/minifi-cpp-flow.key\n")
f.write("nifi.security.client.pass.phrase=abcdefgh\n")
f.write("nifi.security.client.certificate=/tmp/resources/minifi_client.crt\n")
f.write("nifi.security.client.private.key=/tmp/resources/minifi_client.key\n")
f.write("nifi.security.client.ca.certificate=/tmp/resources/root_ca.crt\n")

if not self.options.enable_provenance:
Expand Down
Loading
Loading