diff --git a/.secrets.baseline b/.secrets.baseline index 6efec3d377..5c7045df2f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -149,7 +149,7 @@ "filename": "config/slips.yaml", "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", "is_verified": false, - "line_number": 295 + "line_number": 304 } ], "dataset/test14-malicious-zeek-dir/http.log": [ @@ -7185,5 +7185,5 @@ } ] }, - "generated_at": "2026-04-08T14:13:03Z" + "generated_at": "2026-04-16T20:58:25Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 1347ce71b9..a2ea80c65a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* Add optional live Slips auto-update support controlled by `update.auto_update_slips`. + + 1.1.19 (Mar 31st, 2026) * Add SSH bruteforce detection based on Zeek SSH, software, and notice logs. diff --git a/config/iris_config.yaml b/config/iris_config.yaml index 76625208fc..2e5e1b6111 100644 --- a/config/iris_config.yaml +++ b/config/iris_config.yaml @@ -2,11 +2,11 @@ Identity: GenerateNewKey: true Server: Port: 9010 - Host: 0.0.0.0 + Host: null DhtServerMode: 'true' Redis: Host: 127.0.0.1 - Port: 6379 + Port: 6644 Tl2NlChannel: iris_internal PeerDiscovery: DisableBootstrappingNodes: false diff --git a/config/slips.yaml b/config/slips.yaml index 1132ce092d..ef22d6f1fb 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -1,6 +1,15 @@ # This configuration file controls several aspects of the working of Slips. --- +update: + # Enable automatic live updates of the installed Slips version. + # This setting is separate from the feeds_update_manager module, which only + # updates TI feeds and related files during runtime. + # Automatic Slips updates may overwrite the default config files shipped + # with Slips. If you want to keep your local configuration changes intact, avoid editing the default config files. + # Instead, create separate config files with different names and use those. + auto_update_slips: false + output: # Define the file names for the default output. stdout: slips.log @@ -206,7 +215,7 @@ modules: # Add the names of other modules that you want to disable # (use module snake_case names). Example, # threat_intelligence, blocking, network_discovery, timeline, virustotal, - # rnn_cc_detection, flow_ml_detection, update_manager + # rnn_cc_detection, flow_ml_detection, feeds_update_manager disable: [template] # For each line in timeline file there is a timestamp. @@ -302,7 +311,7 @@ virustotal: ############################# threatintelligence: - # By default, slips starts without the TI files, and runs the update_manager + # By default, slips starts without the TI files, and runs the feeds_update_manager # in the background. If this option is set to true, slips will not start # analyzing the flows until the update manager finished and all TI files are # loaded successfully. diff --git a/docs/immune/auto_update.md b/docs/immune/auto_update.md new file mode 100644 index 0000000000..e4c4815aa6 --- /dev/null +++ b/docs/immune/auto_update.md @@ -0,0 +1,147 @@ +# Slips Auto Update + +## Table of Contents + +- [Overview](#overview) +- [How Auto-Update Works](#how-auto-update-works) + * [New version checks](#new-version-checks) + * [Updating Logic](#updating-logic) + + [Redis handling](#redis-handling) + + [Zeek log handling](#zeek-log-handling) + * [Draining and Shutdown of the old Slips](#draining-and-shutdown-of-the-old-slips) +- [How to use it](#how-to-use-it) +- [PR](#pr) + +## Overview + +Slips auto update functionality was designed to allow a running instance of +Slips to update itself with no downtime during the transition between versions. + +Updates usually consist of: + +A full application stop -> update -> restart sequence. +rather than simple restart. + +That sequence would lead to downtime and temporarily missing of flows during the upgrade. + +The implemented update mechanism was designed around "handover" instead of "restarts". +Where slips checks periodically for new compatible versions, pulls the update, +starts the new version, and orchestrates a controlled handover from the old +version to the new one. + + +## How Auto-Update Works + +```text +Old Slips running + ↓ +Check for compatible update + ↓ +git pull origin master + ↓ +Start new Slips with -u + ↓ +New Slips restores state and starts processing + ↓ +Old Slips drains + ↓ +Old Slips graceful shutdown + ↓ +New Slips continues normally +``` + + +### New version checks + +Slips checks for updates once per day. + +This is handled by the UpdateManager, which: + +- checks whether auto-update is enabled in the config file +- checks whether a new version exists +- checks compatibility before attempting update. + +Compatibility is determined using an `update.json` file hosted with the +deployed version. + +This file includes metadata about the new version such as: + +- latest version, +- backwards compatibility, +- whether new dependencies are needed. + + +The compatibility parser was added so we avoid updating to incompatible new releases. + + +**The update is aborted if:** + +- `auto_update_slips` is disabled in slips config. +- Slips is running on offline input instead of interface +- no newer version exists +- update is incompatible according to `update.json` +- local uncommitted changes are detected during `git pull` +- startup of the new version fails + + +### Updating Logic + + +When designing this, our main goal was zero downtime and zero missed flows during the update. This has the cost of maybe reading a very few duplicate flows, and this was done by +Starting the new Slips before stopping the old one. + +How this is done is: +- The old version starts the new one with the undocumented `-u` flag. +- The `-u` flag tells the new Slips instance that: 1. this is not a fresh run and 2. Slips should continue existing analysis and handle database migrations. +- do not overwrite: output dir, log files, and previous analysis artifacts/metrics. + + +#### Redis handling + +The new Slips does not flush Redis on startup. + +Instead, it appends to the existing Redis state as if the old process never +stopped. + +Without this, ongoing detections, states, and evidence state would be lost. + + +Now what happen when in the very few seconds during handover, a msg from the old slips' pub/sub is published, and the new updated slips receives it? + +To avoid this we added Pub/Sub message versioning, now each pub/sub message includes +the Slips version and consumers ignore messages that belong to the updated version and only read msgs intended +for them. + + +#### Zeek log handling + +The new Slips starts a new zeek process that uses new zeek log files in ```output/zeek_files/slips_vx.y.z```. +This ensures that the old zeek logs are not modified, re-read or overwritten during the update + +This was a major simplification because sharing Zeek logs between versions +introduced complexity and race conditions. + + +### Draining and Shutdown of the old Slips + +Once the updated Slips is confirmed to be running: + +the old Slips begins draining. + +Draining means: + +- stop ingesting new flows. +- finish processing pending flows. + + +PS: the new updated slips version starts reading flows before the old one starts draining to ensure 0 downtime. + +## How to use it + +enable ```auto_update_slips``` in ```config/slips.yaml``` and run slips on your interface. + +now whenever a new version of Slips is available, it will update itself and the new slips will use the same CLI as the old one. + +## PR + +https://github.com/stratosphereips/StratosphereLinuxIPS/pull/1915 diff --git a/docs/iris_module.md b/docs/iris_module.md index dc15dd30db..ab2308f16e 100644 --- a/docs/iris_module.md +++ b/docs/iris_module.md @@ -98,7 +98,7 @@ will be left upon the future developers. * ```go test ./...``` ### Integration Testing -Integration tests are located in ```tests/integration/test_iris.py```. +Integration tests are located in ```tests/integration/test_iris/test_iris.py```. ### Test Messaging The scenario that was modeled in this test refers to a common use case. diff --git a/docs/usage.md b/docs/usage.md index 357c04ce0d..654e097fa8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -529,6 +529,19 @@ Use ```permanent_dir``` to choose where Slips stores databases and runtime-gener This includes persistent artifacts such as ```p2p_trust_runtime/``` and shared module databases like the Fides cache. +**Live Slips auto update** + +Use ```update.auto_update_slips``` to enable or disable automatic live updates of the installed Slips version. + +```yaml +update: + auto_update_slips: false +``` + +This setting is separate from the runtime ```feeds_update_manager``` module, which only updates TI feeds and related files. + +Automatic Slips updates may overwrite the default config files shipped with Slips. If you want to keep local config changes safe, do not modify the default config files. Create and use your own config files with different names instead. +
diff --git a/managers/metadata_manager.py b/managers/metadata_manager.py index a8e02d8ced..7a6a631a8b 100644 --- a/managers/metadata_manager.py +++ b/managers/metadata_manager.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia -import subprocess - # SPDX-License-Identifier: GPL-2.0-only +import subprocess import psutil import sys import os @@ -121,8 +120,9 @@ def set_input_metadata(self): if self.main.args.interface: info.update({"interface": self.main.args.interface}) - if hasattr(self.main, "zeek_dir"): - info.update({"zeek_dir": self.main.zeek_dir}) + zeek_dir = self.main.db.get_zeek_output_dir() + if isinstance(zeek_dir, str) and zeek_dir: + info.update({"zeek_dir": zeek_dir}) if hasattr(self.main, "zeek_bro") and self.main.zeek_bro: info.update({"zeek_version": self.get_zeek_version()}) diff --git a/managers/process_manager.py b/managers/process_manager.py index 2c50143570..3d82c7eed6 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -9,7 +9,6 @@ import sys import time import traceback -import threading from collections import OrderedDict from datetime import datetime from multiprocessing import ( @@ -32,7 +31,10 @@ import modules -from modules.update_manager.update_manager import UpdateManager +from managers.update_manager import UpdateManager +from modules.feeds_update_manager.feeds_update_manager import ( + FeedsUpdateManager, +) from slips_files.common.slips_utils import utils from slips_files.common.abstracts.imodule import ( IModule, @@ -41,9 +43,6 @@ from slips_files.common.style import green from slips_files.common.input_type import InputType -from slips_files.core.database.redis_db.timewindow_updater_thread.tw_updater import ( - timewindow_updater, -) from slips_files.core.evidence_handler import EvidenceHandler from slips_files.core.helpers.bloom_filters_manager import BFManager from slips_files.core.input import Input @@ -88,6 +87,7 @@ def __init__(self, main): # is set by the input process to indicate no more flows are coming # so profiler can safely begin shutdown/joins. self.is_input_done_event = Event() + self.is_slips_live_updating_event = Event() self.read_config() def read_config(self): @@ -99,6 +99,13 @@ def read_config(self): # self.bootstrap_p2p, self.boootstrapping_modules = self.main.conf. # get_bootstrapping_setting() + def start_slips_update_manager(self): + return UpdateManager( + database=self.main.db, + is_slips_live_updating_event=self.is_slips_live_updating_event, + print_func=self.main.print, + ) + def start_output_process(self, stderr, slips_logfile, stdout=""): output_process = Output( stdout=stdout, @@ -108,6 +115,7 @@ def start_output_process(self, stderr, slips_logfile, stdout=""): debug=self.main.args.debug, input_type=self.main.input_type, create_logfiles=False if self.main.args.stopdaemon else True, + slips_args=self.main.args, ) self.slips_logfile = output_process.slips_logfile return output_process @@ -174,10 +182,10 @@ def start_input_process(self): input_information=self.main.input_information, cli_packet_filter=self.main.args.pcapfilter, zeek_or_bro=self.main.zeek_bro, - zeek_dir=self.main.zeek_dir, line_type=self.main.line_type, is_profiler_done_event=self.is_profiler_done_event, is_input_done_event=self.is_input_done_event, + is_slips_live_updating_event=self.is_slips_live_updating_event, ) input_process.start() self.main.print( @@ -458,26 +466,6 @@ def init_bloom_filters_manager(self): self.main.pid, ) - def start_timewindow_updater(self): - """ - Starts a thread that keeps track of the current timewindow if - running on an interface - - why is this not started in the redis db? because each module - has a db insteance, and we don't want a thread per module, - so starrting this thread once in main is enough - """ - if not self.main.args.interface: - return - tw_width: float = self.main.conf.get_tw_width_in_seconds() - t = threading.Thread( - target=timewindow_updater, - name="timewindow_updater", - args=(self.main.db, tw_width, self.termination_event), - daemon=True, - ) - utils.start_thread(t, self.main.db) - def start_update_manager(self, local_files=False, ti_feeds=False): """ starts the update manager process @@ -497,7 +485,7 @@ def start_update_manager(self, local_files=False, ti_feeds=False): with Lock(name="slips_ports_and_orgs"): # pass a dummy termination event for update manager to # update orgs and ports info - update_manager = UpdateManager( + update_manager = FeedsUpdateManager( self.main.logger, self.main.args.output, self.main.redis_port, @@ -541,9 +529,9 @@ def warn_about_pending_modules(self, pending_modules: List[Process]): ) # check if update manager is still alive - if "update_manager" in pending_module_names: + if "feeds_update_manager" in pending_module_names: self.main.print( - "update_manager may take several minutes " + "feeds_update_manager may take several minutes " "to finish updating 45+ TI files." ) @@ -638,6 +626,11 @@ def should_stop_slips(self) -> bool: This function NEVER returns True if the input and profiler are still processing. """ + if self.is_slips_live_updating_event.is_set(): + # slips is auto updating this version of slips should stop and + # the updated one will start + return True + if self.should_run_non_stop(): return False @@ -658,7 +651,7 @@ def is_stop_msg_received(self) -> bool: return ( utils.is_msg_intended_for(message, "control_channel") - and message["data"] == "stop_slips" + and utils.get_msg_payload(message) == "stop_slips" ) def is_debugger_active(self) -> bool: @@ -780,6 +773,21 @@ def get_print_function(self): else: return self.main.print + def _generate_plots(self): + if self.is_slips_live_updating_event: + # slips is updating and will start a new instance, plots + # should be done when slips is actually shutting down at the + # very end of the analysis. + return + + if self.main.conf.generate_performance_plots() is True: + self.plotter = Plotter(self.main.args.output, print) + self.plotter.plot_latency_csv() + self.plotter.plot_profiler_latency_csvs() + self.plotter.plot_throughput_csv() + self.plotter.write_throughput_metrics() + self.plotter.plot_flows_from_conn_log() + def shutdown_gracefully(self): """ Waits for all modules to confirm that they're done processing @@ -787,13 +795,8 @@ def shutdown_gracefully(self): """ try: print = self.get_print_function() - if self.main.conf.generate_performance_plots() is True: - self.plotter = Plotter(self.main.args.output, print) - self.plotter.plot_latency_csv() - self.plotter.plot_profiler_latency_csvs() - self.plotter.plot_throughput_csv() - self.plotter.write_throughput_metrics() - self.plotter.plot_flows_from_conn_log() + + self._generate_plots() if not self.main.args.stopdaemon: print("\n" + "-" * 27) @@ -802,17 +805,19 @@ def shutdown_gracefully(self): self.children: List[BaseProcess] = ( multiprocessing.active_children() ) - # by default, max 15 mins (taken from wait_for_modules_to_finish) - # from this time, all modules should be killed method_start_time = time.time() - # how long to wait for modules to finish in minutes + # how long to wait for modules to finish in minutes before + # killing them timeout: float = self.main.conf.wait_for_modules_to_finish() # convert to seconds timeout *= 60 - # close all tws - self.main.db.check_tw_to_close(close_all=True) + # dont close tws if we're updating, the next slips will continue + # from where this slips left off. + if not self.is_slips_live_updating_event.is_set(): + # close all tws + self.main.db.check_tw_to_close(close_all=True) graceful_shutdown = True if self.main.mode == "daemonized": @@ -868,20 +873,19 @@ def shutdown_gracefully(self): self.kill_all_children() - if self.main.args.save: - self.main.save_the_db() - - if self.main.conf.export_labeled_flows(): - format_ = self.main.conf.export_labeled_flows_to().lower() - self.main.db.export_labeled_flows(format_) - - # if store_a_copy_of_zeek_files is set to yes in slips.yaml - # copy the whole zeek_files dir to the output dir - self.main.store_zeek_dir_copy() + if not self.is_slips_live_updating_event.is_set(): + if self.main.args.save: + self.main.save_the_db() + if self.main.conf.export_labeled_flows(): + format_ = self.main.conf.export_labeled_flows_to().lower() + self.main.db.export_labeled_flows(format_) - # if delete_zeek_files is set to yes in slips.yaml, - # delete zeek_files/ dir - self.main.delete_zeek_files() + # if store_a_copy_of_zeek_files is set to yes in slips.yaml + # copy the whole zeek_files dir to the output dir + self.main.store_zeek_dir_copy() + # if delete_zeek_files is set to yes in slips.yaml, + # delete zeek_files/ dir + self.main.delete_zeek_files() analysis_time, end_date = self.get_analysis_time() self.main.metadata_man.set_analysis_end_date(end_date) @@ -896,6 +900,14 @@ def shutdown_gracefully(self): self.main.db.close_all_dbs() if graceful_shutdown: + if self.is_slips_live_updating_event.is_set(): + print( + "[Process Manager] Slips is live updating, " + "Stopping this instance and starting the new " + "instance now.\n", + log_to_logfiles_only=True, + ) + print( "[Process Manager] Slips shutdown gracefully\n", log_to_logfiles_only=True, diff --git a/managers/redis_manager.py b/managers/redis_manager.py index 989493d1af..ae0903f27e 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -66,9 +66,11 @@ def log_redis_server_pid(self, redis_port: int, redis_pid: int): "Save the DB\n" ) + zeek_dir = self.main.db.get_zeek_output_dir() + f.write( f"{now},{self.main.input_information},{redis_port}," - f"{redis_pid},{self.main.zeek_dir},{self.main.args.output}," + f"{redis_pid},{zeek_dir},{self.main.args.output}," f"{os.getpid()}," f"{bool(self.main.args.daemon)},{self.main.args.save}\n" ) @@ -380,6 +382,14 @@ def get_redis_port(self) -> int: -m or the default port if all ports are unavailable, this function terminates slips """ + # when slips is started by another slips during an update + # handover, make sure the new slips + # continues using the same old redis db and port. + if self.main.args.is_slips_started_by_an_update: + if self.main.args.port: + return int(self.main.args.port) + return DEFAULT_REDIS_PORT + if self.main.args.port: redis_port = int(self.main.args.port) # if the default port is already in use, slips should override it diff --git a/managers/update_manager.py b/managers/update_manager.py new file mode 100644 index 0000000000..5a526b2444 --- /dev/null +++ b/managers/update_manager.py @@ -0,0 +1,365 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + + +""" +Handles updating of slips version +""" + +import json +import re +import subprocess +import sys +import time +from typing import Any, Dict, List, Optional +from urllib import error, request + +import psutil +from git import ( + GitCommandError, + GitError, + InvalidGitRepositoryError, + NoSuchPathError, + Repo, +) +from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils +from slips_files.core.database.database_manager import DBManager + + +class UpdateManager: + def __init__( + self, + database: DBManager = None, + is_slips_live_updating_event=None, + print_func=None, + ): + self.db = database + self.is_slips_live_updating_event = is_slips_live_updating_event + self.is_running_non_stop: bool = self.db.is_running_non_stop() + self.cached_update_info: Optional[Dict[str, Any]] = None + self.conf = ConfigParser() + self.args = self.conf.get_args() + # The very first time, slips is started by the user via CLI. then + # for each new update, it's started by this update manager. + # this func returns true if the user just started slips from cli. + self.is_first_run: bool = ( + True if not self.args.is_slips_started_by_an_update else False + ) + self._read_configuration() + self.last_update_time = 0 + self.print = print_func + + def _read_configuration(self): + self.auto_update_slips_enabled = self.conf.auto_update_slips() + + def _get_master_update_json_link(self) -> Optional[str]: + """ + Build the raw GitHub URL for update.json on the master branch. + + Returns: + The raw update.json URL if the origin remote is supported, + otherwise None. + """ + try: + remote_url = Repo(".").remote("origin").url + except (ValueError, InvalidGitRepositoryError, NoSuchPathError): + return None + + remote_url = remote_url.strip().removesuffix(".git").rstrip("/") + github_prefixes = ( + "https://github.com/", + "http://github.com/", + "ssh://git@github.com/", + ) + + repo_path = None + if remote_url.startswith("git@github.com:"): + repo_path = remote_url.split(":", maxsplit=1)[1] + else: + for prefix in github_prefixes: + if remote_url.startswith(prefix): + repo_path = remote_url.removeprefix(prefix) + break + + if not repo_path: + return None + + return ( + f"https://raw.githubusercontent.com/{repo_path}/master/update.json" + ) + + def _read_master_update_json(self) -> Dict[str, Any]: + """ + Read the update.json file from the origin/master branch of slips repo. + + Returns: + Parsed update metadata if it can be fetched and decoded, + otherwise an empty dictionary. + """ + if self.cached_update_info is not None: + return self.cached_update_info + + update_json_link = self._get_master_update_json_link() + if not update_json_link: + self.cached_update_info = {} + return self.cached_update_info + + try: + with request.urlopen(update_json_link, timeout=5) as response: + update_text = response.read().decode("utf-8") + except (OSError, UnicodeDecodeError, error.URLError): + self.cached_update_info = {} + return self.cached_update_info + + sanitized_update_text = re.sub(r",(\s*[}\]])", r"\1", update_text) + try: + update_data = json.loads(sanitized_update_text) + except json.JSONDecodeError: + self.cached_update_info = {} + return self.cached_update_info + + self.cached_update_info = ( + update_data if isinstance(update_data, dict) else {} + ) + return self.cached_update_info + + def _new_version_has_new_dependencies(self) -> bool: + """ + Check whether the version on master introduces new dependencies. + + Returns: + True if update.json reports new dependencies or the metadata + cannot be read safely, otherwise False. + """ + update_data = self._read_master_update_json() + return bool(update_data.get("has_new_dependencies", True)) + + def _is_new_version_backwards_compatible(self) -> bool: + """ + Check whether the version on master is backwards compatible. + + Returns: + True if update.json marks the update as backwards compatible, + otherwise False. + """ + update_data = self._read_master_update_json() + return bool(update_data.get("backwards_compatible", False)) + + def _is_new_version_available(self) -> bool: + update_data = self._read_master_update_json() + latest_version = update_data.get("version", False) + + if not latest_version: + return False + + return utils.get_current_version() != latest_version + + def git_pull_master(self): + """ + Pull the latest origin/master changes and check them out. + + Returns: + The checked out origin/master commit. + """ + repo = Repo(".") + repo.remote("origin").fetch("master") + repo.git.checkout("origin/master") + self.print( + "Done pulling new version and checking out master " "branch." + ) + + def _get_checkout_overwritten_files( + self, git_error: GitCommandError + ) -> List[str]: + """ + Extract files Git says would be overwritten by checkout. + we'll just print them to the user. + + Parameters: + git_error: The GitPython checkout failure. + + Returns: + The local paths reported by Git, or an empty list if this is not + a local-change checkout conflict. + """ + stderr = getattr(git_error, "stderr", "") or str(git_error) + if ( + "Your local changes to the following files would be " + "overwritten by checkout" not in stderr + ): + return [] + + files = [] + is_file_list = False + for line in stderr.splitlines(): + stripped_line = line.strip().strip("'") + if ( + "Your local changes to the following files would be " + "overwritten by checkout" in stripped_line + ): + is_file_list = True + continue + + if not is_file_list: + continue + + if stripped_line.startswith(("Please commit", "Aborting")): + break + + if stripped_line: + files.append(stripped_line) + + return files + + def _get_target_update_version(self) -> Optional[str]: + """ + Get the target Slips version from cached update metadata. + + Returns: + The update version if known, otherwise None. + """ + update_data = ( + self.cached_update_info or self._read_master_update_json() + ) + version = update_data.get("version") + return version if isinstance(version, str) and version else None + + def _get_updated_slips_command(self) -> List[str]: + """ + Build the command used to start the updated Slips process. + + Returns: + The current Slips cmd plus (-u). If the current Slips was + started with -m, pass the current Redis port explicitly so the + updated process reuses it. + """ + try: + cmd = psutil.Process().cmdline() + except psutil.Error: + cmd = [] + + if not cmd: + cmd = [sys.executable, *sys.argv] + + cmd = [*cmd, "-u"] + + if self.args.multiinstance: + cmd.remove("-m") + redis_port = self.db.get_used_redis_port() + cmd.extend(["-P", str(redis_port)]) + + return cmd + + def start_updated_slips_version(self) -> subprocess.Popen: + """ + Starts the updated Slips as an independent process. + + Returns: + The detached process handle for the updated Slips process. + """ + cmd: List[str] = self._get_updated_slips_command() + + str_cmd = " ".join(cmd) + self.print(f"Starting updated Slips version using command: {str_cmd}") + + # without dev/null redirection, the new updated slips will use the + # same cli as the old slips. so this is intentional. + process = subprocess.Popen( + cmd, + close_fds=True, + ) + + self.print("Done starting the updated Slips version.") + return process + + def _warn_about_aborted_update( + self, git_error: Optional[GitCommandError] = None + ): + overwritten_files = self._get_checkout_overwritten_files(git_error) + if not overwritten_files: + self.print( + "Warning: Aborting Slips update because a git error " + f"occurred: {git_error}" + ) + return + + target_version = self._get_target_update_version() + update_target = ( + f"Slips v{target_version}" + if target_version + else "the new Slips version" + ) + self.print( + f"Warning: Uncommitted changes to {overwritten_files} detected. " + f"Aborting update to {update_target}, please update Slips " + "manually." + ) + + def update_slips(self): + try: + self.git_pull_master() + except GitError as git_error: + self._warn_about_aborted_update(git_error) + return + + self.start_updated_slips_version() + # this event + # - signals input.py to stop recving input and start draining flows + # - and signals the process_manager() to call shutdown_gracefully() + self.is_slips_live_updating_event.set() + + def _did_1d_pass_since_last_update(self) -> bool: + """ + returns true once every 1 day. + """ + update_interval = 60 * 60 * 24 + if time.time() >= self.last_update_time + update_interval: + self.last_update_time = time.time() + return True + return False + + def check_for_update_every_1_day(self) -> bool: + """ + return sTrue if a new compatible version is available and slips + should update itself + """ + if self._did_1d_pass_since_last_update(): + should_update: bool = self.should_update_slips() + + if should_update: + self.print( + "A new version of Slips is available. " + "Updating slips now." + ) + else: + self.print( + "No new version of Slips is available. " + "Slips will check again after 1 day." + ) + return should_update + return False + + def should_update_slips(self) -> bool: + """ + returns true if the auto_update param in the config file is set to + true, and we're running on an interface, and there is a new + compatible version of slips. + """ + if not self.auto_update_slips_enabled: + return False + + if not self.is_running_non_stop: + # only update slips when running on an interface. + return False + + if not self._is_new_version_available(): + return False + + if ( + self._is_new_version_backwards_compatible() + and not self._new_version_has_new_dependencies() + ): + return True + + return False diff --git a/modules/arp/arp.py b/modules/arp/arp.py index e661e1475d..a6848a6795 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -566,7 +566,7 @@ def main(self): # if the tw is closed, remove all its entries from the cache dict if msg := self.get_msg("tw_closed"): - profileid_tw = msg["data"] + profileid_tw = utils.get_msg_payload(msg) # when a tw is closed, this means that it's too # old so we don't check for arp scan in this time # range anymore diff --git a/modules/arp_poisoner/arp_poisoner.py b/modules/arp_poisoner/arp_poisoner.py index a8dae0a976..dc191b6536 100644 --- a/modules/arp_poisoner/arp_poisoner.py +++ b/modules/arp_poisoner/arp_poisoner.py @@ -37,10 +37,10 @@ def init(self): ) self.blocking_logfile_lock = Lock() # clear it - try: - open(self.log_file_path, "w").close() - except FileNotFoundError: - pass + utils.initialize_logfile( + self.log_file_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) self.unblocker = ARPUnblocker( self.db, self.should_stop, self.logger, self.log ) @@ -414,7 +414,7 @@ def main(self): # if slips saw 3 ips, this channel will receive 3 msgs with tw1 # as closed. we're not interested in the ips, we just wanna # know when slips advances to the next tw. - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") twid = profileid_tw[-1] if self.last_closed_tw != twid: self.last_closed_tw = twid diff --git a/modules/blocking/blocking.py b/modules/blocking/blocking.py index f6b4fbe69e..9093952e08 100644 --- a/modules/blocking/blocking.py +++ b/modules/blocking/blocking.py @@ -41,10 +41,10 @@ def init(self): ) self.blocking_logfile_lock = Lock() # clear it - try: - open(self.blocking_log_path, "w").close() - except FileNotFoundError: - pass + utils.initialize_logfile( + self.blocking_log_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) self.last_closed_tw = None self.ap_info: None | Dict[str, str] = self.db.get_ap_info() @@ -274,7 +274,7 @@ def main(self): # if slips saw 3 ips, this channel will receive 3 msgs with tw1 # as closed. we're not interested in the ips, we just wanna # know when slips advances to the next tw. - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") twid = profileid_tw[-1] if self.last_closed_tw != twid: self.last_closed_tw = twid diff --git a/modules/brute_force_detector/brute_force_detector.py b/modules/brute_force_detector/brute_force_detector.py index 2127004f68..cae097f271 100644 --- a/modules/brute_force_detector/brute_force_detector.py +++ b/modules/brute_force_detector/brute_force_detector.py @@ -473,4 +473,4 @@ def main(self): self._handle_ssh(profileid, twid, flow) if msg := self.get_msg("tw_closed"): - self.cleanup_cache_dicts(msg["data"].split("_")) + self.cleanup_cache_dicts(utils.get_msg_payload(msg).split("_")) diff --git a/modules/cesnet/cesnet.py b/modules/cesnet/cesnet.py index a1313daa31..01fbb3561e 100644 --- a/modules/cesnet/cesnet.py +++ b/modules/cesnet/cesnet.py @@ -255,7 +255,12 @@ def pre_main(self): return 1 # create the warden client - self.wclient = Client(**read_cfg(self.configuration_file)) + self.wclient = Client( + **read_cfg(self.configuration_file), + is_slips_started_by_an_update=getattr( + self.args, "is_slips_started_by_an_update", False + ), + ) # All methods return something. # If you want to catch possible errors (for example implement some diff --git a/modules/cesnet/warden_client.py b/modules/cesnet/warden_client.py index 83bacbaa72..bcd021b617 100644 --- a/modules/cesnet/warden_client.py +++ b/modules/cesnet/warden_client.py @@ -17,6 +17,8 @@ from operator import itemgetter from pathlib import Path +from slips_files.common.slips_utils import utils + VERSION = "3.0-beta2" @@ -181,11 +183,13 @@ def __init__( idstore=None, name="org.example.warden.test", secret=None, + is_slips_started_by_an_update: bool = False, ): if errlog is None: errlog = {} self.name = name self.secret = secret + self.is_slips_started_by_an_update = is_slips_started_by_an_update # Init logging as soon as possible and make sure we don't # spit out exceptions but just log or return Error objects self.init_log(errlog, syslog, filelog) @@ -221,7 +225,11 @@ def create_file(self, filepath): # filename = path.basename(filepath) p = Path(dir) p.mkdir(parents=True, exist_ok=True) - open(filepath, "w").close() + utils.initialize_logfile( + filepath, + self.is_slips_started_by_an_update, + create_parent_dirs=False, + ) def init_log(self, errlog: dict, syslog: dict, filelog: dict): def loglevel(lev): @@ -650,6 +658,6 @@ def format_time( def read_cfg(cfgfile): with open(cfgfile, "r") as f: stripcomments = "\n".join( - (l for l in f if not l.lstrip().startswith(("#", "//"))) + (line for line in f if not line.lstrip().startswith(("#", "//"))) ) return json.loads(stripcomments) diff --git a/modules/exporting_alerts/stix_exporter.py b/modules/exporting_alerts/stix_exporter.py index b2eef15a78..875f0b5854 100644 --- a/modules/exporting_alerts/stix_exporter.py +++ b/modules/exporting_alerts/stix_exporter.py @@ -517,7 +517,6 @@ def should_export(self) -> bool: return "stix" in self.export_to def read_configuration(self) -> bool: - """Reads configuration""" conf = ConfigParser() # Available options ['slack','stix'] self.export_to = conf.export_to() diff --git a/modules/update_manager/__init__.py b/modules/feeds_update_manager/__init__.py similarity index 100% rename from modules/update_manager/__init__.py rename to modules/feeds_update_manager/__init__.py diff --git a/modules/update_manager/update_manager.py b/modules/feeds_update_manager/feeds_update_manager.py similarity index 99% rename from modules/update_manager/update_manager.py rename to modules/feeds_update_manager/feeds_update_manager.py index 0c768dc09b..df21f7b8c3 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/feeds_update_manager/feeds_update_manager.py @@ -22,15 +22,15 @@ CannotAcquireLock, ) -from modules.update_manager.timer_manager import InfiniteTimer +from modules.feeds_update_manager.timer_manager import InfiniteTimer from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.abstracts.imodule import IModule from slips_files.common.slips_utils import utils from slips_files.core.helpers.whitelist.whitelist import Whitelist -class UpdateManager(IModule): - name = "update_manager" +class FeedsUpdateManager(IModule): + name = "feeds_update_manager" description = "Update Threat Intelligence files" authors = ["Kamila Babayeva", "Alya Gomaa"] @@ -1480,6 +1480,7 @@ def update_local_whitelist(self): """ parses the local whitelist using the whitelist parser and stores it in the db + is only called when slips starts. """ if self.enable_local_whitelist: self.whitelist.update() diff --git a/modules/update_manager/timer_manager.py b/modules/feeds_update_manager/timer_manager.py similarity index 100% rename from modules/update_manager/timer_manager.py rename to modules/feeds_update_manager/timer_manager.py diff --git a/modules/fides/fides.py b/modules/fides/fides.py index 9e5cc07ae7..80867f7efd 100644 --- a/modules/fides/fides.py +++ b/modules/fides/fides.py @@ -221,7 +221,7 @@ def main(self): if not msg["data"]: return - ip = msg["data"] + ip = utils.get_msg_payload(msg) if utils.detect_ioc_type(ip) != "ip": return @@ -231,6 +231,6 @@ def main(self): self.__intelligence.request_data(ip) # TODO: the code below exists for testing purposes for - # tests/integration_tests/test_fides.py + # tests/integration/test_fides/test_fides.py if msg := self.get_msg("fides2network"): pass diff --git a/modules/flow_alerts/conn.py b/modules/flow_alerts/conn.py index 1ad44ca04d..3a85f496b8 100644 --- a/modules/flow_alerts/conn.py +++ b/modules/flow_alerts/conn.py @@ -840,7 +840,7 @@ async def analyze(self, msg): self.check_device_changing_ips(twid, flow) elif utils.is_msg_intended_for(msg, "tw_closed"): - profileid_tw: List[str] = msg["data"].split("_") + profileid_tw: List[str] = utils.get_msg_payload(msg).split("_") profileid = f"{profileid_tw[0]}_{profileid_tw[1]}" twid = profileid_tw[-1] self.detect_data_upload_in_twid(profileid, twid) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index f0ced354e8..fb31ab7a1c 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -596,7 +596,7 @@ async def main(self): self.get_domain_info(domain) if msg := self.get_msg("new_ip"): - ip = msg["data"] + ip = utils.get_msg_payload(msg) self.handle_new_ip(ip) if msg := self.get_msg("check_jarm_hash"): diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index 556f40929f..12173765c3 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -244,5 +244,5 @@ def main(self): self.check_dhcp_scan(profileid, twid, flow) if msg := self.get_msg("tw_closed"): - profileid_tw: List[str] = msg["data"].split("_") + profileid_tw: List[str] = utils.get_msg_payload(msg).split("_") self.cleanup_cache_dicts(profileid_tw) diff --git a/modules/p2p_trust/p2p_trust.py b/modules/p2p_trust/p2p_trust.py index 47396c4373..bdebcaff6f 100644 --- a/modules/p2p_trust/p2p_trust.py +++ b/modules/p2p_trust/p2p_trust.py @@ -193,6 +193,9 @@ def _configure(self): gopy_channel=self.gopy_channel, pygo_channel=self.pygo_channel, p2p_reports_logfile=self.p2p_reports_logfile, + is_slips_started_by_an_update=getattr( + self.args, "is_slips_started_by_an_update", False + ), ) self.pigeon = None diff --git a/modules/p2p_trust/utils/go_director.py b/modules/p2p_trust/utils/go_director.py index 1ee7b0755d..43abc8115f 100644 --- a/modules/p2p_trust/utils/go_director.py +++ b/modules/p2p_trust/utils/go_director.py @@ -52,6 +52,7 @@ def __init__( gopy_channel: str = "p2p_gopy", pygo_channel: str = "p2p_pygo", p2p_reports_logfile: str = "p2p_reports.log", + is_slips_started_by_an_update: bool = False, ): self.printer = Printer(logger, self.name) @@ -67,7 +68,9 @@ def __init__( self.report_func = report_func self.request_func = request_func # clear the logfile - open(p2p_reports_logfile, "w").close() + utils.initialize_logfile( + p2p_reports_logfile, is_slips_started_by_an_update + ) self.reports_logfile = open(p2p_reports_logfile, "a") self.print(f"Storing peer reports in {p2p_reports_logfile}") # TODO: there should be some better mechanism to add new processing diff --git a/modules/risk_iq/risk_iq.py b/modules/risk_iq/risk_iq.py index 58656cd917..7f8f125a15 100644 --- a/modules/risk_iq/risk_iq.py +++ b/modules/risk_iq/risk_iq.py @@ -93,7 +93,7 @@ def pre_main(self): def main(self): if msg := self.get_msg("new_ip"): - ip = msg["data"] + ip = utils.get_msg_payload(msg) if utils.is_ignored_ip(ip): # return here means keep looping return diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 873cc8dbff..b5bece382a 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -245,7 +245,7 @@ def handle_new_letters(self, msg: Dict): def handle_tw_closed(self, msg: Dict): """handles msgs from the tw_closed channel""" - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") profileid = f"{profileid_tw[0]}_{profileid_tw[1]}" twid = profileid_tw[-1] self.letters_exporter.export(profileid, twid) diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 507ef315e3..f824f67a04 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -1891,7 +1891,7 @@ def pre_main(self): utils.drop_root_privs_permanently() # Load the local Threat Intelligence files that are # stored in the local folder self.path_to_local_ti_files - # The remote files are being loaded by the update_manager + # The remote files are being loaded by the feeds_update_manager local_files = ( "own_malicious_iocs.csv", "own_malicious_JA3.csv", diff --git a/slips/daemon.py b/slips/daemon.py index e037d93b16..3b8630fc7e 100644 --- a/slips/daemon.py +++ b/slips/daemon.py @@ -49,6 +49,8 @@ def print(self, text, **kwargs): def create_std_streams(self): """Create standard steam files and dirs and clear them""" + if self.slips.args.is_slips_started_by_an_update: + return std_streams = [self.stderr, self.stdout, self.logsfile] for file in std_streams: diff --git a/slips/main.py b/slips/main.py index 072b7794cc..90d35b0b60 100644 --- a/slips/main.py +++ b/slips/main.py @@ -40,7 +40,7 @@ class Main: def __init__(self, testing=False): self.name = "main" - self.alerts_default_path = "output/" + self.parent_output_dir = "output/" self.mode = "interactive" self.sigterm_received = False # objects to manage various functionality @@ -73,12 +73,17 @@ def __init__(self, testing=False): self.input_information, self.line_type, ) = self.checker.get_input_type() + self.input_information = os.path.normpath( + self.input_information + ) + self.input_information = self.input_information.replace( + ",", "_" + ) + # If we need zeek (bro), test if we can run it. self.check_zeek_or_bro() self.prepare_output_dir() self.redis_man.start_redis_cache_if_not_running() - # this is the zeek dir slips will be using - self.prepare_zeek_output_dir() self.twid_width = self.conf.get_tw_width() # should be initialised after self.input_type is set self.host_ip_man = HostIPManager(self) @@ -103,15 +108,6 @@ def check_zeek_or_bro(self): return self.zeek_bro - def prepare_zeek_output_dir(self): - from pathlib import Path - - without_ext = Path(self.input_information).stem - if self.conf.store_zeek_files_in_the_output_dir(): - self.zeek_dir = os.path.join(self.args.output, "zeek_files") - else: - self.zeek_dir = f"zeek_files_{without_ext}/" - def terminate_slips(self): """ Shutdown slips, is called when stopping slips before @@ -162,14 +158,44 @@ def store_zeek_dir_copy(self): store_a_copy_of_zeek_files = self.conf.store_a_copy_of_zeek_files() was_running_zeek = self.was_running_zeek() if store_a_copy_of_zeek_files and was_running_zeek: + zeek_dir = self.db.get_zeek_output_dir() + if not isinstance(zeek_dir, str) or not zeek_dir: + return # this is where the copy will be stored dest_zeek_dir = os.path.join(self.args.output, "zeek_files") - copy_tree(self.zeek_dir, dest_zeek_dir) + copy_tree(zeek_dir, dest_zeek_dir) print(f"[Main] Stored a copy of zeek files to {dest_zeek_dir}") def delete_zeek_files(self): - if self.conf.delete_zeek_files(): - shutil.rmtree(self.zeek_dir) + zeek_dir = self.db.get_zeek_output_dir() + if ( + self.conf.delete_zeek_files() + and isinstance(zeek_dir, str) + and zeek_dir + ): + shutil.rmtree(zeek_dir) + + def del_file_or_dir(self, file): + """deletes a file or dir inside the output dir""" + file_path = os.path.join(self.args.output, file) + with contextlib.suppress(Exception): + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + def construct_output_dir_name(self) -> str: + dir_name = os.path.join( + self.parent_output_dir, + os.path.basename( + self.input_information + ), # get pcap name from path + ) + + # add timestamp to avoid conflicts e.g wlp3s0_2022-03-1_03:55 + ts = utils.convert_ts_format(datetime.now(), "%Y-%m-%d_%H:%M:%S") + dir_name += f"_{ts}/" + return dir_name def prepare_output_dir(self): """ @@ -178,47 +204,33 @@ def prepare_output_dir(self): Log dirs are stored in output/_%Y-%m-%d_%H:%M:%S @return: None """ - # default output/ - if "-o" in sys.argv: - # -o is given - # delete all old files in the output dir - if os.path.exists(self.args.output): - for file in os.listdir(self.args.output): - # in integration tests, slips redirects its - # output to slips_output.txt, - # don't delete that file - if self.args.testing and "slips_output.txt" in file: - continue - - file_path = os.path.join(self.args.output, file) - with contextlib.suppress(Exception): - if os.path.isfile(file_path): - os.remove(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - else: - os.makedirs(self.args.output) + if self.args.is_slips_started_by_an_update: + # we should append to existing files in the output dir, + # and never overwrite them. + return + + if not self.args.output: + # the user didnt give slips an output dir to use, construct one + self.args.output = self.construct_output_dir_name() + os.makedirs(self.args.output) + os.chmod(self.args.output, 0o777) return - # self.args.output is the same as self.alerts_default_path - self.input_information = os.path.normpath(self.input_information) - self.input_information = self.input_information.replace(",", "_") - # now that slips can run several instances, - # each created dir will be named after the instance - # that created it - # it should be output/wlp3s0 - self.args.output = os.path.join( - self.alerts_default_path, - os.path.basename( - self.input_information - ), # get pcap name from path - ) - # add timestamp to avoid conflicts wlp3s0_2022-03-1_03:55 - ts = utils.convert_ts_format(datetime.now(), "%Y-%m-%d_%H:%M:%S") - self.args.output += f"_{ts}/" + # -o is given + # delete all old files in the output dir + if os.path.exists(self.args.output): + for file in os.listdir(self.args.output): + # in integration tests, slips redirects its + # output to slips_output.txt, + # don't delete that file + if self.args.testing and "slips_output.txt" in file: + continue + self.del_file_or_dir(file) + + else: + os.makedirs(self.args.output) - os.makedirs(self.args.output) os.chmod(self.args.output, 0o777) def set_mode(self, mode, daemon=""): @@ -519,6 +531,7 @@ def start(self): self.print_version() print("https://stratosphereips.org") print("-" * 27) + self.setup_print_levels() stderr: str = self.get_slips_error_file() slips_logfile: str = self.get_slips_logfile() @@ -528,10 +541,22 @@ def start(self): stderr, slips_logfile ) self.printer = Printer(self.logger, self.name) + self.print(f"Storing Slips logs in {self.args.output}") + self.redis_port: int = self.redis_man.get_redis_port() - # dont start the redis server if it's already started - start_redis_server = not utils.is_port_in_use(self.redis_port) + if self.args.is_slips_started_by_an_update: + # -u means slips is started by slips after it auto-updated, + # the redis server should already be up, this slips should + # just connect to it + start_redis_server = False + self.print( + f"Slips is done auto updating. Currently running " + f"version: {green(utils.get_current_version())}" + ) + else: + # dont start the redis server if it's already started + start_redis_server = not utils.is_port_in_use(self.redis_port) try: self.db = DBManager( @@ -541,6 +566,10 @@ def start(self): self.conf, int(self.pid), start_redis_server=start_redis_server, + # if auto update is enabled, slips starts itself with + # -u, and continues using the same db as the old + # version without overwriting the keys there + flush_db=not self.args.is_slips_started_by_an_update, ) except RuntimeError as e: @@ -576,6 +605,8 @@ def start(self): } ) + self.update_man = self.proc_man.start_slips_update_manager() + # this func should be called as soon as we start the db, # before evdience proc starts. # to be able to use the host IP as analyzer IP in alerts.json @@ -588,9 +619,7 @@ def start(self): 0, ) self.print( - f'Started {green("Main")} process ' - f"[PID" - f" {green(self.pid)}]", + f'Started {green("Main")} process [PID {green(self.pid)}]', 1, 0, ) @@ -678,7 +707,6 @@ def sig_handler(sig, frame): self.c1 = self.db.subscribe("control_channel") self.metadata_man.add_metadata_if_enabled() - self.proc_man.start_timewindow_updater() self.input_process = self.proc_man.start_input_process() self.db.store_pid("main", int(self.pid)) @@ -728,6 +756,8 @@ def sig_handler(sig, frame): ) self.host_ip_man.update_host_ip(host_ips, modified_profiles) + if self.update_man.check_for_update_every_1_day(): + self.update_man.update_slips() except KeyboardInterrupt: # the EINTR error code happens if a signal occurred while diff --git a/slips_files/common/abstracts/imodule.py b/slips_files/common/abstracts/imodule.py index 2d436c221f..34b4502cdb 100644 --- a/slips_files/common/abstracts/imodule.py +++ b/slips_files/common/abstracts/imodule.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only +import json import os import sys import traceback @@ -69,6 +70,7 @@ def __init__( ) self.db.client_setname(self.name) self.keyboard_int_ctr = 0 + self.slips_version: str = utils.get_slips_version() self.init(**kwargs) # should after the module's init() so the module has a chance to # set its own channels @@ -198,6 +200,28 @@ def _pre_main(self): self.channel_tracker = self.init_channel_tracker() return self.pre_main() + def is_msg_version_compatible(self, message: dict) -> bool: + """ + Check whether the incoming pub/sub message matches this module's + Slips version. + """ + if not message or "data" not in message: + return False + + data = message["data"] + if not isinstance(data, str): + return False + + try: + payload = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError): + return False + + if not isinstance(payload, dict): + return False + + return payload.get("version") == self.slips_version + def get_msg(self, channel: str) -> Optional[dict]: try: if channel not in self.channels: @@ -209,6 +233,12 @@ def get_msg(self, channel: str) -> Optional[dict]: message = self.db.get_message(self.channels[channel]) if utils.is_msg_intended_for(message, channel): + # discard msgs if sent be a newer/older version of slips. + # may happen temporarily during handover, where the new + # version starts before the old version stops. + if not self.is_msg_version_compatible(message): + self.channel_tracker[channel]["msg_received"] = False + return None self.channel_tracker[channel]["msg_received"] = True self.db.incr_msgs_received_in_channel(self.name, channel) return message diff --git a/slips_files/common/parsers/arg_parser.py b/slips_files/common/parsers/arg_parser.py index 7407544590..357d2ee1cb 100644 --- a/slips_files/common/parsers/arg_parser.py +++ b/slips_files/common/parsers/arg_parser.py @@ -14,10 +14,10 @@ def __init__(self, *args, **kwargs): super(ArgumentParser, self).__init__(*args, **kwargs) self.program = {key: kwargs[key] for key in kwargs} - self.alerts_default_path = "output/" - def add_argument(self, *args, **kwargs): super(ArgumentParser, self).add_argument(*args, **kwargs) + if kwargs.get("help") == argparse.SUPPRESS: + return option = {"flags": list(args)} for key in kwargs: option[key] = kwargs[key] @@ -209,8 +209,7 @@ def parse_arguments(self): action="store", metavar="", required=False, - default=self.alerts_default_path, - help="Store alerts.json and alerts.txt in the given folder.", + help="Store Slips logs in the given folder.", ) self.add_argument( "-s", @@ -308,6 +307,14 @@ def parse_arguments(self): action="store_true", help="Internal use only, prevents infinite recursion for cpu profiler dev mode multiprocess tracking", ) + # Internal flag used when Slips starts a newer version of itself. + self.add_argument( + "-u", + dest="is_slips_started_by_an_update", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) try: self.add_argument( "-h", diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index 8ee2768059..c7eba4682f 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -143,6 +143,15 @@ def evidence_detection_threshold(self): def packet_filter(self): return self.read_configuration("parameters", "pcapfilter", False) + def auto_update_slips(self) -> bool: + """ + Read whether live Slips version auto-updates are enabled. + + Returns: + True when automatic live updates are enabled, otherwise False. + """ + return self.read_configuration("update", "auto_update_slips", False) + def online_whitelist(self): return self.read_configuration("whitelists", "online_whitelist", False) diff --git a/slips_files/common/performance_profilers/memory_profiler.py b/slips_files/common/performance_profilers/memory_profiler.py index 30d1b39789..7952ec7953 100644 --- a/slips_files/common/performance_profilers/memory_profiler.py +++ b/slips_files/common/performance_profilers/memory_profiler.py @@ -19,6 +19,8 @@ import random from abc import ABCMeta +from slips_files.common.slips_utils import utils + class MemoryProfiler(IPerformanceProfiler): profiler = None @@ -214,7 +216,7 @@ def _handle_signal(self): # print(f"Msg {msg}") pid: int = None try: - pid = int(msg["data"]) + pid = int(utils.get_msg_payload(msg)) except ValueError: msg = self.pid_channel.get_message(timeout=timeout) continue diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 000d4bd4bc..c629dbb819 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -784,6 +784,36 @@ def is_msg_intended_for(self, message, channel): and message["channel"] == channel ) + def get_msg_payload(self, message: dict) -> Any: + """ + Return the actual payload stored in the given message. discards + metadata like "version" and returns the text only. + + Parameters: + message: Pub/sub message returned by Redis. + + Return: + The decoded payload. Wrapped plain-string messages return their text. + """ + data = message["data"] + if not isinstance(data, str): + return data + + try: + decoded = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError): + return data + + if ( + isinstance(decoded, dict) + and "text" in decoded + and "version" in decoded + and len(decoded) == 2 + ): + return decoded["text"] + + return decoded + def get_slips_version(self) -> str: version_file = "VERSION" with open(version_file, "r") as f: @@ -805,6 +835,34 @@ def change_logfiles_ownership(self, file: str, UID, GID): os.system(f"chown {UID}:{GID} {file}") + def initialize_logfile( + self, + logfile_path: str, + started_by_update: bool, + mode: str = "w", + create_parent_dirs: bool = True, + ) -> bool: + """ + Initialize a log file unless Slips was started by an update. + when slips is being updated , we dont want the new version to clear + the used logfiles, instead it will append to them + + :param logfile_path: path to the log file to initialize. + :param started_by_update: whether Slips was started by an update. + :param mode: file mode used to initialize the log file. + :param create_parent_dirs: whether to create missing parent dirs. + :return: True if the file was initialized, False otherwise. + """ + if started_by_update: + return False + + logfile_dir = os.path.dirname(logfile_path) + if create_parent_dirs and logfile_dir: + os.makedirs(logfile_dir, exist_ok=True) + # clear the logfile + open(logfile_path, mode).close() + return True + def get_ip_identification_as_str(self, ip_identification: dict) -> str: id = "" if "DNS_resolution" in ip_identification: @@ -920,6 +978,11 @@ def assert_microseconds(self, ts: str): ts = ts + "0" * (6 - len(ts.split(".")[-1])) return ts + def get_current_version(self) -> str: + with open("VERSION", "r") as version_file: + current_version = version_file.read().strip() + return current_version + def _convert_str_port_to_int(self, port) -> int: if isinstance(port, str): try: diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index dd04f05151..1c927a6768 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -49,11 +49,27 @@ def __init__( start_redis_server=True, **kwargs, ): + """ + Initialize Redis and SQLite database handlers. + + :param logger: output logger used by database handlers. + :param output_dir: directory where database files and logs are stored. + :param redis_port: port used to connect to Redis. + :param conf: loaded Slips configuration parser. + :param main_pid: PID of the main Slips process. + :param start_sqlite: whether to initialize the SQLite handler. + :param start_redis_server: whether to start Redis before connecting. + + :param kwargs: additional RedisDB options. Supported keys are: + flush_db: whether this manager may flush Redis on startup when + the configuration also allows deleting previous DB contents. + """ self.conf = conf self.output_dir = output_dir self.redis_port = redis_port self.logger = logger self.printer = Printer(self.logger, self.name) + # only the main process should ever flush the Redis DB. to avoid # children overwriting values set at the very start of slips if os.getpid() != main_pid: diff --git a/slips_files/core/database/redis_db/constants.py b/slips_files/core/database/redis_db/constants.py index 9d94a23430..7855733242 100644 --- a/slips_files/core/database/redis_db/constants.py +++ b/slips_files/core/database/redis_db/constants.py @@ -3,7 +3,6 @@ class Constants: LOADED_TI_FILES = "loaded_TI_files_number" TI_FILES_INFO = "TI_files_info" - CURRENT_TIMEWINDOW = "current_timewindow" # all keys starting with IoC_* are used for storing IoCs read from # online and offline TI feeds IOC_IPS = "IoC_ips" diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index b834d1f23b..f661011d0e 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -43,6 +43,7 @@ RUNNING_IN_DOCKER = os.environ.get("IS_IN_A_DOCKER_CONTAINER", False) LOCALHOST = "127.0.0.1" +VERSION = utils.get_current_version() class RedisDB( @@ -154,6 +155,7 @@ def __new__( cls.flush_db = flush_db # start the redis server using cli if it's not started? cls.start_server = start_redis_server + cls.logger = logger cls.printer = Printer(logger, cls.name) cls.conf = ConfigParser() cls.args = cls.conf.get_args() @@ -194,6 +196,15 @@ def __init__(self, *args, **kwargs): self.call_mixins_setup() self._init_ttls() self.set_new_incoming_flows(True) + if ( + self.logger + and not self.flush_db + and self.args.is_slips_started_by_an_update + ): + self.print( + "Continuing on previous data stored in redis.", + log_to_logfiles_only=True, + ) @classmethod def _get_conf_file_path(cls, redis_port: Optional[int] = None) -> str: @@ -507,10 +518,28 @@ def ping(self): self.r.ping() self.rcache.ping() + def _add_version_to_msg(self, msg): + if isinstance(msg, str): + try: + msg = json.loads(msg) + msg.update({"version": VERSION}) + msg = json.dumps(msg) + except json.decoder.JSONDecodeError: + # the msg is 1 str + msg = { + "text": msg, + "version": VERSION, + } + msg = json.dumps(msg) + elif isinstance(msg, dict): + msg.update({"version": VERSION}) + return msg + def publish(self, channel, msg, pipeline=None): """Publish a msg in the given channel. adds the instructions to the given pipeline if given and returns the pipeline""" + msg = self._add_version_to_msg(msg) # keeps track of how many msgs were published in the given channel if pipeline is not None: diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index 4ccdd95de3..f3267fc4b0 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1147,17 +1147,6 @@ def check_tw_to_close(self, close_all=False): pipe = self.delete_past_timewindows(profile_tw_to_close, pipe) pipe.execute() - def get_current_timewindow(self) -> Optional[str]: - """returns the current timewindow if slips is running real-time ( - not pcap/log files)""" - if not self.args.interface: - return - - return self.r.get(self.constants.CURRENT_TIMEWINDOW) - - def set_current_timewindow(self, timewindow: str) -> Optional[str]: - self.r.set(self.constants.CURRENT_TIMEWINDOW, timewindow) - def mark_profile_tw_as_modified(self, modified_tw_details: dict): """ PS: this function should be as optimized as possible, it's the main diff --git a/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py b/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py deleted file mode 100644 index f5f0666e36..0000000000 --- a/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Garcia -# SPDX-License-Identifier: GPL-2.0-only - -import time -from threading import Event - - -def timewindow_updater(db, tw_width: float, stop_event: Event): - """ - runs in a thread, wakes up only to update the current timewindow - number in the db and sleeps again for the whole tw width - """ - while not stop_event.is_set(): - now = time.time() - cur_tw = db.get_timewindow(now, "", add_to_db=False) - db.set_current_timewindow(cur_tw) - # to avoid busy waiting - stop_event.wait(tw_width) diff --git a/slips_files/core/evidence_handler.py b/slips_files/core/evidence_handler.py index 40e8d9cf0c..bd79dff769 100644 --- a/slips_files/core/evidence_handler.py +++ b/slips_files/core/evidence_handler.py @@ -68,6 +68,7 @@ def init(self): output_dir=get_alerts_path_inside_output_dir( self.parent_output_dir ), + slips_args=self.args, ) self.logger_thread = threading.Thread( target=self.evidence_logger.run_logger_thread, diff --git a/slips_files/core/evidence_logger.py b/slips_files/core/evidence_logger.py index e1308aa781..a4f6e53566 100644 --- a/slips_files/core/evidence_logger.py +++ b/slips_files/core/evidence_logger.py @@ -17,10 +17,12 @@ def __init__( logger_stop_signal: threading.Event, evidence_logger_q: multiprocessing.Queue, output_dir: str, + slips_args=None, ): self.logger_stop_signal = logger_stop_signal self.evidence_logger_q = evidence_logger_q self.output_dir = output_dir + self.args = slips_args self.read_configuration() # clear output/alerts.log @@ -66,8 +68,13 @@ def clean_file(self, output_dir, file_to_clean): logfile_dir = os.path.dirname(logfile_path) if logfile_dir: os.makedirs(logfile_dir, exist_ok=True) + if os.path.exists(logfile_path): - open(logfile_path, "w").close() + utils.initialize_logfile( + logfile_path, + getattr(self.args, "is_slips_started_by_an_update", False), + create_parent_dirs=False, + ) return open(logfile_path, "a") def print_to_alerts_logfile(self, data: str): diff --git a/slips_files/core/helpers/bloom_filters_manager.py b/slips_files/core/helpers/bloom_filters_manager.py index a44840a6ee..a15b389e4b 100644 --- a/slips_files/core/helpers/bloom_filters_manager.py +++ b/slips_files/core/helpers/bloom_filters_manager.py @@ -60,7 +60,7 @@ def _init_whitelisted_orgs_bf(self): Updates the bloom filters with the whitelisted organization domains, asns, and ips fills the self.org_filters dict - is called from update_manager whether slips did update its local + is called from feeds_update_manager whether slips did update its local org files or not. this goal of calling this is to make sure slips has the bloom filters in mem at all times. diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index c34169217a..16ba8b438d 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -18,7 +18,7 @@ def get_input_type(self) -> tuple: supported input_type values are: interface, argus, suricata, zeek, nfdump, db supported input_information: - given filepath, interface or type of line given in stdin, + given filepath, interface, or type of line given in stdin, comma separated access point interfaces like wlan0,eth0 """ # only defined in stdin lines @@ -259,7 +259,6 @@ def clear_redis_cache(self): print("Deleting Cache DB in Redis.") self.main.redis_man.clear_redis_cache_database() self.main.input_information = "" - self.main.zeek_dir = "" self.main.redis_man.log_redis_server_pid( redis_cache_default_server_port, redis_cache_server_pid ) diff --git a/slips_files/core/input/cyst/cyst_input.py b/slips_files/core/input/cyst/cyst_input.py index 959482eae8..a823a98c74 100644 --- a/slips_files/core/input/cyst/cyst_input.py +++ b/slips_files/core/input/cyst/cyst_input.py @@ -4,6 +4,7 @@ import json from slips_files.common.abstracts.iinput_handler import IInputHandler +from slips_files.common.slips_utils import utils class CystInput(IInputHandler): @@ -30,13 +31,14 @@ def run(self): # todo when to break? cyst should send something like stop? msg = self.input.get_msg("new_module_flow") - if msg and msg["data"] == "stop_process": + if msg and utils.get_msg_payload(msg) == "stop_process": self.input.shutdown_gracefully() return True if msg := self.input.get_msg("new_module_flow"): - msg: str = msg["data"] - msg = json.loads(msg) + msg = utils.get_msg_payload(msg) + if isinstance(msg, str): + msg = json.loads(msg) flow = msg["flow"] src_module = msg["module"] line_info = { diff --git a/slips_files/core/input/input.py b/slips_files/core/input/input.py index de594dc0ec..6d923bd64e 100644 --- a/slips_files/core/input/input.py +++ b/slips_files/core/input/input.py @@ -24,7 +24,6 @@ from slips_files.common.abstracts.icore import ICore from slips_files.common.input_type import InputType -# common imports for all modules from slips_files.common.parsers.config_parser import ConfigParser import multiprocessing @@ -56,10 +55,10 @@ def init( input_information=None, cli_packet_filter=None, zeek_or_bro=None, - zeek_dir=None, line_type=None, is_profiler_done_event: multiprocessing.Event = None, is_input_done_event: multiprocessing.Event = None, + is_slips_live_updating_event: multiprocessing.Event = None, ): self.input_type = input_type self.profiler_queue = profiler_queue @@ -68,7 +67,6 @@ def init( self.line_type: str = line_type # entire path self.given_path: str = input_information - self.zeek_dir: str = zeek_dir self.zeek_or_bro: str = zeek_or_bro self.read_lines_delay = 0 # when input is done processing, it reeleases this semaphore, that's h @@ -92,6 +90,7 @@ def init( self.is_profiler_done_event = is_profiler_done_event # is set by this proc to indicate no more flows are coming self.is_input_done_event = is_input_done_event + self.is_slips_live_updating_event = is_slips_live_updating_event self.is_running_non_stop: bool = self.db.is_running_non_stop() self.input_handlers = self._build_input_handlers() self.active_handler = None diff --git a/slips_files/core/input/zeek/interface_input.py b/slips_files/core/input/zeek/interface_input.py index 071fb8ffd1..dedb5cfa17 100644 --- a/slips_files/core/input/zeek/interface_input.py +++ b/slips_files/core/input/zeek/interface_input.py @@ -21,8 +21,8 @@ def run(self): """ runs when slips is given an interface with -i or 2 interfaces with -ap """ - self.input.zeek_utils.ensure_zeek_dir() - self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") + zeek_dir: str = self.input.zeek_utils.create_zeek_output_dir() + self.input.print(f"Storing zeek log files in {zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() @@ -35,7 +35,7 @@ def run(self): interfaces_to_monitor.update( { self.input.args.interface: { - "dir": self.input.zeek_dir, + "dir": zeek_dir, "type": "main_interface", } } @@ -46,9 +46,7 @@ def run(self): # interfaces, wifi and eth. for _type, interface in self.db.get_ap_info().items(): # _type can be 'wifi_interface' or "ethernet_interface" - dir_to_store_interface_logs = os.path.join( - self.input.zeek_dir, interface - ) + dir_to_store_interface_logs = os.path.join(zeek_dir, interface) interfaces_to_monitor.update( { interface: { @@ -57,6 +55,7 @@ def run(self): } } ) + for interface, interface_info in interfaces_to_monitor.items(): interface_dir = interface_info["dir"] if not os.path.exists(interface_dir): diff --git a/slips_files/core/input/zeek/pcap_input.py b/slips_files/core/input/zeek/pcap_input.py index 9e1e1833ae..b66ae29f33 100644 --- a/slips_files/core/input/zeek/pcap_input.py +++ b/slips_files/core/input/zeek/pcap_input.py @@ -17,8 +17,8 @@ def run(self): """ runs when slips is given a pcap with -f """ - self.input.zeek_utils.ensure_zeek_dir() - self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") + zeek_dir: str = self.input.zeek_utils.create_zeek_output_dir() + self.input.print(f"Storing zeek log files in {zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() @@ -26,7 +26,7 @@ def run(self): # if bro does not receive any new line while reading a pcap self.input.bro_timeout = 30 self.input.zeek_utils.init_zeek( - self.observer, self.input.zeek_dir, self.input.given_path + self.observer, zeek_dir, self.input.given_path ) self.input.lines = self.input.zeek_utils.read_zeek_files() diff --git a/slips_files/core/input/zeek/utils/dos_protector.py b/slips_files/core/input/zeek/utils/dos_protector.py index f7f4efcae9..7bb802c238 100644 --- a/slips_files/core/input/zeek/utils/dos_protector.py +++ b/slips_files/core/input/zeek/utils/dos_protector.py @@ -94,7 +94,7 @@ def _update_flow_sampling_stop_time_if_needed(self) -> bool: return True return False - def get_number_of_flows_to_skip_and_time_to_stop_sampling(self) -> int: + def get_number_of_flows_to_skip(self) -> int: if not self._should_run(): return 0 diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 7cc9ccc8b8..5b77c3f86f 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -8,6 +8,7 @@ import subprocess import threading import time +from pathlib import Path from typing import List, Tuple from re import split @@ -29,6 +30,10 @@ def __init__(self, input_process): self.zeek_threads = [] self.zeek_pids = [] self.dos_protector = DoSProtector(self.input) + self.args = self.input.args + self.print = self.input.print + self.update_msg_printed = False + self.is_running_non_stop = self.input.db.is_running_non_stop() def check_if_time_to_del_rotated_files(self): """ @@ -139,8 +144,12 @@ def get_ts_from_line(self, zeek_line: str): def cache_nxt_line_in_file(self, filename: str, interface: str): """ - reads 1 line of the given file and stores in queue for sending to the profiler - :param: full path to the file. includes the .log extension + reads 1 line of the given file and stores in queue for sending to the + profiler + + :param filename: full path to the file. includes the .log extension + :param interface: interface that generated the Zeek file + :return: True if a line was cached, False otherwise """ file_handle = self.get_file_handle(filename) if not file_handle: @@ -156,14 +165,34 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): # We don't have any waiting line for this file, so proceed try: flows_to_skip_reading_if_under_heavy_load: int = ( - self.dos_protector.get_number_of_flows_to_skip_and_time_to_stop_sampling() + self.dos_protector.get_number_of_flows_to_skip() ) # skips flows for _ in range(flows_to_skip_reading_if_under_heavy_load): file_handle.readline() - zeek_line = file_handle.readline() + while zeek_line := file_handle.readline(): + if zeek_line.startswith("#close"): + # We reached the end of one of the files that we were + # reading. + return False + + if zeek_line.startswith("#fields"): + # this line contains the zeek fields, we want to cache it + # and send it to the profiler normally + nline = zeek_line + # to send the line as early as possible + timestamp = -1 + break + + timestamp, nline = self.get_ts_from_line(zeek_line) + if timestamp: + break + else: + # We reached the end of one of the files that we were reading. + # Wait for more lines to come from another file. + return False except ValueError: # remover thread just finished closing all old handles. @@ -171,23 +200,6 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): # to get the new dict of open handles. return False - # Did the file end? - if not zeek_line or zeek_line.startswith("#close"): - # We reached the end of one of the files that we were reading. - # Wait for more lines to come from another file - return False - - if zeek_line.startswith("#fields"): - # this line contains the zeek fields, we want to cache it and - # send it to the profiler normally - nline = zeek_line - # to send the line as early as possible - timestamp = -1 - else: - timestamp, nline = self.get_ts_from_line(zeek_line) - if not timestamp: - return False - self.file_time[filename] = timestamp # Store the line in the cache self.cache_lines[filename] = { @@ -251,6 +263,15 @@ def get_earliest_line(self): earliest_line = self.cache_lines[file_with_earliest_flow] return earliest_line, file_with_earliest_flow + def _print_update_msg(self): + if not self.update_msg_printed: + self.print( + "Slips is live updating. Slips will stop receiving new " + "flows in this instance and start receiving new flows using " + "the updated version. " + ) + self.update_msg_printed = True + def read_zeek_files(self) -> int: """ Runs when slips is analyzing pcaps, interface, zeek dirs, and zeek @@ -266,8 +287,26 @@ def read_zeek_files(self) -> int: # Try to keep track of when was the last update so we stop this # reading self.last_updated_file_time = datetime.datetime.now() - while not self.input.should_stop(): + is_draining = False + while True: + is_live_updating = ( + self.input.is_slips_live_updating_event is not None + and self.input.is_slips_live_updating_event.is_set() + ) + if is_live_updating and not is_draining: + # Stop Zeek first so this instance has a finite set of + # generated logs to drain before the updated instance + # continues. + self._print_update_msg() + self.shutdown_zeek_runtime() + self.zeek_files = self.input.db.get_all_zeek_files() + is_draining = True + + if self.input.should_stop() and not is_draining: + break + self.check_if_time_to_del_rotated_files() + # implemented in icore.py self.input.store_flows_read_per_second() @@ -277,13 +316,27 @@ def read_zeek_files(self) -> int: # PS: self.zeek_files ties each zeek file to its interface ( # beacause slips supports reading multiple interfaces) + if is_draining: + self.zeek_files = self.input.db.get_all_zeek_files() + + cached_new_line = False for filename, interface in self.zeek_files.items(): if utils.is_ignored_zeek_log_file(filename): continue # reads 1 line from the given file and cache it # from in self.cache_lines - self.cache_nxt_line_in_file(filename, interface) + if self.cache_nxt_line_in_file(filename, interface): + self.last_updated_file_time = datetime.datetime.now() + cached_new_line = True + + if ( + is_draining + and not cached_new_line + and not self.cache_lines + ): + # done draining the flows left + break if self.reached_timeout(): break @@ -295,29 +348,58 @@ def read_zeek_files(self) -> int: continue # self.print('\t> Sent Line: {}'.format(earliest_line), 0, 3) - self.input.give_profiler(earliest_line) self.input.lines += 1 - # when testing, no need to read the whole file! + + # when testing, no need to read the whole file! #TODO this + # is bad practice, fix it if self.input.lines == 10 and self.input.testing: break + # Delete this line from the cache and the time list del self.cache_lines[file_with_earliest_flow] del self.file_time[file_with_earliest_flow] - # Get the new list of files. Since new files may have been created by - # Zeek while we were processing them. + # Get the new list of files. Since new files may have been + # created by Zeek while we were processing them. self.zeek_files = self.input.db.get_all_zeek_files() - self.close_all_handles() except KeyboardInterrupt: pass return self.input.lines - def ensure_zeek_dir(self): - if not os.path.exists(self.input.zeek_dir): - os.makedirs(self.input.zeek_dir) + def _is_auto_update_enabled(self) -> bool: + """ + returns true if slips is analyzing an interface (-i , -ap or -g) + and auto_update is enabled in slips.yaml + """ + return self.is_running_non_stop and self.input.conf.auto_update_slips() + + def create_zeek_output_dir(self) -> str: + """ + Return the Zeek output directory, create it if needed, + and store its path in the DB. + + :return: Directory where Zeek should write log files. + """ + + without_ext = Path(self.input.given_path).stem + if self.input.conf.store_zeek_files_in_the_output_dir(): + zeek_dir = Path(self.input.args.output) / "zeek_files" + else: + zeek_dir = Path(f"zeek_files_{without_ext}") + + if self._is_auto_update_enabled(): + # slips is gonna be auto updating for each new version, we need + # 1 zeek dir for each started version + zeek_dir = Path(zeek_dir) / f"slips_v{utils.get_current_version()}" + + zeek_dir = str(zeek_dir) + + Path(zeek_dir).mkdir(parents=True, exist_ok=True) + self.input.db.set_input_metadata({"zeek_dir": zeek_dir}) + return zeek_dir def init_zeek( self, @@ -336,7 +418,7 @@ def init_zeek( observer.start(zeek_dir, pcap_or_interface) zeek_files = os.listdir(zeek_dir) - if len(zeek_files) > 0: + if len(zeek_files) > 0 and not self.args.is_slips_started_by_an_update: # First clear the zeek folder of old .log files for file_name in zeek_files: os.remove(os.path.join(zeek_dir, file_name)) @@ -403,7 +485,8 @@ def run_zeek(self, zeek_logs_dir, pcap_or_interface, tcpdump_filter=None): print(f"Zeek: {out}") if error: self.input.print( - f"Zeek error. return code: {zeek.returncode} error:{error.strip()}" + f"Zeek error. return code: {zeek.returncode} " + f"error:{error.strip()}" ) def shutdown_zeek_runtime(self): diff --git a/slips_files/core/input/zeek/zeek_dir_input.py b/slips_files/core/input/zeek/zeek_dir_input.py index 5efa851b05..b8acea1c21 100644 --- a/slips_files/core/input/zeek/zeek_dir_input.py +++ b/slips_files/core/input/zeek/zeek_dir_input.py @@ -38,9 +38,7 @@ def run(self): # know the interface, hence the "default" interface = "default" - self.input.zeek_dir = self.input.given_path - - self.observer.start(self.input.zeek_dir, interface) + self.observer.start(self.input.given_path, interface) if self.input.is_running_non_stop: self.file_remover.start() diff --git a/slips_files/core/output.py b/slips_files/core/output.py index 423da0c69f..01a863b82b 100644 --- a/slips_files/core/output.py +++ b/slips_files/core/output.py @@ -48,6 +48,7 @@ def __init__( input_type=False, create_logfiles: bool = True, stdout="", + slips_args=None, ): super().__init__() # when running slips using -e , this var is set and we only @@ -58,6 +59,7 @@ def __init__( self.input_type = input_type self.errors_logfile = stderr self.slips_logfile = slips_logfile + self.args = slips_args if self.verbose > 2: print(f"Verbosity: {self.verbose}. Debugging: {self.debug}") @@ -71,10 +73,10 @@ def __init__( # root (if slips was started by root) os.umask(0) self._read_configuration() - self.create_logfile(self.errors_logfile) - self.log_branch_info(self.errors_logfile) - self.create_logfile(self.slips_logfile) - self.log_branch_info(self.slips_logfile) + if self.create_logfile(self.errors_logfile): + self.log_branch_info(self.errors_logfile) + if self.create_logfile(self.slips_logfile): + self.log_branch_info(self.slips_logfile) utils.change_logfiles_ownership( self.errors_logfile, self.UID, self.GID @@ -112,13 +114,20 @@ def log_branch_info(self, logfile: str): def create_logfile(self, path): """ creates slips.log and errors.log if they don't exist + :return: True if the file was initialized, False otherwise. """ + if getattr(self.args, "is_slips_started_by_an_update", False) is True: + return False + try: open(path, "a").close() + return True except FileNotFoundError: p = Path(os.path.dirname(path)) p.mkdir(parents=True, exist_ok=True) - open(path, "w").close() + return utils.initialize_logfile( + path, False, create_parent_dirs=False + ) def log_line(self, msg: dict): """ diff --git a/tests/integration/config/iris_peer_config.yaml b/tests/integration/config/iris_peer_config.yaml deleted file mode 100644 index d1bf3e4777..0000000000 --- a/tests/integration/config/iris_peer_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -Identity: - GenerateNewKey: true - KeyFile: second.priv -PeerDiscovery: - DisableBootstrappingNodes: false - ListOfMultiAddresses: - - /ip4/127.0.0.1/udp/9010/quic 12D3KooWNKe9xFM85HkWsoGAy4pM3kXrZHGumJwQBBr9T4VzHtNR -Redis: - Host: 127.0.0.1 - Port: 6655 - Tl2NlChannel: iris_internal -Server: - DhtServerMode: 'true' - Host: null - Port: 9006 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000000..ac91ed8e2d --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import importlib.util +import pytest + + +if importlib.util.find_spec("termcolor") is None: + pytest.skip( + "termcolor is required to run integration tests that invoke slips", + allow_module_level=True, + ) diff --git a/tests/integration/test_config_files/__init__.py b/tests/integration/test_config_files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/config/test.yaml b/tests/integration/test_config_files/config_test.yaml similarity index 99% rename from tests/integration/config/test.yaml rename to tests/integration/test_config_files/config_test.yaml index aca544389f..9c8f739576 100644 --- a/tests/integration/config/test.yaml +++ b/tests/integration/test_config_files/config_test.yaml @@ -64,7 +64,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true parameters: analysis_direction: all diff --git a/tests/integration/test.yaml b/tests/integration/test_config_files/test.yaml similarity index 99% rename from tests/integration/test.yaml rename to tests/integration/test_config_files/test.yaml index fb64279b53..d44e29e3c9 100644 --- a/tests/integration/test.yaml +++ b/tests/integration/test_config_files/test.yaml @@ -90,7 +90,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true output: logs: slips.log diff --git a/tests/integration/test2.yaml b/tests/integration/test_config_files/test2.yaml similarity index 99% rename from tests/integration/test2.yaml rename to tests/integration/test_config_files/test2.yaml index 009bb60b1d..fd16490916 100644 --- a/tests/integration/test2.yaml +++ b/tests/integration/test_config_files/test2.yaml @@ -89,7 +89,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true output: logs: slips.log diff --git a/tests/integration/test_config_files.py b/tests/integration/test_config_files/test_config_files.py similarity index 89% rename from tests/integration/test_config_files.py rename to tests/integration/test_config_files/test_config_files.py index 6c6b5c055c..83fc3fbbc0 100644 --- a/tests/integration/test_config_files.py +++ b/tests/integration/test_config_files/test_config_files.py @@ -5,8 +5,6 @@ test/test.yaml and tests/test2.yaml """ -from slips.main import Main -from slips_files.common.input_type import InputType from tests.common_test_utils import ( is_evidence_present, create_output_dir, @@ -22,17 +20,10 @@ import pytest import shutil import os +from pathlib import Path alerts_file = "alerts.log" - - -def create_main_instance(input_information): - """returns an instance of Main() class in slips.py""" - main = Main(testing=True) - main.input_information = input_information - main.input_type = InputType.PCAP - main.line_type = False - return main +TEST_DIR = Path(__file__).resolve().parent @pytest.mark.parametrize( @@ -55,10 +46,10 @@ def test_conf_file(pcap_path, expected_profiles, output_dir, redis_port): binaries=("redis-server",), require_zeek_or_bro=True, ) - config_file = "tests/integration/test.yaml" + config_file = TEST_DIR / "test.yaml" modify_yaml_config( - output_filename=config_file, - output_dir=os.getcwd(), + output_filename=config_file.name, + output_dir=TEST_DIR, changes={ "DisabledAlerts": { "disabled_detections": ["ConnectionWithoutDNS"] @@ -77,7 +68,7 @@ def test_conf_file(pcap_path, expected_profiles, output_dir, redis_port): "template", "ensembling", "flow_ml_detection", - "update_manager", + "feeds_update_manager", ] }, }, @@ -157,10 +148,10 @@ def test_conf_file2(pcap_path, expected_profiles, output_dir, redis_port): binaries=("redis-server",), require_zeek_or_bro=True, ) - config_file = "tests/integration/test2.yaml" + config_file = TEST_DIR / "test2.yaml" modify_yaml_config( - output_filename=config_file, - output_dir=os.getcwd(), + output_filename=config_file.name, + output_dir=TEST_DIR, changes={ "detection": {"evidence_detection_threshold": 0.1}, "parameters": { @@ -172,7 +163,7 @@ def test_conf_file2(pcap_path, expected_profiles, output_dir, redis_port): "template", "ensembling", "flow_ml_detection", - "update_manager", + "feeds_update_manager", ] }, }, diff --git a/tests/integration/test_dataset/__init__.py b/tests/integration/test_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset/test_dataset.py similarity index 100% rename from tests/integration/test_dataset.py rename to tests/integration/test_dataset/test_dataset.py diff --git a/tests/integration/test_fides/__init__.py b/tests/integration/test_fides/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/config/fides.conf.yml b/tests/integration/test_fides/fides.conf.yml similarity index 100% rename from tests/integration/config/fides.conf.yml rename to tests/integration/test_fides/fides.conf.yml diff --git a/tests/integration/config/fides_config.yaml b/tests/integration/test_fides/fides_config.yaml similarity index 99% rename from tests/integration/config/fides_config.yaml rename to tests/integration/test_fides/fides_config.yaml index 734d227b3f..00618fd6fb 100644 --- a/tests/integration/config/fides_config.yaml +++ b/tests/integration/test_fides/fides_config.yaml @@ -206,7 +206,7 @@ virustotal: ############################# threatintelligence: - # by default, slips starts without the TI files, and runs the update_manager in the background + # by default, slips starts without the TI files, and runs the feeds_update_manager in the background # if thi option is set to yes, slips will not start untill the update manager is done # and all TI files are loaded successfully # this is usefull if you want to ensure that slips doesn't miss the detection of any blacklisted IPs diff --git a/tests/integration/test_fides.py b/tests/integration/test_fides/test_fides.py similarity index 98% rename from tests/integration/test_fides.py rename to tests/integration/test_fides/test_fides.py index 242db07289..4eeeab3f3b 100644 --- a/tests/integration/test_fides.py +++ b/tests/integration/test_fides/test_fides.py @@ -30,6 +30,7 @@ alerts_file = "alerts.log" +TEST_DIR = Path(__file__).resolve().parent def ensure_redis_is_running(port): @@ -166,8 +167,7 @@ def get_main_interface(): ) def test_conf_file2(path, output_dir, redis_port): """ - In this test we're using tests/integration/config/fides_config.yaml as fides - config file + In this test we're using the local fides integration config file. """ ensure_redis_is_running(redis_port) output_dir: PosixPath = create_output_dir(output_dir) @@ -185,7 +185,7 @@ def test_conf_file2(path, output_dir, redis_port): "-o", str(output_dir), "-c", - "tests/integration/config/fides_config.yaml", + str(TEST_DIR / "fides_config.yaml"), "-P", str(redis_port), ] @@ -277,7 +277,7 @@ def test_trust_recommendation_response(path, output_dir, redis_port): "-o", str(output_dir), "-c", - "tests/integration/config/fides_config.yaml", + str(TEST_DIR / "fides_config.yaml"), "-P", str(redis_port), ] diff --git a/tests/integration/test_iris.py b/tests/integration/test_iris.py deleted file mode 100644 index 332cfc2f0e..0000000000 --- a/tests/integration/test_iris.py +++ /dev/null @@ -1,345 +0,0 @@ -""" -This file tests 2 different config files other than slips' default config/slips.yaml -test/test.yaml and tests/test2.yaml -""" - -import re -import shutil -from pathlib import PosixPath - -import redis - -from tests.common_test_utils import ( - create_output_dir, - assert_no_errors, - modify_yaml_config, -) -import pytest -import os -import subprocess -import time -import sys - -alerts_file = "alerts.log" - - -def countdown(seconds, message): - """ - counts down from the given number of seconds, printing a message each second. - """ - while seconds > 0: - sys.stdout.write( - f"\rSending {message} in {seconds} " - ) # overwrite the line - sys.stdout.flush() # ensures immediate output - time.sleep(1) # waits for 1 second - seconds -= 1 - sys.stdout.write(f"\rSending {message} now! \n") - - -def message_send(port, channel, message): - # connect to redis database 0 - redis_client = redis.StrictRedis(host="localhost", port=port, db=0) - - # publish the message to the "network2fides" channel - redis_client.publish(channel, message) - - print(f"Test message published to channel '{channel}'.") - - -message_alert_TL_NL = """{ - "type": "tl2nl_alert", - "version": 1, - "data": { - "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - } -}""" - - -message_alert_NL_S = """{ - "type": "nl2tl_alert", - "version": 1, - "data": - "sender": "" - "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" -}""" - - -def check_strings_in_file(string_list, file_path): - # Check if the file exists - if not os.path.exists(file_path): - print(f"File {file_path} does not exist.") - return False - - # Open the file and read its content - try: - with open(file_path, "r") as file: - file_content = file.read() - - # Check if all strings in the list are present in the file content - for string in string_list: - if string not in file_content: - return False - return True - - except Exception as e: - print(f"Error reading file: {e}") - return False - - -def wait_for_file(file_path, timeout_seconds): - """ - Wait until a file exists or the timeout elapses. - - Parameters: - file_path: Path to the expected file. - timeout_seconds: Maximum number of seconds to wait. - - Returns: - True if the file exists before the timeout, otherwise False. - """ - deadline = time.time() + timeout_seconds - while time.time() < deadline: - if os.path.exists(file_path): - return True - time.sleep(1) - return os.path.exists(file_path) - - -def get_default_interface(): - with open("/proc/net/route") as f: - for line in f.readlines()[1:]: - fields = line.strip().split() - if fields[1] == "00000000": # default route - return fields[0] - - -@pytest.mark.parametrize( - "zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port", - [ - ( - "dataset/test13-malicious-dhcpscan-zeek-dir", - "iris_integration_test/", - "peer_iris_integration_test/", - 6644, - 6655, - ) - ], -) -def test_messaging( - zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port -): - """ - Tests whether Iris properly distributes an alert message generated by - Slips to the network (~other peers). - - First Slips instance is a general node in the network, its connection - string is generated and extracted from logs as a normal user would do, - in a very standard use case. - - The second instance of Slips acts as a normal-user-peer that joins the - network via the aforementioned Slips instance, - which extends the standard use case of connecting to such P2P network. - """ - # Two Slips instances are necessary to be run in this test. - default_interface = get_default_interface() - - # Prepare output dir for the main Slips instance. - # The logs of both beers will be clearly separated and kept intact. - output_dir: PosixPath = create_output_dir(output_dir) - output_file = os.path.join(output_dir, "slips_output.txt") - - # Prepare output dir for the main Slips instance. - # The logs of both beers will be clearly separated and kept intact. - output_dir_peer: PosixPath = create_output_dir(peer_output_dir) - output_file_peer = os.path.join(output_dir_peer, "slips_output.txt") - - # this will be used for the extraction of the connection string form the - # logs of iris running under the main Slips - log_file_first_iris = output_dir / "iris/iris_logs.txt" - log_file_second_iris = output_dir_peer / "iris/iris_logs.txt" - - # generate config of first peer - slips_iris_main_config_file = ( - "tests/integration/config/slips_iris_main.yaml" - ) - modify_yaml_config( - input_path="config/slips.yaml", - output_dir=os.path.dirname(slips_iris_main_config_file), - output_filename=os.path.basename(slips_iris_main_config_file), - changes={ - "global_p2p": {"use_global_p2p": True}, - "modules": {"disable": ["template", "updatemanager"]}, - }, - ) - - # that config file will be generated later to be able to add the first - # peer's id to it - iris_peer_config_file = "tests/integration/config/iris_peer_config.yaml" - - # generate config of second peer - iris_config_file = "tests/integration/config/iris_config.yaml" - modify_yaml_config( - input_path="config/slips.yaml", - output_dir=os.path.dirname(iris_config_file), - output_filename=os.path.basename(iris_config_file), - changes={ - "global_p2p": { - "use_global_p2p": True, - "iris_conf": iris_peer_config_file, - }, - "modules": {"disable": ["template", "updatemanager"]}, - }, - ) - - print("running slips ...") - with open(output_file, "w") as log_file: - with open(output_file_peer, "w") as iris_log_file: - # Start the subprocess, redirecting stdout and stderr - # to the same file - # command for the main Slips instance - command = [ - sys.executable, - "./slips.py", - "-t", - "-g", - str(zeek_dir_path), - # dummy interface required by -g - "-i", - default_interface, - "-e", - "1", - "-o", - str(output_dir), - "-c", - slips_iris_main_config_file, # we're using the dafult peer - # config here - "-P", - str(redis_port), - ] - process = subprocess.Popen( - command, - stdout=log_file, - stderr=log_file, - ) - - # First peer (its Iris) needs to be ready and available for - # connections when the second peer tries to reach out to it. - countdown(20, "second peer") - # get the connection string from the first peer and give it - # to the second one so it is reachable - assert wait_for_file( - log_file_first_iris, 30 - ), f"Expected Iris log file was not created: {log_file_first_iris}" - with open(log_file_first_iris, "r") as log: - for line in log: - match = re.search(r"connection string:\s+'(.+)'", line) - if match: - original_conn_string = match.group(1) - break - else: - # if it comes here make sure that port 9010 used by - # iris is free - # sudo lsof -i :9010 - # sudo kill -9 - print("No connection string found in log file.") - exit(1) - - # put the PeerID in the config file of the second peer's Iris - # the goal is for the second iris to be able to find the first - # iris/slips - modify_yaml_config( - input_path="config/iris_config.yaml", - output_dir=os.path.dirname(iris_peer_config_file), - output_filename=os.path.basename(iris_peer_config_file), - changes={ - "Redis": {"Port": 6655}, - "Server": {"Port": 9006}, - "PeerDiscovery": { - "ListOfMultiAddresses": [original_conn_string] - }, - "Identity": {"KeyFile": "second.priv"}, - }, - ) - # generate a second command for the second peer - peer_command = [ - sys.executable, - "./slips.py", - "-t", - "-g", - str(zeek_dir_path), - # dummy interface required by -g - "-i", - default_interface, - "-e", - "1", - "-o", - str(output_dir_peer), - "-c", - iris_config_file, # we're not using the dafult peer config - # here - "-P", - str(peer_redis_port), - ] - peer_process = subprocess.Popen( - peer_command, stdout=iris_log_file, stderr=iris_log_file - ) - - print( - f"Output and errors of first peer are logged in" - f" {output_file}" - ) - - # let Slips properly and fully star with all of its parts and modules. - countdown(80, "Sending msg in fides2network") - # Sending a manual message to make sure there is an alert generated, because - # is is highly probable that both slips have covered their network captures - # before the infrastructure of P2P network was fully up and running - message_send( - redis_port, - message=message_alert_TL_NL, - channel="fides2network", - ) - - # these seconds are the time we give slips to process the msg - countdown(30, "Sending SIGTERM to the 2 peers") - # Kill em with kindness. - os.kill(process.pid, 15) - os.kill(peer_process.pid, 15) - print("SIGTERM sent.") - - print("Sending SIGKILL to the 2 instances of Slips + iris") - # Kill em. Without kindness. - os.kill(process.pid, 9) - print(f"Slips with PID {process.pid} was killed.") - - os.kill(peer_process.pid, 9) - print(f"Slips peer with PID {peer_process.pid} was killed.") - - print("Slips is done, checking for errors in the 2 output dirs.") - assert_no_errors(output_dir) - assert_no_errors(output_dir_peer) - - print("Checking for iris expected logs in the generated log file") - # make sure this string is there in the generated iris logs - # this is how we ensure that iris ran correctly - expected_log_entry = [ - "INFO iris protocols/alert.go:111 received p2p alert message" - ] - assert check_strings_in_file(expected_log_entry, log_file_second_iris) - - print("Deleting the output directories") - shutil.rmtree(output_dir) - shutil.rmtree(output_dir_peer) - os.remove("modules/iris/second.priv") - modify_yaml_config( - input_path="config/iris_config.yaml", - output_dir=os.path.dirname(iris_peer_config_file), - output_filename=os.path.basename(iris_peer_config_file), - changes={ - "Redis": {"Port": 6644}, - "Server": {"Port": 9010}, - "PeerDiscovery": {}, - "Identity": {"KeyFile": "private.key"}, - }, - ) diff --git a/tests/integration/test_iris/__init__.py b/tests/integration/test_iris/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_iris/peer1_config/iris.yaml b/tests/integration/test_iris/peer1_config/iris.yaml new file mode 100644 index 0000000000..e623c04475 --- /dev/null +++ b/tests/integration/test_iris/peer1_config/iris.yaml @@ -0,0 +1,13 @@ +Identity: + GenerateNewKey: true +PeerDiscovery: + DisableBootstrappingNodes: true + ListOfMultiAddresses: [] +Redis: + Host: 127.0.0.1 + Port: 6644 + Tl2NlChannel: iris_internal +Server: + DhtServerMode: 'true' + Host: null + Port: 9010 diff --git a/tests/integration/config/slips_iris_main.yaml b/tests/integration/test_iris/peer1_config/slips.yaml similarity index 96% rename from tests/integration/config/slips_iris_main.yaml rename to tests/integration/test_iris/peer1_config/slips.yaml index d095cc84b4..81977b53a2 100644 --- a/tests/integration/config/slips_iris_main.yaml +++ b/tests/integration/test_iris/peer1_config/slips.yaml @@ -79,7 +79,7 @@ global_p2p: - fides - iris bootstrapping_node: false - iris_conf: config/iris_config.yaml + iris_conf: tests/integration/test_iris/peer1_config/iris.yaml use_global_p2p: true local_p2p: create_p2p_logfile: false @@ -97,7 +97,7 @@ parameters: analysis_direction: out client_ips: [] debug: 0 - default_rotation_interval: 30sec + default_rotation_interval: 1day deletePrevdb: true delete_zeek_files: false export_format: json @@ -127,6 +127,8 @@ threatintelligence: ssl_feeds: config/SSL_feeds.csv ti_files: config/TI_feeds.csv wait_for_TI_to_finish: false +update: + auto_update_slips: false virustotal: api_key_file: config/vt_api_key virustotal_update_period: 259200 diff --git a/tests/integration/config/iris_config.yaml b/tests/integration/test_iris/peer2_config/iris.yaml similarity index 86% rename from tests/integration/config/iris_config.yaml rename to tests/integration/test_iris/peer2_config/iris.yaml index 76625208fc..04dbe81985 100644 --- a/tests/integration/config/iris_config.yaml +++ b/tests/integration/test_iris/peer2_config/iris.yaml @@ -1,14 +1,15 @@ Identity: GenerateNewKey: true -Server: - Port: 9010 - Host: 0.0.0.0 - DhtServerMode: 'true' -Redis: - Host: 127.0.0.1 - Port: 6379 - Tl2NlChannel: iris_internal + KeyFile: private.key PeerDiscovery: DisableBootstrappingNodes: false ListOfMultiAddresses: - /dns/melchior.slips.stratosphere.fel.cvut.cz/udp/6437/quic 12D3KooWJJa9PpMFVP7s3TQs2sedypJXxtMVkphRhgkjGH9EYMfM +Redis: + Host: 127.0.0.1 + Port: 6644 + Tl2NlChannel: iris_internal +Server: + DhtServerMode: 'true' + Host: null + Port: 9010 diff --git a/tests/integration/test_iris/peer2_config/slips.yaml b/tests/integration/test_iris/peer2_config/slips.yaml new file mode 100644 index 0000000000..0a03b12de7 --- /dev/null +++ b/tests/integration/test_iris/peer2_config/slips.yaml @@ -0,0 +1,142 @@ +Debug: + generate_performance_plots: false +DisabledAlerts: + disabled_detections: [] +Docker: + GID: 0 + UID: 0 +Profiling: + cpu_profiler_dev_mode_entries: 500000 + cpu_profiler_enable: false + cpu_profiler_mode: dev + cpu_profiler_multiprocess: true + cpu_profiler_output_limit: 20 + cpu_profiler_sampling_interval: 20 + memory_profiler_enable: false + memory_profiler_mode: live + memory_profiler_multiprocess: true +anomaly_detection_https: + adaptation_score_threshold: 2.0 + adwin_clock: 1 + adwin_delta: 0.01 + adwin_grace_period: 5 + adwin_min_window_length: 5 + baseline_alpha: 0.5 + drift_alpha: 0.05 + empirical_threshold_quantile: 0.995 + flow_zscore_threshold: 3.5 + hourly_zscore_threshold: 3.0 + ja3_min_variants_per_server: 3 + log_verbosity: 3 + max_small_flow_anomalies: 1 + min_baseline_points: 6 + suspicious_alpha: 0.005 + training_alpha: 1 + training_fit_method: welford + training_hours: 2 + use_adwin_drift: true +brute_force_detector: + ssh_attempt_threshold: 9 +cesnet: + configuration_file: config/warden.conf + receive_alerts: false + receive_delay: 86400 + send_alerts: false +detection: + evidence_detection_threshold: 0.25 + popup_alerts: false +exporting_alerts: + TAXII_server: localhost + collection_name: Alerts + direct_export: true + direct_export_max_workers: 12 + direct_export_retry_backoff: 0.5 + direct_export_retry_max: 0 + direct_export_retry_max_delay: 5.0 + direct_export_workers: 4 + discovery_path: /taxii2/ + export_to: [] + port: 1234 + push_delay: 3600 + sensor_name: sensor1 + slack_api_path: config/slack_bot_token_secret + slack_channel_name: proj_slips_alerting_module + taxii_password: changeme_before_installing_a_medallion_server + taxii_timeout: 10 + taxii_username: admin + taxii_version: 2 + use_https: false +flow_alerts: + data_exfiltration_threshold: 500 + entropy_threshold: 5 + long_connection_threshold: 1500 + pastebin_download_threshold: 700 + ssh_succesful_detection_threshold: 4290 +flow_ml_detection: + mode: test +global_p2p: + bootstrapping_modules: + - fides + - iris + bootstrapping_node: false + iris_conf: tests/integration/test_iris/peer2_config/iris.yaml + use_global_p2p: true +local_p2p: + create_p2p_logfile: false + use_p2p: false +modules: + disable: + - template + - updatemanager + timeline_human_timestamp: true +output: + logs: slips.log + stderr: errors.log + stdout: slips.log +parameters: + analysis_direction: out + client_ips: [] + debug: 0 + default_rotation_interval: 1day + deletePrevdb: true + delete_zeek_files: false + export_format: json + export_labeled_flows: false + export_strato_letters: false + keep_rotated_files_for: 1 day + label: normal + metadata_dir: true + pcapfilter: false + permanent_dir: permanent + rotation: true + store_a_copy_of_zeek_files: false + store_zeek_files_in_the_output_dir: true + tcp_inactivity_timeout: 60 + time_window_width: 3600 + verbose: 1 + wait_for_modules_to_finish: 10080 mins +threatintelligence: + RiskIQ_credentials_path: config/RiskIQ_credentials + TI_files_update_period: 86400 + download_path_for_remote_threat_intelligence: modules/threat_intelligence/remote_data_files/ + ja3_feeds: config/JA3_feeds.csv + local_threat_intelligence_files: config/local_ti_files/ + mac_db: https://maclookup.app/downloads/json-database/get-db + mac_db_update: 1209600 + riskiq_update_period: 604800 + ssl_feeds: config/SSL_feeds.csv + ti_files: config/TI_feeds.csv + wait_for_TI_to_finish: false +update: + auto_update_slips: false +virustotal: + api_key_file: config/vt_api_key + virustotal_update_period: 259200 +web_interface: + port: 55000 +whitelists: + enable_local_whitelist: true + enable_online_whitelist: true + local_whitelist_path: config/whitelist.conf + online_whitelist: https://tranco-list.eu/download/X5QNN/10000 + online_whitelist_update_period: 86400 diff --git a/tests/integration/test_iris/test_iris.py b/tests/integration/test_iris/test_iris.py new file mode 100644 index 0000000000..2ba978b4c9 --- /dev/null +++ b/tests/integration/test_iris/test_iris.py @@ -0,0 +1,528 @@ +""" +This file tests 2 different config files other than slips' default config/slips.yaml +test/test.yaml and tests/test2.yaml +""" + +import re +import shutil +from pathlib import PosixPath + +import redis + +from tests.common_test_utils import ( + create_output_dir, + assert_no_errors, + modify_yaml_config, +) +import pytest +import os +import subprocess +import time +import sys +from pathlib import Path + +alerts_file = "alerts.log" +TEST_DIR = Path(__file__).resolve().parent +PEER1_CONFIG_DIR = TEST_DIR / "peer1_config" +PEER2_CONFIG_DIR = TEST_DIR / "peer2_config" + + +def countdown(seconds, message): + """ + counts down from the given number of seconds, printing a message each second. + """ + while seconds > 0: + sys.stdout.write( + f"\rSending {message} in {seconds} " + ) # overwrite the line + sys.stdout.flush() # ensures immediate output + time.sleep(1) # waits for 1 second + seconds -= 1 + sys.stdout.write(f"\rSending {message} now! \n") + + +def message_send(port, channel, message): + # connect to redis database 0 + redis_client = redis.StrictRedis(host="localhost", port=port, db=0) + + # publish the message to the "network2fides" channel + redis_client.publish(channel, message) + + print(f"Test message published to channel '{channel}'.") + + +message_alert_TL_NL = """{ + "type": "tl2nl_alert", + "version": 1, + "data": { + "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + } +}""" + + +message_alert_NL_S = """{ + "type": "nl2tl_alert", + "version": 1, + "data": + "sender": "" + "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" +}""" + + +def check_strings_in_file(string_list, file_path): + # Check if the file exists + if not os.path.exists(file_path): + print(f"File {file_path} does not exist.") + return False + + # Open the file and read its content + try: + with open(file_path, "r") as file: + file_content = file.read() + + # Check if all strings in the list are present in the file content + for string in string_list: + if string not in file_content: + return False + return True + + except Exception as e: + print(f"Error reading file: {e}") + return False + + +def wait_for_file(file_path, timeout_seconds): + """ + Wait until a file exists or the timeout elapses. + + Parameters: + file_path: Path to the expected file. + timeout_seconds: Maximum number of seconds to wait. + + Returns: + True if the file exists before the timeout, otherwise False. + """ + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if os.path.exists(file_path): + return True + time.sleep(1) + return os.path.exists(file_path) + + +def get_default_interface(): + """ + Get the default network interface. + + Returns: + str: Name of the default network interface. + """ + with open("/proc/net/route") as f: + for line in f.readlines()[1:]: + fields = line.strip().split() + if fields[1] == "00000000": # default route + return fields[0] + + +def extract_connection_string(log_file_first_iris): + """ + Extract the first peer connection string from the Iris log file. + + Parameters: + log_file_first_iris: Path to the first peer Iris log file. + + Returns: + str: The extracted connection string. + """ + with open(log_file_first_iris, "r") as log: + for line in log: + match = re.search(r"connection string:\s+'(.+)'", line) + if match: + return match.group(1) + + print("No connection string found in log file.") + exit(1) + + +def assert_peer1_setup(peer1_slips_config): + """ + Assert that the first peer config was generated as expected. + + Parameters: + peer1_slips_config: Path to the first peer Slips config. + + Returns: + None + """ + assert check_strings_in_file( + ["iris_conf: tests/integration/test_iris/peer1_config/iris.yaml"], + peer1_slips_config, + ) + + +def assert_peer2_setup( + peer2_slips_config, + peer2_iris_config, + connection_string, + peer_redis_port, +): + """ + Assert that the second peer config was generated as expected. + + Parameters: + peer2_slips_config: Path to the second peer Slips config. + peer2_iris_config: Path to the second peer Iris config. + connection_string: Multiaddress used to connect to the first peer. + peer_redis_port: Redis port used by the second peer. + + Returns: + None + """ + assert check_strings_in_file( + ["iris_conf: tests/integration/test_iris/peer2_config/iris.yaml"], + peer2_slips_config, + ) + assert check_strings_in_file( + [ + f"Port: {peer_redis_port}", + "Port: 9006", + "DisableBootstrappingNodes: true", + "KeyFile: second.priv", + connection_string, + ], + peer2_iris_config, + ) + + +def assert_peer1_results(output_dir): + """ + Assert the first peer completed without runtime errors. + + Parameters: + output_dir: Output directory of the first peer. + + Returns: + None + """ + assert_no_errors(output_dir) + + +def assert_peer2_results(output_dir_peer, log_file_second_iris): + """ + Assert the second peer completed without runtime errors and started Iris. + + Parameters: + output_dir_peer: Output directory of the second peer. + log_file_second_iris: Path to the second peer Iris log file. + + Returns: + None + """ + assert_no_errors(output_dir_peer) + assert check_strings_in_file( + ["connection string:"], + log_file_second_iris, + ) + + +def prepare_and_start_peer1( + zeek_dir_path, + output_dir, + redis_port, + default_interface, + log_file, +): + """ + Generate config for peer1, start it, and return startup metadata. + + Parameters: + zeek_dir_path: Zeek dataset path used by the test. + output_dir: Output directory for peer1. + redis_port: Redis port used by peer1. + default_interface: Interface required by Slips when using `-g`. + log_file: Open file handle used to capture peer1 output. + + Returns: + tuple: Peer1 process, Iris log path, and Slips config path. + """ + peer1_slips_config = PEER1_CONFIG_DIR / "slips.yaml" + peer1_iris_config = PEER1_CONFIG_DIR / "iris.yaml" + peer1_iris_config_path = Path( + "tests/integration/test_iris/peer1_config" + ) / (peer1_iris_config.name) + + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER1_CONFIG_DIR, + output_filename=peer1_iris_config.name, + changes={ + "Redis": {"Port": redis_port}, + "PeerDiscovery": { + "DisableBootstrappingNodes": True, + "ListOfMultiAddresses": [], + }, + }, + ) + modify_yaml_config( + input_path="config/slips.yaml", + output_dir=PEER1_CONFIG_DIR, + output_filename=peer1_slips_config.name, + changes={ + "global_p2p": { + "use_global_p2p": True, + "iris_conf": str(peer1_iris_config_path), + }, + "modules": {"disable": ["template", "updatemanager"]}, + }, + ) + + command = [ + sys.executable, + "./slips.py", + "-t", + "-g", + str(zeek_dir_path), + "-i", + default_interface, + "-e", + "1", + "-o", + str(output_dir), + "-c", + str(peer1_slips_config), + "-P", + str(redis_port), + ] + print(f"@@@@@@@@@@@@@@@@ {' '.join(command)}") + + process = subprocess.Popen( + command, + stdout=log_file, + stderr=log_file, + ) + log_file_first_iris = output_dir / "iris/iris_logs.txt" + return process, log_file_first_iris, peer1_slips_config + + +def prepare_and_start_peer2( + zeek_dir_path, + output_dir_peer, + peer_redis_port, + default_interface, + connection_string, + log_file, +): + """ + Generate config for peer2, start it, and return startup metadata. + + Parameters: + zeek_dir_path: Zeek dataset path used by the test. + output_dir_peer: Output directory for peer2. + peer_redis_port: Redis port used by peer2. + default_interface: Interface required by Slips when using `-g`. + connection_string: Multiaddress of peer1 used by peer2. + log_file: Open file handle used to capture peer2 output. + + Returns: + tuple: Peer2 process, Iris log path, Slips config path, and Iris config path. + """ + peer2_iris_config = PEER2_CONFIG_DIR / "iris.yaml" + peer2_iris_config_path = Path( + "tests/integration/test_iris/peer2_config" + ) / (peer2_iris_config.name) + peer2_slips_config = PEER2_CONFIG_DIR / "slips.yaml" + + modify_yaml_config( + input_path="config/slips.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_slips_config.name, + changes={ + "global_p2p": { + "use_global_p2p": True, + "iris_conf": str(peer2_iris_config_path), + }, + "modules": {"disable": ["template", "updatemanager"]}, + }, + ) + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_iris_config.name, + changes={ + "Redis": {"Port": peer_redis_port}, + "Server": {"Port": 9006}, + "PeerDiscovery": { + "DisableBootstrappingNodes": True, + "ListOfMultiAddresses": [connection_string], + }, + "Identity": {"KeyFile": "second.priv"}, + }, + ) + + peer_command = [ + sys.executable, + "./slips.py", + "-t", + "-g", + str(zeek_dir_path), + "-i", + default_interface, + "-e", + "1", + "-o", + str(output_dir_peer), + "-c", + str(peer2_slips_config), + "-P", + str(peer_redis_port), + ] + peer_process = subprocess.Popen( + peer_command, stdout=log_file, stderr=log_file + ) + log_file_second_iris = output_dir_peer / "iris/iris_logs.txt" + return ( + peer_process, + log_file_second_iris, + peer2_slips_config, + peer2_iris_config, + ) + + +@pytest.mark.parametrize( + "zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port", + [ + ( + "dataset/test13-malicious-dhcpscan-zeek-dir", + "iris_integration_test/", + "peer_iris_integration_test/", + 6644, + 6655, + ) + ], +) +def test_messaging( + zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port +): + """ + Tests whether Iris properly distributes an alert message generated by + Slips to the network (~other peers). + + First Slips instance is a general node in the network, its connection + string is generated and extracted from logs as a normal user would do, + in a very standard use case. + + The second instance of Slips acts as a normal-user-peer that joins the + network via the aforementioned Slips instance, + which extends the standard use case of connecting to such P2P network. + """ + default_interface = get_default_interface() + + # Two Slips instances are necessary to be run in this test. + + # Prepare output dir for peer1 + output_dir: PosixPath = create_output_dir(output_dir) + output_file = os.path.join(output_dir, "slips_output.txt") + + # Prepare output dir for the peer2 + output_dir_peer: PosixPath = create_output_dir(peer_output_dir) + output_file_peer = os.path.join(output_dir_peer, "slips_output.txt") + + print("running slips ...") + with open(output_file, "w") as log_file: + with open(output_file_peer, "w") as iris_log_file: + process, log_file_first_iris, peer1_slips_config = ( + prepare_and_start_peer1( + zeek_dir_path=zeek_dir_path, + output_dir=output_dir, + redis_port=redis_port, + default_interface=default_interface, + log_file=log_file, + ) + ) + assert_peer1_setup(peer1_slips_config) + + # First peer (its Iris) needs to be ready and available for + # connections when the second peer tries to reach out to it. + countdown(20, "second peer") + # get the connection string from the first peer and give it + # to the second one so it is reachable + assert wait_for_file( + log_file_first_iris, 30 + ), f"Expected Iris log file was not created: {log_file_first_iris}" + original_conn_string = extract_connection_string( + log_file_first_iris + ) + + ( + peer_process, + log_file_second_iris, + peer2_slips_config, + peer2_iris_config, + ) = prepare_and_start_peer2( + zeek_dir_path=zeek_dir_path, + output_dir_peer=output_dir_peer, + peer_redis_port=peer_redis_port, + default_interface=default_interface, + connection_string=original_conn_string, + log_file=iris_log_file, + ) + assert_peer2_setup( + peer2_slips_config=peer2_slips_config, + peer2_iris_config=peer2_iris_config, + connection_string=original_conn_string, + peer_redis_port=peer_redis_port, + ) + + print( + f"Output and errors of first peer are logged in" + f" {output_file}" + ) + + # let Slips properly and fully star with all of its parts and modules. + countdown(80, "Sending msg in fides2network") + # Sending a manual message to make sure there is an alert generated, because + # is is highly probable that both slips have covered their network captures + # before the infrastructure of P2P network was fully up and running + message_send( + redis_port, + message=message_alert_TL_NL, + channel="fides2network", + ) + + # these seconds are the time we give slips to process the msg + countdown(30, "Sending SIGTERM to the 2 peers") + # Kill em with kindness. + os.kill(process.pid, 15) + os.kill(peer_process.pid, 15) + print("SIGTERM sent.") + + print("Sending SIGKILL to the 2 instances of Slips + iris") + # Kill em. Without kindness. + os.kill(process.pid, 9) + print(f"Slips with PID {process.pid} was killed.") + + os.kill(peer_process.pid, 9) + print(f"Slips peer with PID {peer_process.pid} was killed.") + + print("Slips is done, checking for errors in the 2 output dirs.") + assert_peer1_results(output_dir) + assert_peer2_results(output_dir_peer, log_file_second_iris) + + print("Deleting the output directories") + shutil.rmtree(output_dir) + shutil.rmtree(output_dir_peer) + os.remove("modules/iris/second.priv") + + # reset the generated peer2 Iris config back to its default values + # after the test finishes. + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_iris_config.name, + changes={ + "Redis": {"Port": 6644}, + "Server": {"Port": 9010}, + "PeerDiscovery": {}, + "Identity": {"KeyFile": "private.key"}, + }, + ) diff --git a/tests/integration/test_pcap_dataset/__init__.py b/tests/integration/test_pcap_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_pcap_dataset.py b/tests/integration/test_pcap_dataset/test_pcap_dataset.py similarity index 100% rename from tests/integration/test_pcap_dataset.py rename to tests/integration/test_pcap_dataset/test_pcap_dataset.py diff --git a/tests/integration/test_portscans/__init__.py b/tests/integration/test_portscans/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_portscans.py b/tests/integration/test_portscans/test_portscans.py similarity index 100% rename from tests/integration/test_portscans.py rename to tests/integration/test_portscans/test_portscans.py diff --git a/tests/integration/test_zeek_dataset/__init__.py b/tests/integration/test_zeek_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_zeek_dataset.py b/tests/integration/test_zeek_dataset/test_zeek_dataset.py similarity index 100% rename from tests/integration/test_zeek_dataset.py rename to tests/integration/test_zeek_dataset/test_zeek_dataset.py diff --git a/tests/module_factory.py b/tests/module_factory.py index 30972a7af7..9f76c5a859 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only import shutil +import importlib.util +import sys from contextlib import contextmanager from unittest.mock import ( patch, @@ -108,7 +110,16 @@ def create_db_manager_obj( return db def create_main_obj(self): - from slips.main import Main + try: + from slips.main import Main + except ModuleNotFoundError: + module_spec = importlib.util.spec_from_file_location( + "slips_main_for_tests", os.path.join("slips", "main.py") + ) + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = module + module_spec.loader.exec_module(module) + Main = module.Main """returns an instance of Main() class in slips.py""" main = Main(testing=True) @@ -240,14 +251,30 @@ def create_go_director_obj(self, mock_db): @patch(DB_MANAGER, name="mock_db") def create_daemon_object(self, mock_db): - from slips.daemon import Daemon + try: + from slips.daemon import Daemon + + daemon_module = "slips.daemon" + except ModuleNotFoundError: + module_spec = importlib.util.spec_from_file_location( + "slips_daemon_for_tests", os.path.join("slips", "daemon.py") + ) + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = module + module_spec.loader.exec_module(module) + Daemon = module.Daemon + daemon_module = module_spec.name with ( - patch("slips.daemon.Daemon.read_pidfile", return_type=None), - patch("slips.daemon.Daemon.read_configuration"), + patch(f"{daemon_module}.Daemon.read_pidfile", return_value=None), + patch(f"{daemon_module}.Daemon.read_configuration"), patch("builtins.open", mock_open(read_data=None)), ): - daemon = Daemon(MagicMock()) + slips = MagicMock() + slips.args.stopdaemon = True + slips.args.is_slips_started_by_an_update = False + slips.args.output = "output" + daemon = Daemon(slips) daemon.stderr = "errors.log" daemon.stdout = "slips.log" daemon.stdin = "/dev/null" @@ -495,13 +522,12 @@ def create_input_obj( from slips_files.core.input import Input from slips_files.core.output import Output - zeek_tmp_dir = os.path.join(os.getcwd(), "zeek_dir_for_testing") input = Input( logger=Output(), output_dir="dummy_output_dir", redis_port=6379, termination_event=Mock(), - slips_args=Mock(), + slips_args=Mock(output="dummy_output_dir"), conf=Mock(), ppid=Mock(), bloom_filters_manager=Mock(), @@ -511,7 +537,6 @@ def create_input_obj( input_information=input_information, cli_packet_filter=None, zeek_or_bro=check_zeek_or_bro(), - zeek_dir=zeek_tmp_dir, line_type=line_type, is_profiler_done_event=Mock(), ) @@ -659,9 +684,11 @@ def create_spamhaus_obj(self, mock_db): @patch(MODULE_DB_MANAGER, name="mock_db") def create_update_manager_obj(self, mock_db): - from modules.update_manager.update_manager import UpdateManager + from modules.feeds_update_manager.feeds_update_manager import ( + FeedsUpdateManager, + ) - update_manager = UpdateManager( + update_manager = FeedsUpdateManager( logger=self.logger, output_dir="dummy_output_dir", redis_port=6379, diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 73fc26e829..1d64bca694 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -22,9 +22,9 @@ printf "0" | ./slips.py -k # command before running the dataset tests # distribute on 3 workers only because every worker will be spawning 10+ processes -python3 -m pytest -s tests/integration/test_portscans.py -p no:warnings -vv -python3 -m pytest -s tests/integration/test_dataset.py -p no:warnings -vv -python3 -m pytest -s tests/integration/test_config_files.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_portscans/test_portscans.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_dataset/test_dataset.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_config_files/test_config_files.py -p no:warnings -vv printf "0" | ./slips.py -k diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index 386dd9ae35..a5f0b4cbeb 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only +import json import pytest from unittest.mock import Mock, patch from managers.process_manager import ProcessManager @@ -10,7 +11,7 @@ @pytest.mark.parametrize( "input_type, input_information, cli_packet_filter, " - "zeek_or_bro, zeek_dir, line_type", + "zeek_or_bro, line_type", [ # Test case 1: pcap input ( @@ -18,13 +19,12 @@ "test.pcap", "tcp port 80", "zeek", - "/opt/zeek", "conn", ), # Test case 2: zeek input - (InputType.ZEEK, "test.log", "", "bro", "/opt/bro", "dns"), + (InputType.ZEEK, "test.log", "", "bro", "dns"), # Test case 3: stdin input - (InputType.STDIN, "-", "", "zeek", "/opt/zeek", "http"), + (InputType.STDIN, "-", "", "zeek", "http"), ], ) def test_start_input_process( @@ -32,7 +32,6 @@ def test_start_input_process( input_information, cli_packet_filter, zeek_or_bro, - zeek_dir, line_type, ): process_manager = ModuleFactory().create_process_manager_obj() @@ -40,7 +39,6 @@ def test_start_input_process( process_manager.main.input_information = input_information process_manager.main.args.pcapfilter = cli_packet_filter process_manager.main.zeek_bro = zeek_or_bro - process_manager.main.zeek_dir = zeek_dir process_manager.main.line_type = line_type process_manager.main.bloom_filters_man = Mock() @@ -67,10 +65,12 @@ def test_start_input_process( input_information=input_information, cli_packet_filter=cli_packet_filter, zeek_or_bro=zeek_or_bro, - zeek_dir=zeek_dir, line_type=line_type, is_profiler_done_event=process_manager.is_profiler_done_event, is_input_done_event=process_manager.is_input_done_event, + is_slips_live_updating_event=( + process_manager.is_slips_live_updating_event + ), ) mock_input_process.start.assert_called_once() process_manager.main.print.assert_called_once() @@ -79,6 +79,30 @@ def test_start_input_process( ) +@pytest.mark.parametrize( + "is_slips_started_by_an_update,expected_is_set", + [(True, False), (False, False)], +) +def test_init_sets_live_update_event_for_update_start( + is_slips_started_by_an_update, + expected_is_set, +): + """Test that ProcessManager starts with an unset live-update event.""" + main_mock = Mock() + main_mock.args.is_slips_started_by_an_update = ( + is_slips_started_by_an_update + ) + main_mock.conf.get_disabled_modules.return_value = [] + main_mock.conf.is_bootstrapping_node.return_value = False + main_mock.conf.get_bootstrapping_modules.return_value = [] + + process_manager = ProcessManager(main_mock) + + assert process_manager.is_slips_live_updating_event.is_set() is ( + expected_is_set + ) + + @pytest.mark.parametrize( "module_name, modules_to_ignore, expected", [ @@ -115,7 +139,7 @@ def test_print_disabled_modules(): [ # Test case 1: No pending modules, no additional print calls ([], 1), - # Test case 2: Pending modules without update_manager, one additional print call + # Test case 2: Pending modules without feeds_update_manager, one additional print call ([Mock(name="Module1"), Mock(name="Module2")], 1), ], ) @@ -270,6 +294,16 @@ def test_get_analysis_time( (None, True, False), # Test case 2: Message doesn't contain "stop_slips" ({"data": "some_other_message"}, True, False), + # Test case 3: Wrapped plain-text messages should be decoded first + ( + { + "data": json.dumps( + {"text": "stop_slips", "version": "test-version"} + ) + }, + True, + True, + ), # Test case 3: Message contains # "stop_slips" but not intended for control channel ({"data": "stop_slips"}, False, False), @@ -519,12 +553,13 @@ def test_start_update_manager( asyncio_called, ): process_manager = ModuleFactory().create_process_manager_obj() + process_manager.main.args.output = "output" mock_lock_instance = Mock() mock_lock.return_value.__enter__.return_value = mock_lock_instance mock_update_manager = Mock() with patch( - "managers.process_manager.UpdateManager", + "managers.process_manager.FeedsUpdateManager", return_value=mock_update_manager, ): process_manager.start_update_manager( diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index 55341c13fb..49d61ffd09 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -7,7 +7,11 @@ import pytest from tests.module_factory import ModuleFactory -from managers.redis_manager import UserCancelledErr +from managers.redis_manager import ( + DEFAULT_REDIS_PORT, + RedisManager, + UserCancelledErr, +) from slips_files.common.input_type import InputType @@ -48,7 +52,7 @@ def test_log_redis_server_pid_normal_ports( ): redis_manager = ModuleFactory().create_redis_manager_obj() redis_manager.main.input_information = "input_info" - redis_manager.main.zeek_dir = "zeek_dir" + redis_manager.main.db.get_zeek_output_dir.return_value = "zeek_dir" redis_manager.main.args.output = "output_dir" redis_manager.main.args.daemon = is_daemon redis_manager.main.args.save = save_db @@ -171,7 +175,9 @@ def test_check_redis_database(mock_db): mock_db.rcache.ping.return_value = True with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=False), + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=False + ), patch("managers.redis_manager.RedisDB", return_value=mock_db), ): result = redis_manager.start_redis_cache_if_not_running() @@ -188,7 +194,9 @@ def test_check_redis_database_failure(mock_db): mock_db.rcache.ping.side_effect = redis.exceptions.ConnectionError with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=False), + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=False + ), patch("managers.redis_manager.RedisDB", return_value=mock_db), pytest.raises(redis.exceptions.ConnectionError), ): @@ -203,8 +211,12 @@ def test_check_redis_database_uses_running_cache(mock_db): mock_db.rcache.ping.return_value = True with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=True), - patch("managers.redis_manager.RedisDB", return_value=mock_db) as mock_redis, + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=True + ), + patch( + "managers.redis_manager.RedisDB", return_value=mock_db + ) as mock_redis, ): result = redis_manager.start_redis_cache_if_not_running() @@ -549,6 +561,7 @@ def test_get_redis_port( redis_manager = ModuleFactory().create_redis_manager_obj() redis_manager.main.args.port = args_port redis_manager.main.args.multiinstance = multiinstance + redis_manager.main.args.is_slips_started_by_an_update = False with ( patch.object( @@ -576,6 +589,36 @@ def test_get_redis_port( mock_terminate.assert_called() +@pytest.mark.parametrize( + "args_port, expected_port", + [ + ("32768", 32768), + (None, DEFAULT_REDIS_PORT), + ], +) +def test_get_redis_port_started_by_update(args_port, expected_port, mock_db): + redis_manager = RedisManager(Mock()) + redis_manager.main.args = Mock() + redis_manager.main.args.is_slips_started_by_an_update = True + redis_manager.main.args.port = args_port + + with ( + patch.object( + redis_manager, "_get_dbmanager_without_starting_a_new_server" + ) as mock_db_mgr, + patch.object(redis_manager, "confirm_server_altering") as mock_confirm, + patch.object( + redis_manager, "get_random_redis_port" + ) as mock_random_port, + ): + result = redis_manager.get_redis_port() + + assert result == expected_port + mock_db_mgr.assert_not_called() + mock_confirm.assert_not_called() + mock_random_port.assert_not_called() + + @pytest.mark.parametrize( "total1, total2, expected", [ diff --git a/tests/unit/managers/test_slips.py b/tests/unit/managers/test_slips.py index 2ba204b07a..a6e383bb66 100644 --- a/tests/unit/managers/test_slips.py +++ b/tests/unit/managers/test_slips.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-only """Unit test for ../slips.py""" +from unittest.mock import patch + from tests.module_factory import ModuleFactory @@ -49,4 +51,9 @@ def test_load_modules(): def test_clear_redis_cache_database(): redis_manager = ModuleFactory().create_redis_manager_obj() - assert redis_manager.clear_redis_cache_database() + + with patch("redis.StrictRedis") as mock_redis: + mock_redis_instance = mock_redis.return_value + + assert redis_manager.clear_redis_cache_database() + mock_redis_instance.flushdb.assert_called_once() diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py new file mode 100644 index 0000000000..3a243c6d9e --- /dev/null +++ b/tests/unit/managers/test_update_manager.py @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +from unittest.mock import Mock, patch + +import pytest +from git import GitCommandError + +from managers.update_manager import UpdateManager + + +def create_update_manager(): + """ + Create an UpdateManager with mocked external dependencies. + + Returns: + UpdateManager instance for unit tests. + """ + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) + conf.auto_update_slips.return_value = True + + with patch("managers.update_manager.ConfigParser", return_value=conf): + return UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=Mock(), + ) + + +@pytest.mark.parametrize( + "remote_url, expected_link", + [ + ( + "https://github.com/stratosphereips/StratosphereLinuxIPS.git", + "https://raw.githubusercontent.com/stratosphereips/" + "StratosphereLinuxIPS/master/update.json", + ), + ( + "git@github.com:stratosphereips/StratosphereLinuxIPS.git", + "https://raw.githubusercontent.com/stratosphereips/" + "StratosphereLinuxIPS/master/update.json", + ), + ("https://example.com/stratosphereips/StratosphereLinuxIPS.git", None), + ], +) +def test_get_master_update_json_link(remote_url, expected_link): + update_manager = create_update_manager() + repo = Mock() + repo.remote.return_value.url = remote_url + + with patch("managers.update_manager.Repo", return_value=repo): + assert update_manager._get_master_update_json_link() == expected_link + + +@pytest.mark.parametrize( + "update_json, expected_dependencies, expected_compatibility", + [ + ( + '{"has_new_dependencies": true, "backwards_compatible": false}', + True, + False, + ), + ( + "{\n" + ' "has_new_dependencies": false,\n' + ' "backwards_compatible": true,\n' + "}\n", + False, + True, + ), + ], +) +def test_update_json_flags( + update_json, expected_dependencies, expected_compatibility +): + update_manager = create_update_manager() + response = Mock() + response.read.return_value = update_json.encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + + with patch.object( + update_manager, + "_get_master_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/master/update.json" + ), + ), patch( + "managers.update_manager.request.urlopen", return_value=response + ) as mock_urlopen: + assert ( + update_manager._new_version_has_new_dependencies() + == expected_dependencies + ) + assert ( + update_manager._is_new_version_backwards_compatible() + == expected_compatibility + ) + mock_urlopen.assert_called_once() + + +@pytest.mark.parametrize( + "update_payload, expected_dependencies, expected_compatibility", + [ + (None, True, False), + ("not-json", True, False), + ], +) +def test_update_json_fallbacks( + update_payload, + expected_dependencies, + expected_compatibility, +): + update_manager = create_update_manager() + + patches = [ + patch.object( + update_manager, + "_get_master_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/master/update.json" + ), + ) + ] + + if update_payload is None: + patches.append( + patch( + "managers.update_manager.request.urlopen", + side_effect=OSError("network error"), + ) + ) + else: + response = Mock() + response.read.return_value = update_payload.encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + patches.append( + patch( + "managers.update_manager.request.urlopen", + return_value=response, + ) + ) + + with patches[0], patches[1]: + assert ( + update_manager._new_version_has_new_dependencies() + == expected_dependencies + ) + assert ( + update_manager._is_new_version_backwards_compatible() + == expected_compatibility + ) + + +def test_get_updated_slips_command_appends_update_flag(): + update_manager = create_update_manager() + process = Mock() + process.cmdline.return_value = ["python3", "slips.py", "-i", "eth0"] + + with patch("managers.update_manager.psutil.Process", return_value=process): + assert update_manager._get_updated_slips_command() == [ + "python3", + "slips.py", + "-i", + "eth0", + "-u", + ] + + +def test_get_updated_slips_command_passes_multiinstance_redis_port(): + update_manager = create_update_manager() + update_manager.args.multiinstance = True + update_manager.db.get_used_redis_port.return_value = 32768 + process = Mock() + process.cmdline.return_value = [ + "python3", + "slips.py", + "-m", + "-i", + "eth0", + ] + + with patch("managers.update_manager.psutil.Process", return_value=process): + assert update_manager._get_updated_slips_command() == [ + "python3", + "slips.py", + "-i", + "eth0", + "-u", + "-P", + "32768", + ] + + +def test_start_updated_slips_verison_starts_detached_process(): + update_manager = create_update_manager() + update_manager._get_updated_slips_command = Mock( + return_value=["python3", "slips.py", "-i", "eth0", "-u"] + ) + + with patch("managers.update_manager.subprocess.Popen") as popen: + process = update_manager.start_updated_slips_version() + + assert process == popen.return_value + popen.assert_called_once_with( + ["python3", "slips.py", "-i", "eth0", "-u"], + close_fds=True, + ) + + +def test_update_slips_starts_updated_process_before_stopping_current_slips(): + update_manager = create_update_manager() + calls = [] + update_manager.git_pull_master = Mock( + side_effect=lambda: calls.append("git_pull_master") + ) + update_manager.start_updated_slips_version = Mock( + side_effect=lambda: calls.append("start_updated_slips_verison") + ) + update_manager.is_slips_live_updating_event.set = Mock( + side_effect=lambda: calls.append("set_update_event") + ) + + update_manager.update_slips() + + update_manager.git_pull_master.assert_called_once() + update_manager.start_updated_slips_version.assert_called_once() + update_manager.is_slips_live_updating_event.set.assert_called_once() + assert calls == [ + "git_pull_master", + "start_updated_slips_verison", + "set_update_event", + ] + + +def test_update_slips_aborts_when_local_changes_block_checkout(): + """ + Ensure local checkout conflicts abort the update without stopping Slips. + + Returns: + None. + """ + update_manager = create_update_manager() + update_manager.cached_update_info = {"version": "1.2.3"} + git_error = GitCommandError( + "git checkout origin/master", + 1, + stderr=( + "error: Your local changes to the following files would be " + "overwritten by checkout:\n" + "\tconfig/slips.yaml\n" + "Please commit your changes or stash them before you switch " + "branches.\n" + "Aborting" + ), + ) + update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.start_updated_slips_version = Mock() + + update_manager.update_slips() + + update_manager.git_pull_master.assert_called_once() + update_manager.start_updated_slips_version.assert_not_called() + update_manager.is_slips_live_updating_event.set.assert_not_called() + update_manager.print.assert_called_once_with( + "Warning: Uncommitted changes to ['config/slips.yaml'] detected. " + "Aborting update to Slips v1.2.3, please update Slips manually." + ) + + +def test_update_slips_aborts_on_unrelated_git_errors(): + """ + Ensure unexpected git failures abort the update without crashing. + + Returns: + None. + """ + update_manager = create_update_manager() + git_error = GitCommandError( + "git checkout origin/master", + 128, + stderr="fatal: not a git repository", + ) + update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.start_updated_slips_version = Mock() + + update_manager.update_slips() + + update_manager.start_updated_slips_version.assert_not_called() + update_manager.is_slips_live_updating_event.set.assert_not_called() + update_manager.print.assert_called_once_with( + "Warning: Aborting Slips update because a git error occurred: " + f"{git_error}" + ) diff --git a/tests/unit/modules/blocking/test_blocking.py b/tests/unit/modules/blocking/test_blocking.py index 4c1c80f4a0..19aab9e65a 100644 --- a/tests/unit/modules/blocking/test_blocking.py +++ b/tests/unit/modules/blocking/test_blocking.py @@ -206,8 +206,16 @@ def test_main_blocking_logic(block, expected_block_called): @pytest.mark.parametrize( "last_closed_tw, msg_data, should_call", [ - ("tw1", "profileid_123_tw2", True), # new tw, should call update - ("tw2", "profileid_234_tw2", False), # same tw, no update + ( + "tw1", + json.dumps({"text": "profileid_123_tw2", "version": "1.0"}), + True, + ), + ( + "tw2", + json.dumps({"text": "profileid_234_tw2", "version": "1.0"}), + False, + ), ], ) def test_main_tw_closed_triggers_update(last_closed_tw, msg_data, should_call): diff --git a/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py b/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py index 1a0f93d58f..fbb32f5440 100644 --- a/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py +++ b/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py @@ -40,7 +40,7 @@ def test_get_confidence(pre_behavioral_model, expected_confidence): ) def test_handle_tw_closed(msg_data, expected_profileid, expected_twid): cc_detection = ModuleFactory().create_rnn_detection_object() - msg = {"data": msg_data} + msg = {"data": json.dumps({"text": msg_data, "version": "test-version"})} with patch.object(cc_detection.letters_exporter, "export") as mock_export: cc_detection.handle_tw_closed(msg) diff --git a/tests/unit/modules/update_manager/test_update_file_manager.py b/tests/unit/modules/update_manager/test_update_file_manager.py index faa7bc84aa..838ad74bea 100644 --- a/tests/unit/modules/update_manager/test_update_file_manager.py +++ b/tests/unit/modules/update_manager/test_update_file_manager.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only -"""Unit test for modules/update_manager/update_manager.py""" +"""Unit test for modules/feeds_update_manager/feeds_update_manager.py""" from tests.module_factory import ModuleFactory import json diff --git a/tests/unit/slips/test_main.py b/tests/unit/slips/test_main.py index 9efb1b8072..8c095e0a15 100644 --- a/tests/unit/slips/test_main.py +++ b/tests/unit/slips/test_main.py @@ -301,7 +301,8 @@ def test_delete_zeek_files_enabled(): main = ModuleFactory().create_main_obj() main.conf = MagicMock() main.conf.delete_zeek_files.return_value = True - main.zeek_dir = "zeek_dir" + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" with patch("shutil.rmtree") as mock_rmtree: main.delete_zeek_files() @@ -312,13 +313,32 @@ def test_delete_zeek_files_disabled(): main = ModuleFactory().create_main_obj() main.conf = MagicMock() main.conf.delete_zeek_files.return_value = False - main.zeek_dir = "zeek_dir" + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" with patch("shutil.rmtree") as mock_rmtree: main.delete_zeek_files() mock_rmtree.assert_not_called() +def test_store_zeek_dir_copy_reads_zeek_dir_from_db(): + main = ModuleFactory().create_main_obj() + main.conf = MagicMock() + main.conf.store_a_copy_of_zeek_files.return_value = True + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" + main.args.output = "output" + + with ( + patch.object(main, "was_running_zeek", return_value=True), + patch(f"{main.__class__.__module__}.copy_tree") as mock_copy_tree, + patch("builtins.print"), + ): + main.store_zeek_dir_copy() + + mock_copy_tree.assert_called_once_with("zeek_dir", "output/zeek_files") + + # TODO should be moved to utils unit tests after the PR is merged # def test_get_slips_version(): # main = ModuleFactory().create_main_obj() @@ -379,29 +399,6 @@ def test_check_zeek_or_bro_not_found(): mock_terminate.assert_called_once() -@pytest.mark.parametrize( - "store_in_output, expected_dir", - [ - # Test Case 1: Store Zeek files in the output directory - (True, "output/zeek_files"), - # Test Case 2: Use default directory for Zeek files - (False, "zeek_files_inputfile/"), - ], -) -def test_prepare_zeek_output_dir(store_in_output, expected_dir): - main = ModuleFactory().create_main_obj() - main.input_information = "/path/to/inputfile.pcap" - main.args = Mock() - main.args.output = "output" - main.conf = Mock() - main.conf.store_zeek_files_in_the_output_dir.return_value = store_in_output - - with patch("os.path.join", lambda *args: "/".join(args)): - main.prepare_zeek_output_dir() - - assert main.zeek_dir == expected_dir - - def test_terminate_slips_interactive(): main = ModuleFactory().create_main_obj() main.mode = "interactive" @@ -445,6 +442,7 @@ def main_obj(tmp_path): main = ModuleFactory().create_main_obj() main.args = MagicMock() main.args.output = str(tmp_path / "test_output") + main.args.is_slips_started_by_an_update = False return main @@ -488,7 +486,9 @@ def test_prepare_output_dir_without_o_flag( ): main = ModuleFactory().create_main_obj() main.args = MagicMock() - main.alerts_default_path = str(tmp_path) + main.args.is_slips_started_by_an_update = False + main.args.output = None + main.parent_output_dir = str(tmp_path) main.input_information = "/fake/input/wlp3s0" monkeypatch.setattr(sys, "argv", ["script.py"]) # No -o diff --git a/tests/unit/slips_files/common/abstracts/test_imodule.py b/tests/unit/slips_files/common/abstracts/test_imodule.py new file mode 100644 index 0000000000..80d4ef9b8b --- /dev/null +++ b/tests/unit/slips_files/common/abstracts/test_imodule.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json + +from tests.module_factory import ModuleFactory +from slips_files.common.slips_utils import utils + + +def test_imodule_exposes_slips_version(): + ip_info = ModuleFactory().create_ip_info_obj() + + assert ip_info.slips_version == utils.get_slips_version() + + +def test_get_msg_discards_messages_with_different_version(): + ip_info = ModuleFactory().create_ip_info_obj() + ip_info.channels = {"new_ip": "channel_obj"} + ip_info.channel_tracker = ip_info.init_channel_tracker() + ip_info.db.get_message.return_value = { + "channel": "new_ip", + "data": json.dumps({"text": "1.2.3.4", "version": "0.0.0"}), + } + + msg = ip_info.get_msg("new_ip") + + assert msg is None + assert ip_info.channel_tracker["new_ip"]["msg_received"] is False + ip_info.db.incr_msgs_received_in_channel.assert_not_called() + + +def test_get_msg_accepts_messages_with_current_version(): + ip_info = ModuleFactory().create_ip_info_obj() + ip_info.channels = {"new_ip": "channel_obj"} + ip_info.channel_tracker = ip_info.init_channel_tracker() + message = { + "channel": "new_ip", + "data": json.dumps( + {"text": "1.2.3.4", "version": ip_info.slips_version} + ), + } + ip_info.db.get_message.return_value = message + + msg = ip_info.get_msg("new_ip") + + assert msg == message + assert ip_info.channel_tracker["new_ip"]["msg_received"] is True + ip_info.db.incr_msgs_received_in_channel.assert_called_once_with( + ip_info.name, "new_ip" + ) diff --git a/tests/unit/slips_files/common/test_slips_utils.py b/tests/unit/slips_files/common/test_slips_utils.py index faca6d6693..9c1be8097f 100644 --- a/tests/unit/slips_files/common/test_slips_utils.py +++ b/tests/unit/slips_files/common/test_slips_utils.py @@ -27,6 +27,28 @@ def test_get_sha256_hash_from_nonexistent_file(): utils.get_sha256_hash_of_file_contents("nonexistent_file.txt") +def test_initialize_logfile_creates_file(tmp_path): + utils = ModuleFactory().create_utils_obj() + logfile = tmp_path / "logs" / "module.log" + + initialized = utils.initialize_logfile(str(logfile), False) + + assert initialized is True + assert logfile.exists() + assert logfile.read_text() == "" + + +def test_initialize_logfile_skips_file_when_started_by_update(tmp_path): + utils = ModuleFactory().create_utils_obj() + logfile = tmp_path / "module.log" + logfile.write_text("existing\n") + + initialized = utils.initialize_logfile(str(logfile), True) + + assert initialized is False + assert logfile.read_text() == "existing\n" + + @pytest.mark.parametrize( "filepath, expected_result", [ # Testcase 1: Supported file @@ -65,6 +87,29 @@ def test_get_sha256_hash_permission_error(): utils.get_sha256_hash_of_file_contents("restricted_file.txt") +@pytest.mark.parametrize( + "message, expected_payload", + [ + ({"data": "plain-text"}, "plain-text"), + ( + {"data": json.dumps({"text": "stop_slips", "version": "1.0"})}, + "stop_slips", + ), + ( + { + "data": json.dumps( + {"flow": {"uid": "abc"}, "profileid": "profile_1"} + ) + }, + {"flow": {"uid": "abc"}, "profileid": "profile_1"}, + ), + ], +) +def test_get_msg_payload(message, expected_payload): + utils = ModuleFactory().create_utils_obj() + assert utils.get_msg_payload(message) == expected_payload + + @pytest.mark.parametrize( "input_string, expected_output", [ # Testcase1: special chars diff --git a/tests/unit/slips_files/core/database/test_database.py b/tests/unit/slips_files/core/database/test_database.py index 9c54ff69d5..c7343aa964 100644 --- a/tests/unit/slips_files/core/database/test_database.py +++ b/tests/unit/slips_files/core/database/test_database.py @@ -147,11 +147,14 @@ def test_add_mac_addr_with_ipv6_association(): def test_get_the_other_ip_version(): db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) - # profileid is ipv4 + profileid_ipv4 = "profile_192.168.250.250" ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - db.set_ipv6_of_profile(profileid, ipv6) - # the other ip version is ipv6 - other_ip = json.loads(db.get_the_other_ip_version(profileid)) + + db.rdb.get_ipv6_from_profile = Mock(return_value=json.dumps(ipv6)) + + other_ip = json.loads(db.get_the_other_ip_version(profileid_ipv4)) + + db.rdb.get_ipv6_from_profile.assert_called_once_with(profileid_ipv4) assert other_ip == ipv6 diff --git a/tests/unit/slips_files/core/helpers/test_checker.py b/tests/unit/slips_files/core/helpers/test_checker.py index e24c1a6c5b..ce2e45fc4c 100644 --- a/tests/unit/slips_files/core/helpers/test_checker.py +++ b/tests/unit/slips_files/core/helpers/test_checker.py @@ -14,7 +14,6 @@ def test_clear_redis_cache(): checker.clear_redis_cache() checker.main.redis_man.clear_redis_cache_database.assert_called_once() assert checker.main.input_information == "" - assert checker.main.zeek_dir == "" checker.main.redis_man.log_redis_server_pid.assert_called_once_with( 6379, mock.ANY ) diff --git a/tests/unit/slips_files/core/input/test_dos_protector.py b/tests/unit/slips_files/core/input/test_dos_protector.py index 84923bc875..b406ccd8e2 100644 --- a/tests/unit/slips_files/core/input/test_dos_protector.py +++ b/tests/unit/slips_files/core/input/test_dos_protector.py @@ -83,10 +83,7 @@ def test_get_number_of_flows_to_skip_updates_sampling_window_and_prints(): return_value="formatted-ts", ), ): - assert ( - protector.get_number_of_flows_to_skip_and_time_to_stop_sampling() - == 449 - ) + assert protector.get_number_of_flows_to_skip() == 449 assert protector.flow_sampling_stop_time == 160 assert protector._is_now_sampling is True diff --git a/tests/unit/slips_files/core/input/test_input.py b/tests/unit/slips_files/core/input/test_input.py index 4db7e39cfb..fbaf9f1d30 100644 --- a/tests/unit/slips_files/core/input/test_input.py +++ b/tests/unit/slips_files/core/input/test_input.py @@ -21,30 +21,25 @@ def test_handle_pcap_input(tmp_path, input_type, input_information): input_process = ModuleFactory().create_input_obj( input_information, input_type ) - input_process.zeek_dir = tmp_path handler = input_process.input_handlers[input_type] input_process.is_running_non_stop = False handler.file_remover.start = Mock() - # Mock attributes and methods used inside the function - input_process.zeek_utils.ensure_zeek_dir = Mock() + input_process.zeek_utils.create_zeek_output_dir = Mock( + return_value=str(tmp_path) + ) input_process.zeek_utils.init_zeek = Mock() input_process.zeek_utils.read_zeek_files = Mock(return_value=7) assert handler.run() is True - # Assert that the expected methods were called - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once_with( - handler.observer, input_process.zeek_dir, input_process.given_path + handler.observer, str(tmp_path), input_process.given_path ) input_process.zeek_utils.read_zeek_files.assert_called_once() handler.file_remover.start.assert_not_called() assert input_process.lines == 7 - # Clean up any directories created (safe guard) - if os.path.exists(input_process.zeek_dir): - shutil.rmtree(input_process.zeek_dir, ignore_errors=True) - @pytest.mark.parametrize( "zeek_dir, is_tabs", @@ -516,17 +511,23 @@ def test_get_file_handle_non_existing_file(): assert file_handle is False -def test_ensure_zeek_dir_creates_dir(tmp_path): - """Test that ensure_zeek_dir creates the directory if missing.""" +def test_create_zeek_output_dir_creates_dir(tmp_path): + """Test that create_zeek_output_dir creates the directory if missing.""" input_process = ModuleFactory().create_input_obj( "", InputType.ZEEK_LOG_FILE ) - zeek_dir = tmp_path / "zeek_logs" - input_process.zeek_dir = str(zeek_dir) + input_process.given_path = str(tmp_path / "capture.pcap") + input_process.args.output = str(tmp_path / "output") + input_process.zeek_utils.is_running_non_stop = False + input_process.conf.store_zeek_files_in_the_output_dir.return_value = True - assert not os.path.exists(input_process.zeek_dir) - input_process.zeek_utils.ensure_zeek_dir() - assert os.path.exists(input_process.zeek_dir) + zeek_dir = input_process.zeek_utils.create_zeek_output_dir() + + assert zeek_dir == os.path.join(input_process.args.output, "zeek_files") + assert os.path.exists(zeek_dir) + input_process.db.set_input_metadata.assert_called_once_with( + {"zeek_dir": zeek_dir} + ) def test_check_if_time_to_del_rotated_files_deletes_old_files(): diff --git a/tests/unit/slips_files/core/input/test_interface_input.py b/tests/unit/slips_files/core/input/test_interface_input.py index b87f64eb74..f3a4c945d1 100644 --- a/tests/unit/slips_files/core/input/test_interface_input.py +++ b/tests/unit/slips_files/core/input/test_interface_input.py @@ -10,7 +10,7 @@ def test_interface_input_runs_for_single_interface(tmp_path): input_process.args.interface = "eth0" input_process.args.access_point = False input_process.is_running_non_stop = False - input_process.zeek_utils.ensure_zeek_dir = MagicMock() + input_process.zeek_utils.create_zeek_output_dir = MagicMock() input_process.zeek_utils.init_zeek = MagicMock() input_process.zeek_utils.read_zeek_files = MagicMock(return_value=4) @@ -18,7 +18,7 @@ def test_interface_input_runs_for_single_interface(tmp_path): with patch("os.path.exists", return_value=True): assert handler.run() is True - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once() input_process.zeek_utils.read_zeek_files.assert_called_once() assert input_process.lines == 4 diff --git a/tests/unit/slips_files/core/input/test_pcap_input.py b/tests/unit/slips_files/core/input/test_pcap_input.py index b56ab41a91..536923040a 100644 --- a/tests/unit/slips_files/core/input/test_pcap_input.py +++ b/tests/unit/slips_files/core/input/test_pcap_input.py @@ -11,14 +11,14 @@ def test_pcap_input_runs_through_zeek_utils(tmp_path): ) input_process.zeek_dir = str(tmp_path) input_process.is_running_non_stop = False - input_process.zeek_utils.ensure_zeek_dir = MagicMock() + input_process.zeek_utils.create_zeek_output_dir = MagicMock() input_process.zeek_utils.init_zeek = MagicMock() input_process.zeek_utils.read_zeek_files = MagicMock(return_value=3) handler = input_process.input_handlers[InputType.PCAP] assert handler.run() is True - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once() input_process.zeek_utils.read_zeek_files.assert_called_once() assert input_process.lines == 3 diff --git a/tests/unit/slips_files/core/input/test_zeek_input_utils.py b/tests/unit/slips_files/core/input/test_zeek_input_utils.py index 61e0a92290..03fb56e8b6 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_utils.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_utils.py @@ -2,8 +2,11 @@ # SPDX-License-Identifier: GPL-2.0-only import json import pytest +from types import SimpleNamespace +from unittest.mock import Mock from tests.module_factory import ModuleFactory from slips_files.common.input_type import InputType +from slips_files.core.input.zeek.utils.zeek_input_utils import ZeekInputUtils @pytest.mark.parametrize( @@ -35,3 +38,65 @@ def test_get_ts_from_line_returns_timestamp_for_tabs(): ts, line = input_process.zeek_utils.get_ts_from_line("1.5\tfield\n") assert ts == 1.5 assert line == "1.5\tfield\n" + + +def test_read_zeek_files_drains_generated_lines_during_live_update(tmp_path): + test_file = tmp_path / "conn.log" + test_file.write_text( + "\n".join( + [ + json.dumps({"ts": 1, "uid": "flow-1"}), + json.dumps({"ts": 2, "uid": "flow-2"}), + "", + ] + ), + encoding="utf-8", + ) + live_update_event = Mock() + live_update_event.is_set.return_value = True + db = Mock() + db.is_running_non_stop.return_value = False + db.get_all_zeek_files.return_value = {str(test_file): "eth0"} + input_process = SimpleNamespace( + args=Mock(), + bro_timeout=100, + conf=Mock(), + db=db, + give_profiler=Mock(), + is_slips_live_updating_event=live_update_event, + is_zeek_tabs=False, + lines=0, + print=Mock(), + should_stop=Mock(return_value=True), + store_flows_read_per_second=Mock(), + ) + zeek_utils = ZeekInputUtils(input_process) + zeek_utils.shutdown_zeek_runtime = Mock() + get_flows_to_skip = Mock(return_value=0) + zeek_utils.dos_protector.get_number_of_flows_to_skip = get_flows_to_skip + + assert zeek_utils.read_zeek_files() == 2 + assert input_process.give_profiler.call_count == 2 + zeek_utils.shutdown_zeek_runtime.assert_called_once() + assert get_flows_to_skip.call_count >= 2 + + +@pytest.mark.parametrize( + "store_in_output, expected_dir", + [ + (True, "output/zeek_files"), + (False, "zeek_files_inputfile"), + ], +) +def test_create_zeek_output_dir(store_in_output, expected_dir): + input_process = ModuleFactory().create_input_obj( + "pcaps/inputfile.pcap", InputType.PCAP + ) + input_process.zeek_dir = None + input_process.args.output = "output" + input_process.zeek_utils.is_running_non_stop = False + input_process.conf.store_zeek_files_in_the_output_dir.return_value = ( + store_in_output + ) + + assert input_process.zeek_utils.create_zeek_output_dir() == expected_dir diff --git a/update.json b/update.json new file mode 100644 index 0000000000..a9bb925390 --- /dev/null +++ b/update.json @@ -0,0 +1,6 @@ +{ +"version": "1.1.19", +"release_date": "2026-04-01T14:39:56+02:00", +"backwards_compatible": true, +"has_new_dependencies": false, +}