From cca6498d2604c5723bbccc0ecb52123637342c76 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 22:59:13 +0200 Subject: [PATCH 01/51] update secrets baseline --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 6efec3d377..5c7045df2f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -149,7 +149,7 @@ "filename": "config/slips.yaml", "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", "is_verified": false, - "line_number": 295 + "line_number": 304 } ], "dataset/test14-malicious-zeek-dir/http.log": [ @@ -7185,5 +7185,5 @@ } ] }, - "generated_at": "2026-04-08T14:13:03Z" + "generated_at": "2026-04-16T20:58:25Z" } From 482bc4a8862b3b9416c94de34295f261c39a12b4 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 22:59:16 +0200 Subject: [PATCH 02/51] add an auto update config param --- config/slips.yaml | 9 +++++++++ docs/usage.md | 13 +++++++++++++ modules/update_manager/update_manager.py | 1 + slips_files/common/parsers/config_parser.py | 9 +++++++++ 4 files changed, 32 insertions(+) diff --git a/config/slips.yaml b/config/slips.yaml index 1132ce092d..a8090636bf 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -1,6 +1,15 @@ # This configuration file controls several aspects of the working of Slips. --- +update: + # Enable automatic live updates of the installed Slips version. + # This setting is separate from the update_manager module, which only + # updates TI feeds and related files during runtime. + # Automatic Slips updates may overwrite the default config files shipped + # with Slips. If you want to keep your local configuration changes intact, avoid editing the default config files. + # Instead, create separate config files with different names and use those. + auto_update: false + output: # Define the file names for the default output. stdout: slips.log diff --git a/docs/usage.md b/docs/usage.md index 357c04ce0d..8d9c39b88e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -529,6 +529,19 @@ Use ```permanent_dir``` to choose where Slips stores databases and runtime-gener This includes persistent artifacts such as ```p2p_trust_runtime/``` and shared module databases like the Fides cache. +**Live Slips auto update** + +Use ```update.auto_update``` to enable or disable automatic live updates of the installed Slips version. + +```yaml +update: + auto_update: false +``` + +This setting is separate from the runtime ```update_manager``` module, which only updates TI feeds and related files. + +Automatic Slips updates may overwrite the default config files shipped with Slips. If you want to keep local config changes safe, do not modify the default config files. Create and use your own config files with different names instead. +
diff --git a/modules/update_manager/update_manager.py b/modules/update_manager/update_manager.py index 0c768dc09b..46f0f82390 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/update_manager/update_manager.py @@ -1480,6 +1480,7 @@ def update_local_whitelist(self): """ parses the local whitelist using the whitelist parser and stores it in the db + is only called when slips starts. """ if self.enable_local_whitelist: self.whitelist.update() diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index 8ee2768059..32648e4fbc 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -143,6 +143,15 @@ def evidence_detection_threshold(self): def packet_filter(self): return self.read_configuration("parameters", "pcapfilter", False) + def auto_update(self) -> bool: + """ + Read whether live Slips version auto-updates are enabled. + + Returns: + True when automatic live updates are enabled, otherwise False. + """ + return self.read_configuration("update", "auto_update", False) + def online_whitelist(self): return self.read_configuration("whitelists", "online_whitelist", False) From b24031a562350306382d427f7d15c248ffd0a6b3 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 23:19:13 +0200 Subject: [PATCH 03/51] add an update.json file for deployed slips to check for updates --- update.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 update.json diff --git a/update.json b/update.json new file mode 100644 index 0000000000..a9bb925390 --- /dev/null +++ b/update.json @@ -0,0 +1,6 @@ +{ +"version": "1.1.19", +"release_date": "2026-04-01T14:39:56+02:00", +"backwards_compatible": true, +"has_new_dependencies": false, +} From a0facaed6ee9e9d6550a2795da8149fde4e0330e Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 23:22:06 +0200 Subject: [PATCH 04/51] call the auto update param auto_update_slips instead of auto_update --- config/slips.yaml | 2 +- managers/metadata_manager.py | 3 +-- modules/exporting_alerts/stix_exporter.py | 1 - slips_files/common/parsers/config_parser.py | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/config/slips.yaml b/config/slips.yaml index a8090636bf..2d98b98139 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -8,7 +8,7 @@ update: # Automatic Slips updates may overwrite the default config files shipped # with Slips. If you want to keep your local configuration changes intact, avoid editing the default config files. # Instead, create separate config files with different names and use those. - auto_update: false + auto_update_slips: false output: # Define the file names for the default output. diff --git a/managers/metadata_manager.py b/managers/metadata_manager.py index a8e02d8ced..def73ad133 100644 --- a/managers/metadata_manager.py +++ b/managers/metadata_manager.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia -import subprocess - # SPDX-License-Identifier: GPL-2.0-only +import subprocess import psutil import sys import os diff --git a/modules/exporting_alerts/stix_exporter.py b/modules/exporting_alerts/stix_exporter.py index b2eef15a78..875f0b5854 100644 --- a/modules/exporting_alerts/stix_exporter.py +++ b/modules/exporting_alerts/stix_exporter.py @@ -517,7 +517,6 @@ def should_export(self) -> bool: return "stix" in self.export_to def read_configuration(self) -> bool: - """Reads configuration""" conf = ConfigParser() # Available options ['slack','stix'] self.export_to = conf.export_to() diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index 32648e4fbc..c7eba4682f 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -143,14 +143,14 @@ def evidence_detection_threshold(self): def packet_filter(self): return self.read_configuration("parameters", "pcapfilter", False) - def auto_update(self) -> bool: + def auto_update_slips(self) -> bool: """ Read whether live Slips version auto-updates are enabled. Returns: True when automatic live updates are enabled, otherwise False. """ - return self.read_configuration("update", "auto_update", False) + return self.read_configuration("update", "auto_update_slips", False) def online_whitelist(self): return self.read_configuration("whitelists", "online_whitelist", False) From 04ae7f03f74a598807c7998131311ba601777e8c Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 23:23:00 +0200 Subject: [PATCH 05/51] add an UpdateManager class that handles checking for updates and doing orchestration. --- managers/update_manager.py | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 managers/update_manager.py diff --git a/managers/update_manager.py b/managers/update_manager.py new file mode 100644 index 0000000000..a7ef4b4860 --- /dev/null +++ b/managers/update_manager.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + + +""" +Handles updating of slips version +""" + +from slips_files.common.parsers.config_parser import ConfigParser + + +class UpdateManager: + def __init__(self, is_first_run: bool): + self.read_configuration() + self.is_first_run = is_first_run + + def read_configuration(self): + conf = ConfigParser() + self.update_slips = conf.auto_update_slips() + + def is_first_run(self) -> bool: + """ + The very first time, slips is started by the user via CLI. then + for each new update, it's started by this update manager. + this func returns true if the user just started slips from cli. + """ + return self.is_first_run + + def update_slips_version(self): + if self.is_first_run(): + # we're not live updating, there isnt going to be an older + # version of slips draining in this case. + ... + else: + # prep for handover. old version to the new one. + ... + + def should_update_slips(self) -> bool: + if not self.update_slips: + return False + + # Never live update when analyzing anything other than an interface + # If not running on interface: return false + # return (new_version_available() and + # new_version_supports_backwards_compatibility()): From 9e1ff16cd3139b0740ef8b209cafd59e9065722d Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 16 Apr 2026 23:45:46 +0200 Subject: [PATCH 06/51] update manager: add a parser for update.json to check for backwards compatibility --- managers/update_manager.py | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/managers/update_manager.py b/managers/update_manager.py index a7ef4b4860..caccef408c 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -6,6 +6,12 @@ Handles updating of slips version """ +import json +import re +from typing import Any, Dict, Optional +from urllib import error, request + +from git import InvalidGitRepositoryError, NoSuchPathError, Repo from slips_files.common.parsers.config_parser import ConfigParser @@ -13,11 +19,105 @@ class UpdateManager: def __init__(self, is_first_run: bool): self.read_configuration() self.is_first_run = is_first_run + self.cached_update_info: Optional[Dict[str, Any]] = None def read_configuration(self): conf = ConfigParser() self.update_slips = conf.auto_update_slips() + def _get_master_update_json_link(self) -> Optional[str]: + """ + Build the raw GitHub URL for update.json on the master branch. + + Returns: + The raw update.json URL if the origin remote is supported, + otherwise None. + """ + try: + remote_url = Repo(".").remote("origin").url + except (ValueError, InvalidGitRepositoryError, NoSuchPathError): + return None + + remote_url = remote_url.strip().removesuffix(".git").rstrip("/") + github_prefixes = ( + "https://github.com/", + "http://github.com/", + "ssh://git@github.com/", + ) + + repo_path = None + if remote_url.startswith("git@github.com:"): + repo_path = remote_url.split(":", maxsplit=1)[1] + else: + for prefix in github_prefixes: + if remote_url.startswith(prefix): + repo_path = remote_url.removeprefix(prefix) + break + + if not repo_path: + return None + + return ( + f"https://raw.githubusercontent.com/{repo_path}/master/update.json" + ) + + def _read_master_update_json(self) -> Dict[str, Any]: + """ + Read the update.json metadata from the origin/master branch. + + Returns: + Parsed update metadata if it can be fetched and decoded, + otherwise an empty dictionary. + """ + if self.cached_update_info is not None: + return self.cached_update_info + + update_json_link = self._get_master_update_json_link() + if not update_json_link: + self.cached_update_info = {} + return self.cached_update_info + + try: + with request.urlopen(update_json_link, timeout=5) as response: + update_text = response.read().decode("utf-8") + except (OSError, UnicodeDecodeError, error.URLError): + self.cached_update_info = {} + return self.cached_update_info + + sanitized_update_text = re.sub(r",(\s*[}\]])", r"\1", update_text) + try: + update_data = json.loads(sanitized_update_text) + except json.JSONDecodeError: + self.cached_update_info = {} + return self.cached_update_info + + self.cached_update_info = ( + update_data if isinstance(update_data, dict) else {} + ) + return self.cached_update_info + + def new_version_has_new_dependencies(self) -> bool: + """ + Check whether the version on master introduces new dependencies. + + Returns: + True if update.json reports new dependencies or the metadata + cannot be read safely, otherwise False. + """ + update_data = self._read_master_update_json() + return bool(update_data.get("has_new_dependencies", True)) + + def new_version_is_backwards_compatible(self) -> bool: + """ + Check whether the version on master is backwards compatible. + + Returns: + True if update.json marks the update as backwards compatible, + otherwise False. + """ + update_data = self._read_master_update_json() + return bool(update_data.get("backwards_compatible", False)) + def is_first_run(self) -> bool: """ The very first time, slips is started by the user via CLI. then From a0d172a714e6269822201190b35eb6d6e69f7c67 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 17 Apr 2026 00:18:04 +0200 Subject: [PATCH 07/51] add an undocumented flag (-u) that will only be used by and older slips to indicate to the updated slips that handover is happening --- slips_files/common/parsers/arg_parser.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/slips_files/common/parsers/arg_parser.py b/slips_files/common/parsers/arg_parser.py index 7407544590..c5144c3a80 100644 --- a/slips_files/common/parsers/arg_parser.py +++ b/slips_files/common/parsers/arg_parser.py @@ -18,6 +18,8 @@ def __init__(self, *args, **kwargs): def add_argument(self, *args, **kwargs): super(ArgumentParser, self).add_argument(*args, **kwargs) + if kwargs.get("help") == argparse.SUPPRESS: + return option = {"flags": list(args)} for key in kwargs: option[key] = kwargs[key] @@ -308,6 +310,14 @@ def parse_arguments(self): action="store_true", help="Internal use only, prevents infinite recursion for cpu profiler dev mode multiprocess tracking", ) + # Internal flag used when Slips starts a newer version of itself. + self.add_argument( + "-u", + dest="is_slips_started_by_an_update", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) try: self.add_argument( "-h", From 289bdfa254242c55e8c666ecf82c51d58a46db77 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 17 Apr 2026 00:41:21 +0200 Subject: [PATCH 08/51] add a function to check if a new slips version is available by comparing master's version and the local one --- managers/update_manager.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index caccef408c..d38d06c21a 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -118,16 +118,18 @@ def new_version_is_backwards_compatible(self) -> bool: update_data = self._read_master_update_json() return bool(update_data.get("backwards_compatible", False)) - def is_first_run(self) -> bool: - """ - The very first time, slips is started by the user via CLI. then - for each new update, it's started by this update manager. - this func returns true if the user just started slips from cli. - """ - return self.is_first_run + def _is_new_version_available(self) -> bool: + update_data = self._read_master_update_json() + latest_version = bool(update_data.get("version", False)) + + if not latest_version: + return False + + current_version = open("VERSION").read().strip() + return current_version == latest_version - def update_slips_version(self): - if self.is_first_run(): + def _update_slips_version(self): + if self.is_first_run: # we're not live updating, there isnt going to be an older # version of slips draining in this case. ... From 2f9f47ed9847256220ffa2f535f0e347d7be6e56 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 17 Apr 2026 00:46:10 +0200 Subject: [PATCH 09/51] update_manager: only update slips if the auto_update param in the config file is set, and we're running on an interface, and there is a new compatible version of slips available. --- managers/update_manager.py | 54 +++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index d38d06c21a..307176fb60 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -13,17 +13,26 @@ from git import InvalidGitRepositoryError, NoSuchPathError, Repo from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.core.database.database_manager import DBManager class UpdateManager: - def __init__(self, is_first_run: bool): - self.read_configuration() - self.is_first_run = is_first_run + def __init__(self, db: DBManager): + self.db = db + self.is_running_non_stop: bool = self.db.is_running_non_stop() self.cached_update_info: Optional[Dict[str, Any]] = None + self.conf = ConfigParser() + self.args = self.conf.get_args() + # The very first time, slips is started by the user via CLI. then + # for each new update, it's started by this update manager. + # this func returns true if the user just started slips from cli. + self.is_first_run: bool = ( + True if not (self.args.is_slips_started_by_an_update) else False + ) + self._read_configuration() - def read_configuration(self): - conf = ConfigParser() - self.update_slips = conf.auto_update_slips() + def _read_configuration(self): + self.auto_update_slips_enabled = self.conf.auto_update_slips() def _get_master_update_json_link(self) -> Optional[str]: """ @@ -63,7 +72,7 @@ def _get_master_update_json_link(self) -> Optional[str]: def _read_master_update_json(self) -> Dict[str, Any]: """ - Read the update.json metadata from the origin/master branch. + Read the update.json file from the origin/master branch of slips repo. Returns: Parsed update metadata if it can be fetched and decoded, @@ -96,7 +105,7 @@ def _read_master_update_json(self) -> Dict[str, Any]: ) return self.cached_update_info - def new_version_has_new_dependencies(self) -> bool: + def _new_version_has_new_dependencies(self) -> bool: """ Check whether the version on master introduces new dependencies. @@ -107,7 +116,7 @@ def new_version_has_new_dependencies(self) -> bool: update_data = self._read_master_update_json() return bool(update_data.get("has_new_dependencies", True)) - def new_version_is_backwards_compatible(self) -> bool: + def _new_version_is_backwards_compatible(self) -> bool: """ Check whether the version on master is backwards compatible. @@ -128,7 +137,7 @@ def _is_new_version_available(self) -> bool: current_version = open("VERSION").read().strip() return current_version == latest_version - def _update_slips_version(self): + def _update_slips(self): if self.is_first_run: # we're not live updating, there isnt going to be an older # version of slips draining in this case. @@ -138,10 +147,25 @@ def _update_slips_version(self): ... def should_update_slips(self) -> bool: - if not self.update_slips: + """ + returns true if the auto_update param in the config file is set to + true, and we're running on an interface, and there is a new + compatible version of slips. + """ + if not self.auto_update_slips_enabled: + return False + + if not self.is_running_non_stop: + # only update slips when running on an interface. + return False + + if not self._is_new_version_available(): return False - # Never live update when analyzing anything other than an interface - # If not running on interface: return false - # return (new_version_available() and - # new_version_supports_backwards_compatibility()): + if ( + self._new_version_is_backwards_compatible() + and not self._new_version_has_new_dependencies() + ): + return True + + return False From 4ed5e795f3755f83fc03ce7488936383f9cfc80f Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 00:04:39 +0200 Subject: [PATCH 10/51] update_manager: check for available updates once a day --- managers/update_manager.py | 42 +++++++++++++++++++++++++++++--------- slips/main.py | 7 +++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 307176fb60..dc341e6122 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -8,6 +8,7 @@ import json import re +import time from typing import Any, Dict, Optional from urllib import error, request @@ -116,7 +117,7 @@ def _new_version_has_new_dependencies(self) -> bool: update_data = self._read_master_update_json() return bool(update_data.get("has_new_dependencies", True)) - def _new_version_is_backwards_compatible(self) -> bool: + def _is_new_version_backwards_compatible(self) -> bool: """ Check whether the version on master is backwards compatible. @@ -137,14 +138,35 @@ def _is_new_version_available(self) -> bool: current_version = open("VERSION").read().strip() return current_version == latest_version - def _update_slips(self): - if self.is_first_run: - # we're not live updating, there isnt going to be an older - # version of slips draining in this case. - ... - else: - # prep for handover. old version to the new one. - ... + def update_slips(self): + # if self.is_first_run: + # # we're not live updating, there isnt going to be an older + # # version of slips draining in this case. + # ... + # else: + # # prep for handover. old version to the new one. + # ... + # self.is_slips_live_updating.set() + ... + + def _did_1d_pass_since_last_update(self) -> bool: + """ + returns true once every 1 day. + """ + update_interval = 60 * 60 * 24 + if time.time() >= self.last_update_time + update_interval: + self.last_update_time = time.time() + return True + return False + + def check_for_update_every_1_day(self) -> bool: + """ + return sTrue if a new compatible version is available and slips + should update itself + """ + if self._did_1d_pass_since_last_update(): + return self.should_update_slips() + return False def should_update_slips(self) -> bool: """ @@ -163,7 +185,7 @@ def should_update_slips(self) -> bool: return False if ( - self._new_version_is_backwards_compatible() + self._is_new_version_backwards_compatible() and not self._new_version_has_new_dependencies() ): return True diff --git a/slips/main.py b/slips/main.py index 072b7794cc..6d46e6f560 100644 --- a/slips/main.py +++ b/slips/main.py @@ -21,6 +21,7 @@ from managers.profilers_manager import ProfilersManager from managers.redis_manager import RedisManager from managers.ui_manager import UIManager +from managers.update_manager import UpdateManager from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.performance_paths import get_performance_plots_dir from slips_files.common.printer import Printer @@ -576,6 +577,10 @@ def start(self): } ) + self.update_man = UpdateManager( + self.db, self.proc_man.is_slips_live_updating + ) + # this func should be called as soon as we start the db, # before evdience proc starts. # to be able to use the host IP as analyzer IP in alerts.json @@ -728,6 +733,8 @@ def sig_handler(sig, frame): ) self.host_ip_man.update_host_ip(host_ips, modified_profiles) + if self.update_man.check_for_update_every_1_day(): + self.update_man.update_slips() except KeyboardInterrupt: # the EINTR error code happens if a signal occurred while From 955e257dbe13dd0aaf7c162352143d229472e060 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 00:04:57 +0200 Subject: [PATCH 11/51] Add functions to keep track of the offset of the last line read in each zeek log file --- slips_files/core/database/database_manager.py | 6 +++++ .../core/database/redis_db/database.py | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index dd04f05151..fbe1a74c02 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -631,6 +631,12 @@ def add_zeek_file(self, *args, **kwargs): def get_all_zeek_files(self, *args, **kwargs): return self.rdb.get_all_zeek_files(*args, **kwargs) + def store_open_zeek_files_offsets(self, *args, **kwargs): + return self.rdb.store_open_zeek_files_offsets(*args, **kwargs) + + def get_open_zeek_files_offsets(self, *args, **kwargs): + return self.rdb.get_open_zeek_files_offsets(*args, **kwargs) + def get_gateway_ip(self, *args, **kwargs): return self.rdb.get_gateway_ip(*args, **kwargs) diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index b834d1f23b..81df256740 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -1280,6 +1280,28 @@ def get_all_zeek_files(self) -> set: """Return all entries from the list of zeek files""" return self.r.hgetall(self.constants.ZEEK_FILES) + def store_open_zeek_files_offsets(self, zeek_files_offsets: dict): + """ + Store the current offsets for open Zeek files. + + :param zeek_files_offsets: Mapping of Zeek logfile path to offset. + :return: None + """ + self.r.delete(self.constants.ZEEK_OPEN_FILES_OFFSETS) + if zeek_files_offsets: + self.r.hset( + self.constants.ZEEK_OPEN_FILES_OFFSETS, + mapping=zeek_files_offsets, + ) + + def get_open_zeek_files_offsets(self) -> dict: + """ + Retrieve the current offsets for open Zeek files. + + :return: Mapping of Zeek logfile path to offset. + """ + return self.r.hgetall(self.constants.ZEEK_OPEN_FILES_OFFSETS) + def _get_gw_info(self, interface: str) -> Dict[str, str] | None: """ gets the gw of the given interface, when slips is runnuning on a From fe26c4fef161cc38e0d20acbf0b3a662639c9cce Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 00:18:03 +0200 Subject: [PATCH 12/51] rename the old Updatemanager to FeedsUpdateManager to avoid name matching with slips version update manager --- config/slips.yaml | 6 +++--- managers/process_manager.py | 10 ++++++---- .../__init__.py | 0 .../feeds_update_manager.py} | 4 ++-- .../timer_manager.py | 0 modules/threat_intelligence/threat_intelligence.py | 2 +- slips_files/core/helpers/bloom_filters_manager.py | 2 +- tests/integration/config/fides_config.yaml | 2 +- tests/integration/config/test.yaml | 2 +- tests/integration/test.yaml | 2 +- tests/integration/test2.yaml | 2 +- tests/integration/test_config_files.py | 4 ++-- tests/module_factory.py | 4 ++-- tests/unit/managers/test_process_manager.py | 2 +- .../modules/update_manager/test_update_file_manager.py | 2 +- 15 files changed, 23 insertions(+), 21 deletions(-) rename modules/{update_manager => feeds_update_manager}/__init__.py (100%) rename modules/{update_manager/update_manager.py => feeds_update_manager/feeds_update_manager.py} (99%) rename modules/{update_manager => feeds_update_manager}/timer_manager.py (100%) diff --git a/config/slips.yaml b/config/slips.yaml index 2d98b98139..ef22d6f1fb 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -3,7 +3,7 @@ --- update: # Enable automatic live updates of the installed Slips version. - # This setting is separate from the update_manager module, which only + # This setting is separate from the feeds_update_manager module, which only # updates TI feeds and related files during runtime. # Automatic Slips updates may overwrite the default config files shipped # with Slips. If you want to keep your local configuration changes intact, avoid editing the default config files. @@ -215,7 +215,7 @@ modules: # Add the names of other modules that you want to disable # (use module snake_case names). Example, # threat_intelligence, blocking, network_discovery, timeline, virustotal, - # rnn_cc_detection, flow_ml_detection, update_manager + # rnn_cc_detection, flow_ml_detection, feeds_update_manager disable: [template] # For each line in timeline file there is a timestamp. @@ -311,7 +311,7 @@ virustotal: ############################# threatintelligence: - # By default, slips starts without the TI files, and runs the update_manager + # By default, slips starts without the TI files, and runs the feeds_update_manager # in the background. If this option is set to true, slips will not start # analyzing the flows until the update manager finished and all TI files are # loaded successfully. diff --git a/managers/process_manager.py b/managers/process_manager.py index 2c50143570..9af8bd4551 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -32,7 +32,9 @@ import modules -from modules.update_manager.update_manager import UpdateManager +from modules.feeds_update_manager.feeds_update_manager import ( + FeedsUpdateManager, +) from slips_files.common.slips_utils import utils from slips_files.common.abstracts.imodule import ( IModule, @@ -497,7 +499,7 @@ def start_update_manager(self, local_files=False, ti_feeds=False): with Lock(name="slips_ports_and_orgs"): # pass a dummy termination event for update manager to # update orgs and ports info - update_manager = UpdateManager( + update_manager = FeedsUpdateManager( self.main.logger, self.main.args.output, self.main.redis_port, @@ -541,9 +543,9 @@ def warn_about_pending_modules(self, pending_modules: List[Process]): ) # check if update manager is still alive - if "update_manager" in pending_module_names: + if "feeds_update_manager" in pending_module_names: self.main.print( - "update_manager may take several minutes " + "feeds_update_manager may take several minutes " "to finish updating 45+ TI files." ) diff --git a/modules/update_manager/__init__.py b/modules/feeds_update_manager/__init__.py similarity index 100% rename from modules/update_manager/__init__.py rename to modules/feeds_update_manager/__init__.py diff --git a/modules/update_manager/update_manager.py b/modules/feeds_update_manager/feeds_update_manager.py similarity index 99% rename from modules/update_manager/update_manager.py rename to modules/feeds_update_manager/feeds_update_manager.py index 46f0f82390..43afa4f71c 100644 --- a/modules/update_manager/update_manager.py +++ b/modules/feeds_update_manager/feeds_update_manager.py @@ -29,8 +29,8 @@ from slips_files.core.helpers.whitelist.whitelist import Whitelist -class UpdateManager(IModule): - name = "update_manager" +class FeedsUpdateManager(IModule): + name = "feeds_update_manager" description = "Update Threat Intelligence files" authors = ["Kamila Babayeva", "Alya Gomaa"] diff --git a/modules/update_manager/timer_manager.py b/modules/feeds_update_manager/timer_manager.py similarity index 100% rename from modules/update_manager/timer_manager.py rename to modules/feeds_update_manager/timer_manager.py diff --git a/modules/threat_intelligence/threat_intelligence.py b/modules/threat_intelligence/threat_intelligence.py index 507ef315e3..f824f67a04 100644 --- a/modules/threat_intelligence/threat_intelligence.py +++ b/modules/threat_intelligence/threat_intelligence.py @@ -1891,7 +1891,7 @@ def pre_main(self): utils.drop_root_privs_permanently() # Load the local Threat Intelligence files that are # stored in the local folder self.path_to_local_ti_files - # The remote files are being loaded by the update_manager + # The remote files are being loaded by the feeds_update_manager local_files = ( "own_malicious_iocs.csv", "own_malicious_JA3.csv", diff --git a/slips_files/core/helpers/bloom_filters_manager.py b/slips_files/core/helpers/bloom_filters_manager.py index a44840a6ee..a15b389e4b 100644 --- a/slips_files/core/helpers/bloom_filters_manager.py +++ b/slips_files/core/helpers/bloom_filters_manager.py @@ -60,7 +60,7 @@ def _init_whitelisted_orgs_bf(self): Updates the bloom filters with the whitelisted organization domains, asns, and ips fills the self.org_filters dict - is called from update_manager whether slips did update its local + is called from feeds_update_manager whether slips did update its local org files or not. this goal of calling this is to make sure slips has the bloom filters in mem at all times. diff --git a/tests/integration/config/fides_config.yaml b/tests/integration/config/fides_config.yaml index 734d227b3f..00618fd6fb 100644 --- a/tests/integration/config/fides_config.yaml +++ b/tests/integration/config/fides_config.yaml @@ -206,7 +206,7 @@ virustotal: ############################# threatintelligence: - # by default, slips starts without the TI files, and runs the update_manager in the background + # by default, slips starts without the TI files, and runs the feeds_update_manager in the background # if thi option is set to yes, slips will not start untill the update manager is done # and all TI files are loaded successfully # this is usefull if you want to ensure that slips doesn't miss the detection of any blacklisted IPs diff --git a/tests/integration/config/test.yaml b/tests/integration/config/test.yaml index aca544389f..9c8f739576 100644 --- a/tests/integration/config/test.yaml +++ b/tests/integration/config/test.yaml @@ -64,7 +64,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true parameters: analysis_direction: all diff --git a/tests/integration/test.yaml b/tests/integration/test.yaml index fb64279b53..d44e29e3c9 100644 --- a/tests/integration/test.yaml +++ b/tests/integration/test.yaml @@ -90,7 +90,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true output: logs: slips.log diff --git a/tests/integration/test2.yaml b/tests/integration/test2.yaml index 009bb60b1d..fd16490916 100644 --- a/tests/integration/test2.yaml +++ b/tests/integration/test2.yaml @@ -89,7 +89,7 @@ modules: - template - ensembling - flow_ml_detection - - update_manager + - feeds_update_manager timeline_human_timestamp: true output: logs: slips.log diff --git a/tests/integration/test_config_files.py b/tests/integration/test_config_files.py index 6c6b5c055c..fc1da3c6c4 100644 --- a/tests/integration/test_config_files.py +++ b/tests/integration/test_config_files.py @@ -77,7 +77,7 @@ def test_conf_file(pcap_path, expected_profiles, output_dir, redis_port): "template", "ensembling", "flow_ml_detection", - "update_manager", + "feeds_update_manager", ] }, }, @@ -172,7 +172,7 @@ def test_conf_file2(pcap_path, expected_profiles, output_dir, redis_port): "template", "ensembling", "flow_ml_detection", - "update_manager", + "feeds_update_manager", ] }, }, diff --git a/tests/module_factory.py b/tests/module_factory.py index 30972a7af7..6851169b55 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -659,9 +659,9 @@ def create_spamhaus_obj(self, mock_db): @patch(MODULE_DB_MANAGER, name="mock_db") def create_update_manager_obj(self, mock_db): - from modules.update_manager.update_manager import UpdateManager + from modules.update_manager.update_manager import FeedsUpdateManager - update_manager = UpdateManager( + update_manager = FeedsUpdateManager( logger=self.logger, output_dir="dummy_output_dir", redis_port=6379, diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index 386dd9ae35..bc9125b661 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -115,7 +115,7 @@ def test_print_disabled_modules(): [ # Test case 1: No pending modules, no additional print calls ([], 1), - # Test case 2: Pending modules without update_manager, one additional print call + # Test case 2: Pending modules without feeds_update_manager, one additional print call ([Mock(name="Module1"), Mock(name="Module2")], 1), ], ) diff --git a/tests/unit/modules/update_manager/test_update_file_manager.py b/tests/unit/modules/update_manager/test_update_file_manager.py index faa7bc84aa..838ad74bea 100644 --- a/tests/unit/modules/update_manager/test_update_file_manager.py +++ b/tests/unit/modules/update_manager/test_update_file_manager.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only -"""Unit test for modules/update_manager/update_manager.py""" +"""Unit test for modules/feeds_update_manager/feeds_update_manager.py""" from tests.module_factory import ModuleFactory import json From 1ec413c5cc943cf9f9f75da0bca2c2f53d6c09b4 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 00:20:21 +0200 Subject: [PATCH 13/51] process_manager: create a mp event and pass it to input and update manager as a way for telling input that slips is currently updating. --- managers/process_manager.py | 9 +++++++++ managers/update_manager.py | 6 ++++-- slips/main.py | 5 +---- slips_files/core/database/redis_db/constants.py | 1 + slips_files/core/input/input.py | 3 ++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 9af8bd4551..2d5e225a45 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -32,6 +32,7 @@ import modules +from managers.update_manager import UpdateManager from modules.feeds_update_manager.feeds_update_manager import ( FeedsUpdateManager, ) @@ -90,6 +91,7 @@ def __init__(self, main): # is set by the input process to indicate no more flows are coming # so profiler can safely begin shutdown/joins. self.is_input_done_event = Event() + self.is_slips_live_updating = Event() self.read_config() def read_config(self): @@ -101,6 +103,12 @@ def read_config(self): # self.bootstrap_p2p, self.boootstrapping_modules = self.main.conf. # get_bootstrapping_setting() + def start_slips_update_manager(self): + return UpdateManager( + database=self.main.db, + is_slips_live_updating=self.is_slips_live_updating, + ) + def start_output_process(self, stderr, slips_logfile, stdout=""): output_process = Output( stdout=stdout, @@ -180,6 +188,7 @@ def start_input_process(self): line_type=self.main.line_type, is_profiler_done_event=self.is_profiler_done_event, is_input_done_event=self.is_input_done_event, + is_slips_live_updating=self.is_slips_live_updating, ) input_process.start() self.main.print( diff --git a/managers/update_manager.py b/managers/update_manager.py index dc341e6122..9115e5c1cb 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -18,8 +18,9 @@ class UpdateManager: - def __init__(self, db: DBManager): + def __init__(self, db: DBManager = None, is_slips_live_updating=None): self.db = db + self.is_slips_live_updating = is_slips_live_updating self.is_running_non_stop: bool = self.db.is_running_non_stop() self.cached_update_info: Optional[Dict[str, Any]] = None self.conf = ConfigParser() @@ -28,9 +29,10 @@ def __init__(self, db: DBManager): # for each new update, it's started by this update manager. # this func returns true if the user just started slips from cli. self.is_first_run: bool = ( - True if not (self.args.is_slips_started_by_an_update) else False + True if not self.args.is_slips_started_by_an_update else False ) self._read_configuration() + self.last_update_time = 0 def _read_configuration(self): self.auto_update_slips_enabled = self.conf.auto_update_slips() diff --git a/slips/main.py b/slips/main.py index 6d46e6f560..b4ce5f095e 100644 --- a/slips/main.py +++ b/slips/main.py @@ -21,7 +21,6 @@ from managers.profilers_manager import ProfilersManager from managers.redis_manager import RedisManager from managers.ui_manager import UIManager -from managers.update_manager import UpdateManager from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.performance_paths import get_performance_plots_dir from slips_files.common.printer import Printer @@ -577,9 +576,7 @@ def start(self): } ) - self.update_man = UpdateManager( - self.db, self.proc_man.is_slips_live_updating - ) + self.update_man = self.proc_man.start_slips_update_manager() # this func should be called as soon as we start the db, # before evdience proc starts. diff --git a/slips_files/core/database/redis_db/constants.py b/slips_files/core/database/redis_db/constants.py index 9d94a23430..83fc9de27b 100644 --- a/slips_files/core/database/redis_db/constants.py +++ b/slips_files/core/database/redis_db/constants.py @@ -43,6 +43,7 @@ class Constants: LABELS = "labels" MSGS_PUBLISHED_AT_RUNTIME = "msgs_published_at_runtime" ZEEK_FILES = "zeekfiles" + ZEEK_OPEN_FILES_OFFSETS = "zeek_open_files_offsets" DEFAULT_GATEWAY = "default_gateway" IS_CYST_ENABLED = "is_cyst_enabled" LOCAL_NETWORK = "local_network" diff --git a/slips_files/core/input/input.py b/slips_files/core/input/input.py index de594dc0ec..ca97955192 100644 --- a/slips_files/core/input/input.py +++ b/slips_files/core/input/input.py @@ -24,7 +24,6 @@ from slips_files.common.abstracts.icore import ICore from slips_files.common.input_type import InputType -# common imports for all modules from slips_files.common.parsers.config_parser import ConfigParser import multiprocessing @@ -60,6 +59,7 @@ def init( line_type=None, is_profiler_done_event: multiprocessing.Event = None, is_input_done_event: multiprocessing.Event = None, + is_slips_live_updating: multiprocessing.Event = None, ): self.input_type = input_type self.profiler_queue = profiler_queue @@ -92,6 +92,7 @@ def init( self.is_profiler_done_event = is_profiler_done_event # is set by this proc to indicate no more flows are coming self.is_input_done_event = is_input_done_event + self.is_slips_live_updating = is_slips_live_updating self.is_running_non_stop: bool = self.db.is_running_non_stop() self.input_handlers = self._build_input_handlers() self.active_handler = None From b72d2b29d340300fe8670719cc1b238bb5f3247b Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 00:32:16 +0200 Subject: [PATCH 14/51] store the last read zeek logs offsets in the db when input detects that slips is auto updating --- .../core/input/zeek/utils/zeek_input_utils.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 7cc9ccc8b8..51f98fab9c 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -22,6 +22,10 @@ def __init__(self, input_process): self.open_file_handles = {} self.open_file_handlers_lock = threading.RLock() self.cache_lines = {} + # used to start reading from where slips left off after slips auto + # updates. + self.zeek_logs_offsets = {} + self.last_consumed_offsets = {} self.file_time = {} self.last_updated_file_time = None self.rotated_files_to_delete: List[Tuple[str, float]] = [] @@ -164,6 +168,7 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): file_handle.readline() zeek_line = file_handle.readline() + line_end_offset = file_handle.tell() except ValueError: # remover thread just finished closing all old handles. @@ -195,8 +200,22 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): "data": nline, "interface": interface, } + self.zeek_logs_offsets[filename] = line_end_offset return True + def store_current_open_zeek_files_offsets_in_db(self): + """ + Store the last committed offset for each currently open Zeek file. + + :return: None + """ + with self.open_file_handlers_lock: + current_offsets = { + filename: str(self.last_consumed_offsets.get(filename, 0)) + for filename in self.open_file_handles + } + self.input.db.store_open_zeek_files_offsets(current_offsets) + def reached_timeout(self) -> bool: # If we don't have any cached lines to send, # it may mean that new lines are not arriving. Check @@ -251,6 +270,20 @@ def get_earliest_line(self): earliest_line = self.cache_lines[file_with_earliest_flow] return earliest_line, file_with_earliest_flow + def _get_newest_known_offset(self, file): + previously_saved_offset = self.last_consumed_offsets.get(file, 0) + return self.zeek_logs_offsets.get(file, previously_saved_offset) + + def _update_offsets(self, file): + newest_offset = self._get_newest_known_offset(file) + self.last_consumed_offsets[file] = newest_offset + + if ( + self.input.is_slips_live_updating is not None + and self.input.is_slips_live_updating.is_set() + ): + self.store_current_open_zeek_files_offsets_in_db() + def read_zeek_files(self) -> int: """ Runs when slips is analyzing pcaps, interface, zeek dirs, and zeek @@ -263,6 +296,8 @@ def read_zeek_files(self) -> int: # that file self.file_time = {} self.cache_lines = {} + self.zeek_logs_offsets = {} + self.last_consumed_offsets = {} # Try to keep track of when was the last update so we stop this # reading self.last_updated_file_time = datetime.datetime.now() @@ -298,11 +333,15 @@ def read_zeek_files(self) -> int: self.input.give_profiler(earliest_line) self.input.lines += 1 + + self._update_offsets(file_with_earliest_flow) + # when testing, no need to read the whole file! if self.input.lines == 10 and self.input.testing: break # Delete this line from the cache and the time list del self.cache_lines[file_with_earliest_flow] + self.zeek_logs_offsets.pop(file_with_earliest_flow, None) del self.file_time[file_with_earliest_flow] # Get the new list of files. Since new files may have been created by From 4f2032840541f869bfa263cb4420be60419eb059 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 01:02:31 +0200 Subject: [PATCH 15/51] send the current slips version with each pub/sub msg to avoid processing msgs sent by the updated version of slips during handover --- managers/update_manager.py | 10 ++++++---- slips_files/common/slips_utils.py | 5 +++++ .../core/database/redis_db/database.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 9115e5c1cb..ff867a2239 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -14,12 +14,15 @@ from git import InvalidGitRepositoryError, NoSuchPathError, Repo from slips_files.common.parsers.config_parser import ConfigParser +from slips_files.common.slips_utils import utils from slips_files.core.database.database_manager import DBManager class UpdateManager: - def __init__(self, db: DBManager = None, is_slips_live_updating=None): - self.db = db + def __init__( + self, database: DBManager = None, is_slips_live_updating=None + ): + self.db = database self.is_slips_live_updating = is_slips_live_updating self.is_running_non_stop: bool = self.db.is_running_non_stop() self.cached_update_info: Optional[Dict[str, Any]] = None @@ -137,8 +140,7 @@ def _is_new_version_available(self) -> bool: if not latest_version: return False - current_version = open("VERSION").read().strip() - return current_version == latest_version + return utils.get_current_version() == latest_version def update_slips(self): # if self.is_first_run: diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 000d4bd4bc..9c2dc8639e 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -920,6 +920,11 @@ def assert_microseconds(self, ts: str): ts = ts + "0" * (6 - len(ts.split(".")[-1])) return ts + def get_current_version(self) -> str: + with open("VERSION", "r") as version_file: + current_version = version_file.read().strip() + return current_version + def _convert_str_port_to_int(self, port) -> int: if isinstance(port, str): try: diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 81df256740..6e7ac525bf 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -43,6 +43,7 @@ RUNNING_IN_DOCKER = os.environ.get("IS_IN_A_DOCKER_CONTAINER", False) LOCALHOST = "127.0.0.1" +VERSION = utils.get_current_version() class RedisDB( @@ -507,10 +508,28 @@ def ping(self): self.r.ping() self.rcache.ping() + def _add_version_to_msg(self, msg): + if isinstance(msg, str): + try: + msg = json.loads(msg) + msg.update({"version": VERSION}) + msg = json.dumps(msg) + except json.decoder.JSONDecodeError: + # the msg is 1 str + msg = { + "text": msg, + "version": VERSION, + } + msg = json.dumps(msg) + elif isinstance(msg, dict): + msg.update({"version": VERSION}) + return msg + def publish(self, channel, msg, pipeline=None): """Publish a msg in the given channel. adds the instructions to the given pipeline if given and returns the pipeline""" + msg = self._add_version_to_msg(msg) # keeps track of how many msgs were published in the given channel if pipeline is not None: From 0c9ee47b30b4f53d9d05c9c1f024faa3668e40fc Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 01:04:52 +0200 Subject: [PATCH 16/51] imodule: ensure that the pub/sub recvd msg doesn't belong to the updated slips run. only processes ones intended for the current version. --- slips_files/common/abstracts/imodule.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/slips_files/common/abstracts/imodule.py b/slips_files/common/abstracts/imodule.py index 2d436c221f..34b4502cdb 100644 --- a/slips_files/common/abstracts/imodule.py +++ b/slips_files/common/abstracts/imodule.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only +import json import os import sys import traceback @@ -69,6 +70,7 @@ def __init__( ) self.db.client_setname(self.name) self.keyboard_int_ctr = 0 + self.slips_version: str = utils.get_slips_version() self.init(**kwargs) # should after the module's init() so the module has a chance to # set its own channels @@ -198,6 +200,28 @@ def _pre_main(self): self.channel_tracker = self.init_channel_tracker() return self.pre_main() + def is_msg_version_compatible(self, message: dict) -> bool: + """ + Check whether the incoming pub/sub message matches this module's + Slips version. + """ + if not message or "data" not in message: + return False + + data = message["data"] + if not isinstance(data, str): + return False + + try: + payload = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError): + return False + + if not isinstance(payload, dict): + return False + + return payload.get("version") == self.slips_version + def get_msg(self, channel: str) -> Optional[dict]: try: if channel not in self.channels: @@ -209,6 +233,12 @@ def get_msg(self, channel: str) -> Optional[dict]: message = self.db.get_message(self.channels[channel]) if utils.is_msg_intended_for(message, channel): + # discard msgs if sent be a newer/older version of slips. + # may happen temporarily during handover, where the new + # version starts before the old version stops. + if not self.is_msg_version_compatible(message): + self.channel_tracker[channel]["msg_received"] = False + return None self.channel_tracker[channel]["msg_received"] = True self.db.incr_msgs_received_in_channel(self.name, channel) return message From 52bf7fe5ff04e53e46a37eeb0bec29643f7ed52a Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 18 Apr 2026 01:07:56 +0200 Subject: [PATCH 17/51] extract the txt msg only from the pubsub msgs and ignore metadata like slips version (the version is checked earlier by get_msg()) --- managers/process_manager.py | 2 +- modules/arp/arp.py | 2 +- modules/arp_poisoner/arp_poisoner.py | 2 +- modules/blocking/blocking.py | 2 +- .../brute_force_detector.py | 2 +- .../feeds_update_manager.py | 2 +- modules/fides/fides.py | 2 +- modules/flow_alerts/conn.py | 2 +- modules/ip_info/ip_info.py | 2 +- .../network_discovery/network_discovery.py | 2 +- modules/risk_iq/risk_iq.py | 2 +- modules/rnn_cc_detection/rnn_cc_detection.py | 2 +- .../performance_profilers/memory_profiler.py | 4 ++- slips_files/common/slips_utils.py | 30 +++++++++++++++++++ slips_files/core/input/cyst/cyst_input.py | 8 +++-- 15 files changed, 50 insertions(+), 16 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 2d5e225a45..c4119561ae 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -669,7 +669,7 @@ def is_stop_msg_received(self) -> bool: return ( utils.is_msg_intended_for(message, "control_channel") - and message["data"] == "stop_slips" + and utils.get_msg_payload(message) == "stop_slips" ) def is_debugger_active(self) -> bool: diff --git a/modules/arp/arp.py b/modules/arp/arp.py index e661e1475d..a6848a6795 100644 --- a/modules/arp/arp.py +++ b/modules/arp/arp.py @@ -566,7 +566,7 @@ def main(self): # if the tw is closed, remove all its entries from the cache dict if msg := self.get_msg("tw_closed"): - profileid_tw = msg["data"] + profileid_tw = utils.get_msg_payload(msg) # when a tw is closed, this means that it's too # old so we don't check for arp scan in this time # range anymore diff --git a/modules/arp_poisoner/arp_poisoner.py b/modules/arp_poisoner/arp_poisoner.py index a8dae0a976..1d85f72362 100644 --- a/modules/arp_poisoner/arp_poisoner.py +++ b/modules/arp_poisoner/arp_poisoner.py @@ -414,7 +414,7 @@ def main(self): # if slips saw 3 ips, this channel will receive 3 msgs with tw1 # as closed. we're not interested in the ips, we just wanna # know when slips advances to the next tw. - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") twid = profileid_tw[-1] if self.last_closed_tw != twid: self.last_closed_tw = twid diff --git a/modules/blocking/blocking.py b/modules/blocking/blocking.py index f6b4fbe69e..b449cb1168 100644 --- a/modules/blocking/blocking.py +++ b/modules/blocking/blocking.py @@ -274,7 +274,7 @@ def main(self): # if slips saw 3 ips, this channel will receive 3 msgs with tw1 # as closed. we're not interested in the ips, we just wanna # know when slips advances to the next tw. - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") twid = profileid_tw[-1] if self.last_closed_tw != twid: self.last_closed_tw = twid diff --git a/modules/brute_force_detector/brute_force_detector.py b/modules/brute_force_detector/brute_force_detector.py index 2127004f68..cae097f271 100644 --- a/modules/brute_force_detector/brute_force_detector.py +++ b/modules/brute_force_detector/brute_force_detector.py @@ -473,4 +473,4 @@ def main(self): self._handle_ssh(profileid, twid, flow) if msg := self.get_msg("tw_closed"): - self.cleanup_cache_dicts(msg["data"].split("_")) + self.cleanup_cache_dicts(utils.get_msg_payload(msg).split("_")) diff --git a/modules/feeds_update_manager/feeds_update_manager.py b/modules/feeds_update_manager/feeds_update_manager.py index 43afa4f71c..df21f7b8c3 100644 --- a/modules/feeds_update_manager/feeds_update_manager.py +++ b/modules/feeds_update_manager/feeds_update_manager.py @@ -22,7 +22,7 @@ CannotAcquireLock, ) -from modules.update_manager.timer_manager import InfiniteTimer +from modules.feeds_update_manager.timer_manager import InfiniteTimer from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.abstracts.imodule import IModule from slips_files.common.slips_utils import utils diff --git a/modules/fides/fides.py b/modules/fides/fides.py index 9e5cc07ae7..3837573070 100644 --- a/modules/fides/fides.py +++ b/modules/fides/fides.py @@ -221,7 +221,7 @@ def main(self): if not msg["data"]: return - ip = msg["data"] + ip = utils.get_msg_payload(msg) if utils.detect_ioc_type(ip) != "ip": return diff --git a/modules/flow_alerts/conn.py b/modules/flow_alerts/conn.py index 1ad44ca04d..3a85f496b8 100644 --- a/modules/flow_alerts/conn.py +++ b/modules/flow_alerts/conn.py @@ -840,7 +840,7 @@ async def analyze(self, msg): self.check_device_changing_ips(twid, flow) elif utils.is_msg_intended_for(msg, "tw_closed"): - profileid_tw: List[str] = msg["data"].split("_") + profileid_tw: List[str] = utils.get_msg_payload(msg).split("_") profileid = f"{profileid_tw[0]}_{profileid_tw[1]}" twid = profileid_tw[-1] self.detect_data_upload_in_twid(profileid, twid) diff --git a/modules/ip_info/ip_info.py b/modules/ip_info/ip_info.py index f0ced354e8..fb31ab7a1c 100644 --- a/modules/ip_info/ip_info.py +++ b/modules/ip_info/ip_info.py @@ -596,7 +596,7 @@ async def main(self): self.get_domain_info(domain) if msg := self.get_msg("new_ip"): - ip = msg["data"] + ip = utils.get_msg_payload(msg) self.handle_new_ip(ip) if msg := self.get_msg("check_jarm_hash"): diff --git a/modules/network_discovery/network_discovery.py b/modules/network_discovery/network_discovery.py index 556f40929f..12173765c3 100644 --- a/modules/network_discovery/network_discovery.py +++ b/modules/network_discovery/network_discovery.py @@ -244,5 +244,5 @@ def main(self): self.check_dhcp_scan(profileid, twid, flow) if msg := self.get_msg("tw_closed"): - profileid_tw: List[str] = msg["data"].split("_") + profileid_tw: List[str] = utils.get_msg_payload(msg).split("_") self.cleanup_cache_dicts(profileid_tw) diff --git a/modules/risk_iq/risk_iq.py b/modules/risk_iq/risk_iq.py index 58656cd917..7f8f125a15 100644 --- a/modules/risk_iq/risk_iq.py +++ b/modules/risk_iq/risk_iq.py @@ -93,7 +93,7 @@ def pre_main(self): def main(self): if msg := self.get_msg("new_ip"): - ip = msg["data"] + ip = utils.get_msg_payload(msg) if utils.is_ignored_ip(ip): # return here means keep looping return diff --git a/modules/rnn_cc_detection/rnn_cc_detection.py b/modules/rnn_cc_detection/rnn_cc_detection.py index 873cc8dbff..b5bece382a 100644 --- a/modules/rnn_cc_detection/rnn_cc_detection.py +++ b/modules/rnn_cc_detection/rnn_cc_detection.py @@ -245,7 +245,7 @@ def handle_new_letters(self, msg: Dict): def handle_tw_closed(self, msg: Dict): """handles msgs from the tw_closed channel""" - profileid_tw = msg["data"].split("_") + profileid_tw = utils.get_msg_payload(msg).split("_") profileid = f"{profileid_tw[0]}_{profileid_tw[1]}" twid = profileid_tw[-1] self.letters_exporter.export(profileid, twid) diff --git a/slips_files/common/performance_profilers/memory_profiler.py b/slips_files/common/performance_profilers/memory_profiler.py index 30d1b39789..7952ec7953 100644 --- a/slips_files/common/performance_profilers/memory_profiler.py +++ b/slips_files/common/performance_profilers/memory_profiler.py @@ -19,6 +19,8 @@ import random from abc import ABCMeta +from slips_files.common.slips_utils import utils + class MemoryProfiler(IPerformanceProfiler): profiler = None @@ -214,7 +216,7 @@ def _handle_signal(self): # print(f"Msg {msg}") pid: int = None try: - pid = int(msg["data"]) + pid = int(utils.get_msg_payload(msg)) except ValueError: msg = self.pid_channel.get_message(timeout=timeout) continue diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 9c2dc8639e..4a93563e2b 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -784,6 +784,36 @@ def is_msg_intended_for(self, message, channel): and message["channel"] == channel ) + def get_msg_payload(self, message: dict) -> Any: + """ + Return the actual payload stored in the given message. discards + metadata like "version" and returns the text only. + + Parameters: + message: Pub/sub message returned by Redis. + + Return: + The decoded payload. Wrapped plain-string messages return their text. + """ + data = message["data"] + if not isinstance(data, str): + return data + + try: + decoded = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError): + return data + + if ( + isinstance(decoded, dict) + and "text" in decoded + and "version" in decoded + and len(decoded) == 2 + ): + return decoded["text"] + + return decoded + def get_slips_version(self) -> str: version_file = "VERSION" with open(version_file, "r") as f: diff --git a/slips_files/core/input/cyst/cyst_input.py b/slips_files/core/input/cyst/cyst_input.py index 959482eae8..a823a98c74 100644 --- a/slips_files/core/input/cyst/cyst_input.py +++ b/slips_files/core/input/cyst/cyst_input.py @@ -4,6 +4,7 @@ import json from slips_files.common.abstracts.iinput_handler import IInputHandler +from slips_files.common.slips_utils import utils class CystInput(IInputHandler): @@ -30,13 +31,14 @@ def run(self): # todo when to break? cyst should send something like stop? msg = self.input.get_msg("new_module_flow") - if msg and msg["data"] == "stop_process": + if msg and utils.get_msg_payload(msg) == "stop_process": self.input.shutdown_gracefully() return True if msg := self.input.get_msg("new_module_flow"): - msg: str = msg["data"] - msg = json.loads(msg) + msg = utils.get_msg_payload(msg) + if isinstance(msg, str): + msg = json.loads(msg) flow = msg["flow"] src_module = msg["module"] line_info = { From 63563778ed657cf3437cb326b917148cade6ed44 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 15:55:51 +0200 Subject: [PATCH 18/51] main: don't overwrite the output dir when slips is started with -u & refactor --- slips/main.py | 91 ++++++++++++++---------- slips_files/common/parsers/arg_parser.py | 5 +- slips_files/core/helpers/checker.py | 2 +- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/slips/main.py b/slips/main.py index b4ce5f095e..91f67f0fc6 100644 --- a/slips/main.py +++ b/slips/main.py @@ -40,7 +40,7 @@ class Main: def __init__(self, testing=False): self.name = "main" - self.alerts_default_path = "output/" + self.parent_output_dir = "output/" self.mode = "interactive" self.sigterm_received = False # objects to manage various functionality @@ -73,6 +73,13 @@ def __init__(self, testing=False): self.input_information, self.line_type, ) = self.checker.get_input_type() + self.input_information = os.path.normpath( + self.input_information + ) + self.input_information = self.input_information.replace( + ",", "_" + ) + # If we need zeek (bro), test if we can run it. self.check_zeek_or_bro() self.prepare_output_dir() @@ -171,6 +178,28 @@ def delete_zeek_files(self): if self.conf.delete_zeek_files(): shutil.rmtree(self.zeek_dir) + def del_file_or_dir(self, file): + """deletes a file or dir inside the output dir""" + file_path = os.path.join(self.args.output, file) + with contextlib.suppress(Exception): + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + def construct_output_dir_name(self) -> str: + dir_name = os.path.join( + self.parent_output_dir, + os.path.basename( + self.input_information + ), # get pcap name from path + ) + + # add timestamp to avoid conflicts e.g wlp3s0_2022-03-1_03:55 + ts = utils.convert_ts_format(datetime.now(), "%Y-%m-%d_%H:%M:%S") + dir_name += f"_{ts}/" + return dir_name + def prepare_output_dir(self): """ Clears the output dir if it already exists , or creates a @@ -178,47 +207,33 @@ def prepare_output_dir(self): Log dirs are stored in output/_%Y-%m-%d_%H:%M:%S @return: None """ - # default output/ - if "-o" in sys.argv: - # -o is given - # delete all old files in the output dir - if os.path.exists(self.args.output): - for file in os.listdir(self.args.output): - # in integration tests, slips redirects its - # output to slips_output.txt, - # don't delete that file - if self.args.testing and "slips_output.txt" in file: - continue - - file_path = os.path.join(self.args.output, file) - with contextlib.suppress(Exception): - if os.path.isfile(file_path): - os.remove(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - else: - os.makedirs(self.args.output) + if self.args.is_slips_started_by_an_update: + # we should append to existing files in the output dir, + # and never overwrite them. + return + + if not self.args.output: + # the user didnt give slips an output dir to use, construct one + self.args.output = self.construct_output_dir_name() + os.makedirs(self.args.output) + os.chmod(self.args.output, 0o777) return - # self.args.output is the same as self.alerts_default_path - self.input_information = os.path.normpath(self.input_information) - self.input_information = self.input_information.replace(",", "_") - # now that slips can run several instances, - # each created dir will be named after the instance - # that created it - # it should be output/wlp3s0 - self.args.output = os.path.join( - self.alerts_default_path, - os.path.basename( - self.input_information - ), # get pcap name from path - ) - # add timestamp to avoid conflicts wlp3s0_2022-03-1_03:55 - ts = utils.convert_ts_format(datetime.now(), "%Y-%m-%d_%H:%M:%S") - self.args.output += f"_{ts}/" + # -o is given + # delete all old files in the output dir + if os.path.exists(self.args.output): + for file in os.listdir(self.args.output): + # in integration tests, slips redirects its + # output to slips_output.txt, + # don't delete that file + if self.args.testing and "slips_output.txt" in file: + continue + self.del_file_or_dir(file) + + else: + os.makedirs(self.args.output) - os.makedirs(self.args.output) os.chmod(self.args.output, 0o777) def set_mode(self, mode, daemon=""): diff --git a/slips_files/common/parsers/arg_parser.py b/slips_files/common/parsers/arg_parser.py index c5144c3a80..357d2ee1cb 100644 --- a/slips_files/common/parsers/arg_parser.py +++ b/slips_files/common/parsers/arg_parser.py @@ -14,8 +14,6 @@ def __init__(self, *args, **kwargs): super(ArgumentParser, self).__init__(*args, **kwargs) self.program = {key: kwargs[key] for key in kwargs} - self.alerts_default_path = "output/" - def add_argument(self, *args, **kwargs): super(ArgumentParser, self).add_argument(*args, **kwargs) if kwargs.get("help") == argparse.SUPPRESS: @@ -211,8 +209,7 @@ def parse_arguments(self): action="store", metavar="", required=False, - default=self.alerts_default_path, - help="Store alerts.json and alerts.txt in the given folder.", + help="Store Slips logs in the given folder.", ) self.add_argument( "-s", diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index c34169217a..7a56db80a2 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -18,7 +18,7 @@ def get_input_type(self) -> tuple: supported input_type values are: interface, argus, suricata, zeek, nfdump, db supported input_information: - given filepath, interface or type of line given in stdin, + given filepath, interface, or type of line given in stdin, comma separated access point interfaces like wlan0,eth0 """ # only defined in stdin lines From aae1052a9f50e053151f5dee06355782d5c14dc6 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 16:17:56 +0200 Subject: [PATCH 19/51] dont clear output log files on startup if slips is started with -u --- modules/arp_poisoner/arp_poisoner.py | 3 ++- modules/blocking/blocking.py | 3 ++- modules/p2p_trust/utils/go_director.py | 3 ++- slips/daemon.py | 2 ++ slips_files/core/evidence_logger.py | 6 +++++- slips_files/core/output.py | 3 ++- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/modules/arp_poisoner/arp_poisoner.py b/modules/arp_poisoner/arp_poisoner.py index 1d85f72362..872fac5db8 100644 --- a/modules/arp_poisoner/arp_poisoner.py +++ b/modules/arp_poisoner/arp_poisoner.py @@ -38,7 +38,8 @@ def init(self): self.blocking_logfile_lock = Lock() # clear it try: - open(self.log_file_path, "w").close() + if not self.args.is_slips_started_by_an_update: + open(self.log_file_path, "w").close() except FileNotFoundError: pass self.unblocker = ARPUnblocker( diff --git a/modules/blocking/blocking.py b/modules/blocking/blocking.py index b449cb1168..db11419685 100644 --- a/modules/blocking/blocking.py +++ b/modules/blocking/blocking.py @@ -42,7 +42,8 @@ def init(self): self.blocking_logfile_lock = Lock() # clear it try: - open(self.blocking_log_path, "w").close() + if not self.args.is_slips_started_by_an_update: + open(self.blocking_log_path, "w").close() except FileNotFoundError: pass self.last_closed_tw = None diff --git a/modules/p2p_trust/utils/go_director.py b/modules/p2p_trust/utils/go_director.py index 1ee7b0755d..a619fdec76 100644 --- a/modules/p2p_trust/utils/go_director.py +++ b/modules/p2p_trust/utils/go_director.py @@ -67,7 +67,8 @@ def __init__( self.report_func = report_func self.request_func = request_func # clear the logfile - open(p2p_reports_logfile, "w").close() + if not self.args.is_slips_started_by_an_update: + open(p2p_reports_logfile, "w").close() self.reports_logfile = open(p2p_reports_logfile, "a") self.print(f"Storing peer reports in {p2p_reports_logfile}") # TODO: there should be some better mechanism to add new processing diff --git a/slips/daemon.py b/slips/daemon.py index e037d93b16..653e64bcb6 100644 --- a/slips/daemon.py +++ b/slips/daemon.py @@ -49,6 +49,8 @@ def print(self, text, **kwargs): def create_std_streams(self): """Create standard steam files and dirs and clear them""" + if self.args.is_slips_started_by_an_update: + return std_streams = [self.stderr, self.stdout, self.logsfile] for file in std_streams: diff --git a/slips_files/core/evidence_logger.py b/slips_files/core/evidence_logger.py index e1308aa781..585c6cba75 100644 --- a/slips_files/core/evidence_logger.py +++ b/slips_files/core/evidence_logger.py @@ -66,7 +66,11 @@ def clean_file(self, output_dir, file_to_clean): logfile_dir = os.path.dirname(logfile_path) if logfile_dir: os.makedirs(logfile_dir, exist_ok=True) - if os.path.exists(logfile_path): + + if ( + os.path.exists(logfile_path) + and not self.args.is_slips_started_by_an_update + ): open(logfile_path, "w").close() return open(logfile_path, "a") diff --git a/slips_files/core/output.py b/slips_files/core/output.py index 423da0c69f..e76e4b0221 100644 --- a/slips_files/core/output.py +++ b/slips_files/core/output.py @@ -118,7 +118,8 @@ def create_logfile(self, path): except FileNotFoundError: p = Path(os.path.dirname(path)) p.mkdir(parents=True, exist_ok=True) - open(path, "w").close() + if not self.args.is_slips_started_by_an_update: + open(path, "w").close() def log_line(self, msg: dict): """ From ffe67f44df7136afda4d73d54fd2c2c94a32357b Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 16:39:49 +0200 Subject: [PATCH 20/51] Add a centralized func (utils.initialize_logfile) for initializing logfiles, it doesnt clear logfiles when slips is started with -u --- managers/process_manager.py | 1 + modules/arp_poisoner/arp_poisoner.py | 9 ++-- modules/blocking/blocking.py | 9 ++-- modules/cesnet/cesnet.py | 7 ++- modules/cesnet/warden_client.py | 12 ++++- modules/p2p_trust/p2p_trust.py | 3 ++ modules/p2p_trust/utils/go_director.py | 6 ++- slips_files/common/slips_utils.py | 28 +++++++++++ slips_files/core/evidence_handler.py | 1 + slips_files/core/evidence_logger.py | 13 +++-- slips_files/core/output.py | 20 +++++--- tests/unit/slips/test_main.py | 2 +- .../common/abstracts/test_imodule.py | 49 +++++++++++++++++++ 13 files changed, 133 insertions(+), 27 deletions(-) create mode 100644 tests/unit/slips_files/common/abstracts/test_imodule.py diff --git a/managers/process_manager.py b/managers/process_manager.py index c4119561ae..93cb38700d 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -118,6 +118,7 @@ def start_output_process(self, stderr, slips_logfile, stdout=""): debug=self.main.args.debug, input_type=self.main.input_type, create_logfiles=False if self.main.args.stopdaemon else True, + slips_args=self.main.args, ) self.slips_logfile = output_process.slips_logfile return output_process diff --git a/modules/arp_poisoner/arp_poisoner.py b/modules/arp_poisoner/arp_poisoner.py index 872fac5db8..dc191b6536 100644 --- a/modules/arp_poisoner/arp_poisoner.py +++ b/modules/arp_poisoner/arp_poisoner.py @@ -37,11 +37,10 @@ def init(self): ) self.blocking_logfile_lock = Lock() # clear it - try: - if not self.args.is_slips_started_by_an_update: - open(self.log_file_path, "w").close() - except FileNotFoundError: - pass + utils.initialize_logfile( + self.log_file_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) self.unblocker = ARPUnblocker( self.db, self.should_stop, self.logger, self.log ) diff --git a/modules/blocking/blocking.py b/modules/blocking/blocking.py index db11419685..9093952e08 100644 --- a/modules/blocking/blocking.py +++ b/modules/blocking/blocking.py @@ -41,11 +41,10 @@ def init(self): ) self.blocking_logfile_lock = Lock() # clear it - try: - if not self.args.is_slips_started_by_an_update: - open(self.blocking_log_path, "w").close() - except FileNotFoundError: - pass + utils.initialize_logfile( + self.blocking_log_path, + getattr(self.args, "is_slips_started_by_an_update", False), + ) self.last_closed_tw = None self.ap_info: None | Dict[str, str] = self.db.get_ap_info() diff --git a/modules/cesnet/cesnet.py b/modules/cesnet/cesnet.py index a1313daa31..01fbb3561e 100644 --- a/modules/cesnet/cesnet.py +++ b/modules/cesnet/cesnet.py @@ -255,7 +255,12 @@ def pre_main(self): return 1 # create the warden client - self.wclient = Client(**read_cfg(self.configuration_file)) + self.wclient = Client( + **read_cfg(self.configuration_file), + is_slips_started_by_an_update=getattr( + self.args, "is_slips_started_by_an_update", False + ), + ) # All methods return something. # If you want to catch possible errors (for example implement some diff --git a/modules/cesnet/warden_client.py b/modules/cesnet/warden_client.py index 83bacbaa72..bcd021b617 100644 --- a/modules/cesnet/warden_client.py +++ b/modules/cesnet/warden_client.py @@ -17,6 +17,8 @@ from operator import itemgetter from pathlib import Path +from slips_files.common.slips_utils import utils + VERSION = "3.0-beta2" @@ -181,11 +183,13 @@ def __init__( idstore=None, name="org.example.warden.test", secret=None, + is_slips_started_by_an_update: bool = False, ): if errlog is None: errlog = {} self.name = name self.secret = secret + self.is_slips_started_by_an_update = is_slips_started_by_an_update # Init logging as soon as possible and make sure we don't # spit out exceptions but just log or return Error objects self.init_log(errlog, syslog, filelog) @@ -221,7 +225,11 @@ def create_file(self, filepath): # filename = path.basename(filepath) p = Path(dir) p.mkdir(parents=True, exist_ok=True) - open(filepath, "w").close() + utils.initialize_logfile( + filepath, + self.is_slips_started_by_an_update, + create_parent_dirs=False, + ) def init_log(self, errlog: dict, syslog: dict, filelog: dict): def loglevel(lev): @@ -650,6 +658,6 @@ def format_time( def read_cfg(cfgfile): with open(cfgfile, "r") as f: stripcomments = "\n".join( - (l for l in f if not l.lstrip().startswith(("#", "//"))) + (line for line in f if not line.lstrip().startswith(("#", "//"))) ) return json.loads(stripcomments) diff --git a/modules/p2p_trust/p2p_trust.py b/modules/p2p_trust/p2p_trust.py index 47396c4373..bdebcaff6f 100644 --- a/modules/p2p_trust/p2p_trust.py +++ b/modules/p2p_trust/p2p_trust.py @@ -193,6 +193,9 @@ def _configure(self): gopy_channel=self.gopy_channel, pygo_channel=self.pygo_channel, p2p_reports_logfile=self.p2p_reports_logfile, + is_slips_started_by_an_update=getattr( + self.args, "is_slips_started_by_an_update", False + ), ) self.pigeon = None diff --git a/modules/p2p_trust/utils/go_director.py b/modules/p2p_trust/utils/go_director.py index a619fdec76..43abc8115f 100644 --- a/modules/p2p_trust/utils/go_director.py +++ b/modules/p2p_trust/utils/go_director.py @@ -52,6 +52,7 @@ def __init__( gopy_channel: str = "p2p_gopy", pygo_channel: str = "p2p_pygo", p2p_reports_logfile: str = "p2p_reports.log", + is_slips_started_by_an_update: bool = False, ): self.printer = Printer(logger, self.name) @@ -67,8 +68,9 @@ def __init__( self.report_func = report_func self.request_func = request_func # clear the logfile - if not self.args.is_slips_started_by_an_update: - open(p2p_reports_logfile, "w").close() + utils.initialize_logfile( + p2p_reports_logfile, is_slips_started_by_an_update + ) self.reports_logfile = open(p2p_reports_logfile, "a") self.print(f"Storing peer reports in {p2p_reports_logfile}") # TODO: there should be some better mechanism to add new processing diff --git a/slips_files/common/slips_utils.py b/slips_files/common/slips_utils.py index 4a93563e2b..c629dbb819 100644 --- a/slips_files/common/slips_utils.py +++ b/slips_files/common/slips_utils.py @@ -835,6 +835,34 @@ def change_logfiles_ownership(self, file: str, UID, GID): os.system(f"chown {UID}:{GID} {file}") + def initialize_logfile( + self, + logfile_path: str, + started_by_update: bool, + mode: str = "w", + create_parent_dirs: bool = True, + ) -> bool: + """ + Initialize a log file unless Slips was started by an update. + when slips is being updated , we dont want the new version to clear + the used logfiles, instead it will append to them + + :param logfile_path: path to the log file to initialize. + :param started_by_update: whether Slips was started by an update. + :param mode: file mode used to initialize the log file. + :param create_parent_dirs: whether to create missing parent dirs. + :return: True if the file was initialized, False otherwise. + """ + if started_by_update: + return False + + logfile_dir = os.path.dirname(logfile_path) + if create_parent_dirs and logfile_dir: + os.makedirs(logfile_dir, exist_ok=True) + # clear the logfile + open(logfile_path, mode).close() + return True + def get_ip_identification_as_str(self, ip_identification: dict) -> str: id = "" if "DNS_resolution" in ip_identification: diff --git a/slips_files/core/evidence_handler.py b/slips_files/core/evidence_handler.py index 40e8d9cf0c..bd79dff769 100644 --- a/slips_files/core/evidence_handler.py +++ b/slips_files/core/evidence_handler.py @@ -68,6 +68,7 @@ def init(self): output_dir=get_alerts_path_inside_output_dir( self.parent_output_dir ), + slips_args=self.args, ) self.logger_thread = threading.Thread( target=self.evidence_logger.run_logger_thread, diff --git a/slips_files/core/evidence_logger.py b/slips_files/core/evidence_logger.py index 585c6cba75..a4f6e53566 100644 --- a/slips_files/core/evidence_logger.py +++ b/slips_files/core/evidence_logger.py @@ -17,10 +17,12 @@ def __init__( logger_stop_signal: threading.Event, evidence_logger_q: multiprocessing.Queue, output_dir: str, + slips_args=None, ): self.logger_stop_signal = logger_stop_signal self.evidence_logger_q = evidence_logger_q self.output_dir = output_dir + self.args = slips_args self.read_configuration() # clear output/alerts.log @@ -67,11 +69,12 @@ def clean_file(self, output_dir, file_to_clean): if logfile_dir: os.makedirs(logfile_dir, exist_ok=True) - if ( - os.path.exists(logfile_path) - and not self.args.is_slips_started_by_an_update - ): - open(logfile_path, "w").close() + if os.path.exists(logfile_path): + utils.initialize_logfile( + logfile_path, + getattr(self.args, "is_slips_started_by_an_update", False), + create_parent_dirs=False, + ) return open(logfile_path, "a") def print_to_alerts_logfile(self, data: str): diff --git a/slips_files/core/output.py b/slips_files/core/output.py index e76e4b0221..01a863b82b 100644 --- a/slips_files/core/output.py +++ b/slips_files/core/output.py @@ -48,6 +48,7 @@ def __init__( input_type=False, create_logfiles: bool = True, stdout="", + slips_args=None, ): super().__init__() # when running slips using -e , this var is set and we only @@ -58,6 +59,7 @@ def __init__( self.input_type = input_type self.errors_logfile = stderr self.slips_logfile = slips_logfile + self.args = slips_args if self.verbose > 2: print(f"Verbosity: {self.verbose}. Debugging: {self.debug}") @@ -71,10 +73,10 @@ def __init__( # root (if slips was started by root) os.umask(0) self._read_configuration() - self.create_logfile(self.errors_logfile) - self.log_branch_info(self.errors_logfile) - self.create_logfile(self.slips_logfile) - self.log_branch_info(self.slips_logfile) + if self.create_logfile(self.errors_logfile): + self.log_branch_info(self.errors_logfile) + if self.create_logfile(self.slips_logfile): + self.log_branch_info(self.slips_logfile) utils.change_logfiles_ownership( self.errors_logfile, self.UID, self.GID @@ -112,14 +114,20 @@ def log_branch_info(self, logfile: str): def create_logfile(self, path): """ creates slips.log and errors.log if they don't exist + :return: True if the file was initialized, False otherwise. """ + if getattr(self.args, "is_slips_started_by_an_update", False) is True: + return False + try: open(path, "a").close() + return True except FileNotFoundError: p = Path(os.path.dirname(path)) p.mkdir(parents=True, exist_ok=True) - if not self.args.is_slips_started_by_an_update: - open(path, "w").close() + return utils.initialize_logfile( + path, False, create_parent_dirs=False + ) def log_line(self, msg: dict): """ diff --git a/tests/unit/slips/test_main.py b/tests/unit/slips/test_main.py index 9efb1b8072..ab84d62894 100644 --- a/tests/unit/slips/test_main.py +++ b/tests/unit/slips/test_main.py @@ -488,7 +488,7 @@ def test_prepare_output_dir_without_o_flag( ): main = ModuleFactory().create_main_obj() main.args = MagicMock() - main.alerts_default_path = str(tmp_path) + main.parent_output_dir = str(tmp_path) main.input_information = "/fake/input/wlp3s0" monkeypatch.setattr(sys, "argv", ["script.py"]) # No -o diff --git a/tests/unit/slips_files/common/abstracts/test_imodule.py b/tests/unit/slips_files/common/abstracts/test_imodule.py new file mode 100644 index 0000000000..80d4ef9b8b --- /dev/null +++ b/tests/unit/slips_files/common/abstracts/test_imodule.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import json + +from tests.module_factory import ModuleFactory +from slips_files.common.slips_utils import utils + + +def test_imodule_exposes_slips_version(): + ip_info = ModuleFactory().create_ip_info_obj() + + assert ip_info.slips_version == utils.get_slips_version() + + +def test_get_msg_discards_messages_with_different_version(): + ip_info = ModuleFactory().create_ip_info_obj() + ip_info.channels = {"new_ip": "channel_obj"} + ip_info.channel_tracker = ip_info.init_channel_tracker() + ip_info.db.get_message.return_value = { + "channel": "new_ip", + "data": json.dumps({"text": "1.2.3.4", "version": "0.0.0"}), + } + + msg = ip_info.get_msg("new_ip") + + assert msg is None + assert ip_info.channel_tracker["new_ip"]["msg_received"] is False + ip_info.db.incr_msgs_received_in_channel.assert_not_called() + + +def test_get_msg_accepts_messages_with_current_version(): + ip_info = ModuleFactory().create_ip_info_obj() + ip_info.channels = {"new_ip": "channel_obj"} + ip_info.channel_tracker = ip_info.init_channel_tracker() + message = { + "channel": "new_ip", + "data": json.dumps( + {"text": "1.2.3.4", "version": ip_info.slips_version} + ), + } + ip_info.db.get_message.return_value = message + + msg = ip_info.get_msg("new_ip") + + assert msg == message + assert ip_info.channel_tracker["new_ip"]["msg_received"] is True + ip_info.db.incr_msgs_received_in_channel.assert_called_once_with( + ip_info.name, "new_ip" + ) From 1b7647963f1ff76bbf68ed7b18afe5a8996a5c27 Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 22:42:34 +0200 Subject: [PATCH 21/51] if slips is started with -u, read zeek log files from the offsets stored in the db, not from the beginning. --- managers/update_manager.py | 5 +- .../core/input/zeek/utils/zeek_input_utils.py | 52 +++++++++++++++++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index ff867a2239..1f466c8069 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -150,7 +150,10 @@ def update_slips(self): # else: # # prep for handover. old version to the new one. # ... - # self.is_slips_live_updating.set() + + # this event signals input.py to save the current zeek offsets in + # the db + self.is_slips_live_updating.set() ... def _did_1d_pass_since_last_update(self) -> bool: diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 51f98fab9c..ff033af81a 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -97,6 +97,7 @@ def get_file_handle(self, filename: str): try: # First time opening this file. file_handle = open(filename, "r") + self.restore_file_offset(filename, file_handle) self.open_file_handles[filename] = file_handle # now that we replaced the old handle with the newly created file handle # delete the old .log file, that has a timestamp in its name. @@ -113,6 +114,48 @@ def get_file_handle(self, filename: str): return False return file_handle + def restore_zeek_files_offsets_from_db(self): + """ + Restore previously stored Zeek logfile offsets from the database. + + :return: Mapping of Zeek logfile paths to integer offsets. + """ + restored_offsets = {} + stored_offsets = self.input.db.get_open_zeek_files_offsets() + + for filename, offset in stored_offsets.items(): + try: + restored_offsets[filename] = int(offset) + except (TypeError, ValueError): + self.input.print( + f"Ignoring invalid stored Zeek offset for {filename}: " + f"{offset}", + log_to_logfiles_only=True, + ) + + self.last_consumed_offsets = restored_offsets + self.zeek_logs_offsets = {} + return restored_offsets + + def restore_file_offset(self, filename: str, file_handle): + """ + Move a newly opened Zeek logfile handle to its restored offset. + This is useful when slips auto updates, to start reading from where + slips left off before updating + + :param filename: Zeek logfile path. + :param file_handle: Open file handle for the Zeek logfile. + :return: None + """ + offset = self.last_consumed_offsets.get(filename) + if offset is None: + return + + if offset > os.path.getsize(filename): + return + + file_handle.seek(offset) + def get_ts_from_line(self, zeek_line: str): """ used only by zeek log files @@ -278,10 +321,7 @@ def _update_offsets(self, file): newest_offset = self._get_newest_known_offset(file) self.last_consumed_offsets[file] = newest_offset - if ( - self.input.is_slips_live_updating is not None - and self.input.is_slips_live_updating.is_set() - ): + if self.args.is_slips_started_by_an_update: self.store_current_open_zeek_files_offsets_in_db() def read_zeek_files(self) -> int: @@ -298,6 +338,8 @@ def read_zeek_files(self) -> int: self.cache_lines = {} self.zeek_logs_offsets = {} self.last_consumed_offsets = {} + if self.args.is_slips_live_updating: + self.restore_zeek_files_offsets_from_db() # Try to keep track of when was the last update so we stop this # reading self.last_updated_file_time = datetime.datetime.now() @@ -375,7 +417,7 @@ def init_zeek( observer.start(zeek_dir, pcap_or_interface) zeek_files = os.listdir(zeek_dir) - if len(zeek_files) > 0: + if len(zeek_files) > 0 and not self.args.is_slips_live_updating: # First clear the zeek folder of old .log files for file_name in zeek_files: os.remove(os.path.join(zeek_dir, file_name)) From 92656c57f1abaf8b271c25c8b1408ebaa2c9eccc Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 23:04:57 +0200 Subject: [PATCH 22/51] dont flush the redis server as soon as slips connects to redis if slips is started with -u, instead append data to the server, as if slips never stopped --- managers/process_manager.py | 4 ++-- slips/main.py | 22 ++++++++++++++----- slips_files/core/database/database_manager.py | 16 ++++++++++++++ .../core/database/redis_db/database.py | 5 +++++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 93cb38700d..79252064e7 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -475,9 +475,9 @@ def start_timewindow_updater(self): Starts a thread that keeps track of the current timewindow if running on an interface - why is this not started in the redis db? because each module + why is this not started by the redis db? because each module has a db insteance, and we don't want a thread per module, - so starrting this thread once in main is enough + so starting this thread once in main is enough """ if not self.main.args.interface: return diff --git a/slips/main.py b/slips/main.py index 91f67f0fc6..87076eccb3 100644 --- a/slips/main.py +++ b/slips/main.py @@ -534,6 +534,7 @@ def start(self): self.print_version() print("https://stratosphereips.org") print("-" * 27) + self.setup_print_levels() stderr: str = self.get_slips_error_file() slips_logfile: str = self.get_slips_logfile() @@ -543,10 +544,22 @@ def start(self): stderr, slips_logfile ) self.printer = Printer(self.logger, self.name) + self.print(f"Storing Slips logs in {self.args.output}") + self.redis_port: int = self.redis_man.get_redis_port() - # dont start the redis server if it's already started - start_redis_server = not utils.is_port_in_use(self.redis_port) + if self.args.is_slips_live_updating: + # -u means slips is started by slips after it auto-updated, + # the redis server should already be up, this slips should + # just connect to it + start_redis_server = False + self.print( + f"Slips is done auto updating. Currently running " + f"version: {green(utils.get_current_version())}" + ) + else: + # dont start the redis server if it's already started + start_redis_server = not utils.is_port_in_use(self.redis_port) try: self.db = DBManager( @@ -556,6 +569,7 @@ def start(self): self.conf, int(self.pid), start_redis_server=start_redis_server, + flush_db=not self.args.is_slips_started_by_an_update, ) except RuntimeError as e: @@ -605,9 +619,7 @@ def start(self): 0, ) self.print( - f'Started {green("Main")} process ' - f"[PID" - f" {green(self.pid)}]", + f'Started {green("Main")} process [PID {green(self.pid)}]', 1, 0, ) diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index fbe1a74c02..8343041e39 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -49,11 +49,27 @@ def __init__( start_redis_server=True, **kwargs, ): + """ + Initialize Redis and SQLite database handlers. + + :param logger: output logger used by database handlers. + :param output_dir: directory where database files and logs are stored. + :param redis_port: port used to connect to Redis. + :param conf: loaded Slips configuration parser. + :param main_pid: PID of the main Slips process. + :param start_sqlite: whether to initialize the SQLite handler. + :param start_redis_server: whether to start Redis before connecting. + + :param kwargs: additional RedisDB options. Supported keys are: + flush_db: whether this manager may flush Redis on startup when + the configuration also allows deleting previous DB contents. + """ self.conf = conf self.output_dir = output_dir self.redis_port = redis_port self.logger = logger self.printer = Printer(self.logger, self.name) + # only the main process should ever flush the Redis DB. to avoid # children overwriting values set at the very start of slips if os.getpid() != main_pid: diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 6e7ac525bf..c14c1d78e9 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -195,6 +195,11 @@ def __init__(self, *args, **kwargs): self.call_mixins_setup() self._init_ttls() self.set_new_incoming_flows(True) + if not self.flush_db: + self.print( + "Continuing on previous data stored in redis.", + log_to_logfiles_only=True, + ) @classmethod def _get_conf_file_path(cls, redis_port: Optional[int] = None) -> str: From a068bbb5b1580c683abc026650807264e0eddbdf Mon Sep 17 00:00:00 2001 From: alya Date: Mon, 20 Apr 2026 23:08:05 +0200 Subject: [PATCH 23/51] delete unused timewindow updater thread --- managers/process_manager.py | 24 ------------------- slips/main.py | 1 - .../core/database/redis_db/constants.py | 1 - .../core/database/redis_db/profile_handler.py | 11 --------- .../timewindow_updater_thread/tw_updater.py | 18 -------------- 5 files changed, 55 deletions(-) delete mode 100644 slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py diff --git a/managers/process_manager.py b/managers/process_manager.py index 79252064e7..9c02543539 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -9,7 +9,6 @@ import sys import time import traceback -import threading from collections import OrderedDict from datetime import datetime from multiprocessing import ( @@ -44,9 +43,6 @@ from slips_files.common.style import green from slips_files.common.input_type import InputType -from slips_files.core.database.redis_db.timewindow_updater_thread.tw_updater import ( - timewindow_updater, -) from slips_files.core.evidence_handler import EvidenceHandler from slips_files.core.helpers.bloom_filters_manager import BFManager from slips_files.core.input import Input @@ -470,26 +466,6 @@ def init_bloom_filters_manager(self): self.main.pid, ) - def start_timewindow_updater(self): - """ - Starts a thread that keeps track of the current timewindow if - running on an interface - - why is this not started by the redis db? because each module - has a db insteance, and we don't want a thread per module, - so starting this thread once in main is enough - """ - if not self.main.args.interface: - return - tw_width: float = self.main.conf.get_tw_width_in_seconds() - t = threading.Thread( - target=timewindow_updater, - name="timewindow_updater", - args=(self.main.db, tw_width, self.termination_event), - daemon=True, - ) - utils.start_thread(t, self.main.db) - def start_update_manager(self, local_files=False, ti_feeds=False): """ starts the update manager process diff --git a/slips/main.py b/slips/main.py index 87076eccb3..0c5c87b3f4 100644 --- a/slips/main.py +++ b/slips/main.py @@ -707,7 +707,6 @@ def sig_handler(sig, frame): self.c1 = self.db.subscribe("control_channel") self.metadata_man.add_metadata_if_enabled() - self.proc_man.start_timewindow_updater() self.input_process = self.proc_man.start_input_process() self.db.store_pid("main", int(self.pid)) diff --git a/slips_files/core/database/redis_db/constants.py b/slips_files/core/database/redis_db/constants.py index 83fc9de27b..bffc9f65df 100644 --- a/slips_files/core/database/redis_db/constants.py +++ b/slips_files/core/database/redis_db/constants.py @@ -3,7 +3,6 @@ class Constants: LOADED_TI_FILES = "loaded_TI_files_number" TI_FILES_INFO = "TI_files_info" - CURRENT_TIMEWINDOW = "current_timewindow" # all keys starting with IoC_* are used for storing IoCs read from # online and offline TI feeds IOC_IPS = "IoC_ips" diff --git a/slips_files/core/database/redis_db/profile_handler.py b/slips_files/core/database/redis_db/profile_handler.py index 4ccdd95de3..f3267fc4b0 100644 --- a/slips_files/core/database/redis_db/profile_handler.py +++ b/slips_files/core/database/redis_db/profile_handler.py @@ -1147,17 +1147,6 @@ def check_tw_to_close(self, close_all=False): pipe = self.delete_past_timewindows(profile_tw_to_close, pipe) pipe.execute() - def get_current_timewindow(self) -> Optional[str]: - """returns the current timewindow if slips is running real-time ( - not pcap/log files)""" - if not self.args.interface: - return - - return self.r.get(self.constants.CURRENT_TIMEWINDOW) - - def set_current_timewindow(self, timewindow: str) -> Optional[str]: - self.r.set(self.constants.CURRENT_TIMEWINDOW, timewindow) - def mark_profile_tw_as_modified(self, modified_tw_details: dict): """ PS: this function should be as optimized as possible, it's the main diff --git a/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py b/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py deleted file mode 100644 index f5f0666e36..0000000000 --- a/slips_files/core/database/redis_db/timewindow_updater_thread/tw_updater.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Garcia -# SPDX-License-Identifier: GPL-2.0-only - -import time -from threading import Event - - -def timewindow_updater(db, tw_width: float, stop_event: Event): - """ - runs in a thread, wakes up only to update the current timewindow - number in the db and sleeps again for the whole tw width - """ - while not stop_event.is_set(): - now = time.time() - cur_tw = db.get_timewindow(now, "", add_to_db=False) - db.set_current_timewindow(cur_tw) - # to avoid busy waiting - stop_event.wait(tw_width) From 4c17fb787fa210ee9491a0ac8d7ac09c793588f6 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 00:40:13 +0200 Subject: [PATCH 24/51] fix the db unable to log when starting the cache redis db --- slips/main.py | 2 +- slips_files/core/database/redis_db/database.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/slips/main.py b/slips/main.py index 0c5c87b3f4..08b961fd82 100644 --- a/slips/main.py +++ b/slips/main.py @@ -548,7 +548,7 @@ def start(self): self.print(f"Storing Slips logs in {self.args.output}") self.redis_port: int = self.redis_man.get_redis_port() - if self.args.is_slips_live_updating: + if self.args.is_slips_started_by_an_update: # -u means slips is started by slips after it auto-updated, # the redis server should already be up, this slips should # just connect to it diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index c14c1d78e9..91d73db979 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -155,6 +155,7 @@ def __new__( cls.flush_db = flush_db # start the redis server using cli if it's not started? cls.start_server = start_redis_server + cls.logger = logger cls.printer = Printer(logger, cls.name) cls.conf = ConfigParser() cls.args = cls.conf.get_args() @@ -195,7 +196,7 @@ def __init__(self, *args, **kwargs): self.call_mixins_setup() self._init_ttls() self.set_new_incoming_flows(True) - if not self.flush_db: + if self.logger and not self.flush_db: self.print( "Continuing on previous data stored in redis.", log_to_logfiles_only=True, From 6b29b7ccb06cfabc9b2512d930e23830ed2c8c92 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 00:43:02 +0200 Subject: [PATCH 25/51] refactoring for clarity --- managers/process_manager.py | 6 ++--- managers/update_manager.py | 6 ++--- slips_files/core/input/input.py | 4 +-- .../core/input/zeek/utils/zeek_input_utils.py | 4 +-- tests/unit/managers/test_process_manager.py | 27 +++++++++++++++++++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 9c02543539..300e1b3c40 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -87,7 +87,7 @@ def __init__(self, main): # is set by the input process to indicate no more flows are coming # so profiler can safely begin shutdown/joins. self.is_input_done_event = Event() - self.is_slips_live_updating = Event() + self.is_slips_live_updating_event = Event() self.read_config() def read_config(self): @@ -102,7 +102,7 @@ def read_config(self): def start_slips_update_manager(self): return UpdateManager( database=self.main.db, - is_slips_live_updating=self.is_slips_live_updating, + is_slips_live_updating_event=self.is_slips_live_updating_event, ) def start_output_process(self, stderr, slips_logfile, stdout=""): @@ -185,7 +185,7 @@ def start_input_process(self): line_type=self.main.line_type, is_profiler_done_event=self.is_profiler_done_event, is_input_done_event=self.is_input_done_event, - is_slips_live_updating=self.is_slips_live_updating, + is_slips_live_updating_event=self.is_slips_live_updating_event, ) input_process.start() self.main.print( diff --git a/managers/update_manager.py b/managers/update_manager.py index 1f466c8069..46ea2ec02b 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -20,10 +20,10 @@ class UpdateManager: def __init__( - self, database: DBManager = None, is_slips_live_updating=None + self, database: DBManager = None, is_slips_live_updating_event=None ): self.db = database - self.is_slips_live_updating = is_slips_live_updating + self.is_slips_live_updating_event = is_slips_live_updating_event self.is_running_non_stop: bool = self.db.is_running_non_stop() self.cached_update_info: Optional[Dict[str, Any]] = None self.conf = ConfigParser() @@ -153,7 +153,7 @@ def update_slips(self): # this event signals input.py to save the current zeek offsets in # the db - self.is_slips_live_updating.set() + self.is_slips_live_updating_event.set() ... def _did_1d_pass_since_last_update(self) -> bool: diff --git a/slips_files/core/input/input.py b/slips_files/core/input/input.py index ca97955192..a5204fce50 100644 --- a/slips_files/core/input/input.py +++ b/slips_files/core/input/input.py @@ -59,7 +59,7 @@ def init( line_type=None, is_profiler_done_event: multiprocessing.Event = None, is_input_done_event: multiprocessing.Event = None, - is_slips_live_updating: multiprocessing.Event = None, + is_slips_live_updating_event: multiprocessing.Event = None, ): self.input_type = input_type self.profiler_queue = profiler_queue @@ -92,7 +92,7 @@ def init( self.is_profiler_done_event = is_profiler_done_event # is set by this proc to indicate no more flows are coming self.is_input_done_event = is_input_done_event - self.is_slips_live_updating = is_slips_live_updating + self.is_slips_live_updating_event = is_slips_live_updating_event self.is_running_non_stop: bool = self.db.is_running_non_stop() self.input_handlers = self._build_input_handlers() self.active_handler = None diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index ff033af81a..f2c34cadc5 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -338,7 +338,7 @@ def read_zeek_files(self) -> int: self.cache_lines = {} self.zeek_logs_offsets = {} self.last_consumed_offsets = {} - if self.args.is_slips_live_updating: + if self.args.is_slips_started_by_an_update: self.restore_zeek_files_offsets_from_db() # Try to keep track of when was the last update so we stop this # reading @@ -417,7 +417,7 @@ def init_zeek( observer.start(zeek_dir, pcap_or_interface) zeek_files = os.listdir(zeek_dir) - if len(zeek_files) > 0 and not self.args.is_slips_live_updating: + if len(zeek_files) > 0 and not self.args.is_slips_started_by_an_update: # First clear the zeek folder of old .log files for file_name in zeek_files: os.remove(os.path.join(zeek_dir, file_name)) diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index bc9125b661..4eb5e232ff 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -71,6 +71,9 @@ def test_start_input_process( line_type=line_type, is_profiler_done_event=process_manager.is_profiler_done_event, is_input_done_event=process_manager.is_input_done_event, + is_slips_live_updating_event=( + process_manager.is_slips_live_updating_event + ), ) mock_input_process.start.assert_called_once() process_manager.main.print.assert_called_once() @@ -79,6 +82,30 @@ def test_start_input_process( ) +@pytest.mark.parametrize( + "is_slips_started_by_an_update,expected_is_set", + [(True, True), (False, False)], +) +def test_init_sets_live_update_event_for_update_start( + is_slips_started_by_an_update, + expected_is_set, +): + """Test that -u startup marks Slips as live-updating.""" + main_mock = Mock() + main_mock.args.is_slips_started_by_an_update = ( + is_slips_started_by_an_update + ) + main_mock.conf.get_disabled_modules.return_value = [] + main_mock.conf.is_bootstrapping_node.return_value = False + main_mock.conf.get_bootstrapping_modules.return_value = [] + + process_manager = ProcessManager(main_mock) + + assert process_manager.is_slips_live_updating_event.is_set() is ( + expected_is_set + ) + + @pytest.mark.parametrize( "module_name, modules_to_ignore, expected", [ From d78e7087aef8c429f3cc8bba3095f4327772284d Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 00:43:19 +0200 Subject: [PATCH 26/51] update unit tests --- tests/unit/managers/test_process_manager.py | 11 +++++ tests/unit/modules/blocking/test_blocking.py | 12 ++++- .../rnn_cc_detection/test_rnn_cc_detection.py | 2 +- .../slips_files/common/test_slips_utils.py | 45 +++++++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index 4eb5e232ff..dcf1d7a5c0 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only +import json import pytest from unittest.mock import Mock, patch from managers.process_manager import ProcessManager @@ -297,6 +298,16 @@ def test_get_analysis_time( (None, True, False), # Test case 2: Message doesn't contain "stop_slips" ({"data": "some_other_message"}, True, False), + # Test case 3: Wrapped plain-text messages should be decoded first + ( + { + "data": json.dumps( + {"text": "stop_slips", "version": "test-version"} + ) + }, + True, + True, + ), # Test case 3: Message contains # "stop_slips" but not intended for control channel ({"data": "stop_slips"}, False, False), diff --git a/tests/unit/modules/blocking/test_blocking.py b/tests/unit/modules/blocking/test_blocking.py index 4c1c80f4a0..19aab9e65a 100644 --- a/tests/unit/modules/blocking/test_blocking.py +++ b/tests/unit/modules/blocking/test_blocking.py @@ -206,8 +206,16 @@ def test_main_blocking_logic(block, expected_block_called): @pytest.mark.parametrize( "last_closed_tw, msg_data, should_call", [ - ("tw1", "profileid_123_tw2", True), # new tw, should call update - ("tw2", "profileid_234_tw2", False), # same tw, no update + ( + "tw1", + json.dumps({"text": "profileid_123_tw2", "version": "1.0"}), + True, + ), + ( + "tw2", + json.dumps({"text": "profileid_234_tw2", "version": "1.0"}), + False, + ), ], ) def test_main_tw_closed_triggers_update(last_closed_tw, msg_data, should_call): diff --git a/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py b/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py index 1a0f93d58f..fbb32f5440 100644 --- a/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py +++ b/tests/unit/modules/rnn_cc_detection/test_rnn_cc_detection.py @@ -40,7 +40,7 @@ def test_get_confidence(pre_behavioral_model, expected_confidence): ) def test_handle_tw_closed(msg_data, expected_profileid, expected_twid): cc_detection = ModuleFactory().create_rnn_detection_object() - msg = {"data": msg_data} + msg = {"data": json.dumps({"text": msg_data, "version": "test-version"})} with patch.object(cc_detection.letters_exporter, "export") as mock_export: cc_detection.handle_tw_closed(msg) diff --git a/tests/unit/slips_files/common/test_slips_utils.py b/tests/unit/slips_files/common/test_slips_utils.py index faca6d6693..9c1be8097f 100644 --- a/tests/unit/slips_files/common/test_slips_utils.py +++ b/tests/unit/slips_files/common/test_slips_utils.py @@ -27,6 +27,28 @@ def test_get_sha256_hash_from_nonexistent_file(): utils.get_sha256_hash_of_file_contents("nonexistent_file.txt") +def test_initialize_logfile_creates_file(tmp_path): + utils = ModuleFactory().create_utils_obj() + logfile = tmp_path / "logs" / "module.log" + + initialized = utils.initialize_logfile(str(logfile), False) + + assert initialized is True + assert logfile.exists() + assert logfile.read_text() == "" + + +def test_initialize_logfile_skips_file_when_started_by_update(tmp_path): + utils = ModuleFactory().create_utils_obj() + logfile = tmp_path / "module.log" + logfile.write_text("existing\n") + + initialized = utils.initialize_logfile(str(logfile), True) + + assert initialized is False + assert logfile.read_text() == "existing\n" + + @pytest.mark.parametrize( "filepath, expected_result", [ # Testcase 1: Supported file @@ -65,6 +87,29 @@ def test_get_sha256_hash_permission_error(): utils.get_sha256_hash_of_file_contents("restricted_file.txt") +@pytest.mark.parametrize( + "message, expected_payload", + [ + ({"data": "plain-text"}, "plain-text"), + ( + {"data": json.dumps({"text": "stop_slips", "version": "1.0"})}, + "stop_slips", + ), + ( + { + "data": json.dumps( + {"flow": {"uid": "abc"}, "profileid": "profile_1"} + ) + }, + {"flow": {"uid": "abc"}, "profileid": "profile_1"}, + ), + ], +) +def test_get_msg_payload(message, expected_payload): + utils = ModuleFactory().create_utils_obj() + assert utils.get_msg_payload(message) == expected_payload + + @pytest.mark.parametrize( "input_string, expected_output", [ # Testcase1: special chars From a5a1a49c2aa0d422c7eb112c2214e3657e487618 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 23:38:59 +0200 Subject: [PATCH 27/51] removing the creation of the used zeek dir from main.py --- managers/metadata_manager.py | 5 ++-- managers/process_manager.py | 2 +- managers/redis_manager.py | 4 ++- slips/main.py | 25 ++++++++----------- slips_files/core/helpers/checker.py | 1 - .../core/input/zeek/interface_input.py | 1 + .../core/input/zeek/utils/dos_protector.py | 2 +- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/managers/metadata_manager.py b/managers/metadata_manager.py index def73ad133..7a6a631a8b 100644 --- a/managers/metadata_manager.py +++ b/managers/metadata_manager.py @@ -120,8 +120,9 @@ def set_input_metadata(self): if self.main.args.interface: info.update({"interface": self.main.args.interface}) - if hasattr(self.main, "zeek_dir"): - info.update({"zeek_dir": self.main.zeek_dir}) + zeek_dir = self.main.db.get_zeek_output_dir() + if isinstance(zeek_dir, str) and zeek_dir: + info.update({"zeek_dir": zeek_dir}) if hasattr(self.main, "zeek_bro") and self.main.zeek_bro: info.update({"zeek_version": self.get_zeek_version()}) diff --git a/managers/process_manager.py b/managers/process_manager.py index 300e1b3c40..a4d92cb77f 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -103,6 +103,7 @@ def start_slips_update_manager(self): return UpdateManager( database=self.main.db, is_slips_live_updating_event=self.is_slips_live_updating_event, + print_func=self.main.print, ) def start_output_process(self, stderr, slips_logfile, stdout=""): @@ -181,7 +182,6 @@ def start_input_process(self): input_information=self.main.input_information, cli_packet_filter=self.main.args.pcapfilter, zeek_or_bro=self.main.zeek_bro, - zeek_dir=self.main.zeek_dir, line_type=self.main.line_type, is_profiler_done_event=self.is_profiler_done_event, is_input_done_event=self.is_input_done_event, diff --git a/managers/redis_manager.py b/managers/redis_manager.py index 989493d1af..2216037625 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -66,9 +66,11 @@ def log_redis_server_pid(self, redis_port: int, redis_pid: int): "Save the DB\n" ) + zeek_dir = self.main.db.get_zeek_output_dir() + f.write( f"{now},{self.main.input_information},{redis_port}," - f"{redis_pid},{self.main.zeek_dir},{self.main.args.output}," + f"{redis_pid},{zeek_dir},{self.main.args.output}," f"{os.getpid()}," f"{bool(self.main.args.daemon)},{self.main.args.save}\n" ) diff --git a/slips/main.py b/slips/main.py index 08b961fd82..fd5cf5521d 100644 --- a/slips/main.py +++ b/slips/main.py @@ -84,8 +84,6 @@ def __init__(self, testing=False): self.check_zeek_or_bro() self.prepare_output_dir() self.redis_man.start_redis_cache_if_not_running() - # this is the zeek dir slips will be using - self.prepare_zeek_output_dir() self.twid_width = self.conf.get_tw_width() # should be initialised after self.input_type is set self.host_ip_man = HostIPManager(self) @@ -110,15 +108,6 @@ def check_zeek_or_bro(self): return self.zeek_bro - def prepare_zeek_output_dir(self): - from pathlib import Path - - without_ext = Path(self.input_information).stem - if self.conf.store_zeek_files_in_the_output_dir(): - self.zeek_dir = os.path.join(self.args.output, "zeek_files") - else: - self.zeek_dir = f"zeek_files_{without_ext}/" - def terminate_slips(self): """ Shutdown slips, is called when stopping slips before @@ -169,14 +158,22 @@ def store_zeek_dir_copy(self): store_a_copy_of_zeek_files = self.conf.store_a_copy_of_zeek_files() was_running_zeek = self.was_running_zeek() if store_a_copy_of_zeek_files and was_running_zeek: + zeek_dir = self.db.get_zeek_output_dir() + if not isinstance(zeek_dir, str) or not zeek_dir: + return # this is where the copy will be stored dest_zeek_dir = os.path.join(self.args.output, "zeek_files") - copy_tree(self.zeek_dir, dest_zeek_dir) + copy_tree(zeek_dir, dest_zeek_dir) print(f"[Main] Stored a copy of zeek files to {dest_zeek_dir}") def delete_zeek_files(self): - if self.conf.delete_zeek_files(): - shutil.rmtree(self.zeek_dir) + zeek_dir = self.db.get_zeek_output_dir() + if ( + self.conf.delete_zeek_files() + and isinstance(zeek_dir, str) + and zeek_dir + ): + shutil.rmtree(zeek_dir) def del_file_or_dir(self, file): """deletes a file or dir inside the output dir""" diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index 7a56db80a2..16ba8b438d 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -259,7 +259,6 @@ def clear_redis_cache(self): print("Deleting Cache DB in Redis.") self.main.redis_man.clear_redis_cache_database() self.main.input_information = "" - self.main.zeek_dir = "" self.main.redis_man.log_redis_server_pid( redis_cache_default_server_port, redis_cache_server_pid ) diff --git a/slips_files/core/input/zeek/interface_input.py b/slips_files/core/input/zeek/interface_input.py index 071fb8ffd1..9b8eedc389 100644 --- a/slips_files/core/input/zeek/interface_input.py +++ b/slips_files/core/input/zeek/interface_input.py @@ -57,6 +57,7 @@ def run(self): } } ) + for interface, interface_info in interfaces_to_monitor.items(): interface_dir = interface_info["dir"] if not os.path.exists(interface_dir): diff --git a/slips_files/core/input/zeek/utils/dos_protector.py b/slips_files/core/input/zeek/utils/dos_protector.py index f7f4efcae9..7bb802c238 100644 --- a/slips_files/core/input/zeek/utils/dos_protector.py +++ b/slips_files/core/input/zeek/utils/dos_protector.py @@ -94,7 +94,7 @@ def _update_flow_sampling_stop_time_if_needed(self) -> bool: return True return False - def get_number_of_flows_to_skip_and_time_to_stop_sampling(self) -> int: + def get_number_of_flows_to_skip(self) -> int: if not self._should_run(): return 0 From aa821a489c187937180e0b9e25a27947576347f8 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 23:41:06 +0200 Subject: [PATCH 28/51] add prints when slips is storing and restoring zeek offsets --- .../core/input/zeek/utils/zeek_input_utils.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index f2c34cadc5..6bce258e73 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -33,6 +33,9 @@ def __init__(self, input_process): self.zeek_threads = [] self.zeek_pids = [] self.dos_protector = DoSProtector(self.input) + self.args = self.input.args + self.print = self.input.print + self.update_msg_printed = False def check_if_time_to_del_rotated_files(self): """ @@ -120,13 +123,20 @@ def restore_zeek_files_offsets_from_db(self): :return: Mapping of Zeek logfile paths to integer offsets. """ + self.print( + "Restoring zeek file offsets from the db to " + "continue analysis " + "from where the old slips version left off." + ) restored_offsets = {} stored_offsets = self.input.db.get_open_zeek_files_offsets() for filename, offset in stored_offsets.items(): try: + print(f"@@@@@@@@@@@@@@@@ restored {filename}: {offset}") restored_offsets[filename] = int(offset) except (TypeError, ValueError): + print(f"@@@@@@@@@@@@@@@@ err??? {filename}") self.input.print( f"Ignoring invalid stored Zeek offset for {filename}: " f"{offset}", @@ -203,7 +213,7 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): # We don't have any waiting line for this file, so proceed try: flows_to_skip_reading_if_under_heavy_load: int = ( - self.dos_protector.get_number_of_flows_to_skip_and_time_to_stop_sampling() + self.dos_protector.get_number_of_flows_to_skip() ) # skips flows @@ -317,11 +327,22 @@ def _get_newest_known_offset(self, file): previously_saved_offset = self.last_consumed_offsets.get(file, 0) return self.zeek_logs_offsets.get(file, previously_saved_offset) + def _print_update_msg(self): + if not self.update_msg_printed: + self.print( + "Slips is live updating. Storing last read zeek log " + "files offsets in the db. " + ) + self.update_msg_printed = True + def _update_offsets(self, file): newest_offset = self._get_newest_known_offset(file) self.last_consumed_offsets[file] = newest_offset - - if self.args.is_slips_started_by_an_update: + # an old version of slips is shutting down, and a new one will + # start soon, save the offsets in the db so the new one can restore + # them and continue reading from where the old one left off + if self.input.is_slips_live_updating_event.is_set(): + self._print_update_msg() self.store_current_open_zeek_files_offsets_in_db() def read_zeek_files(self) -> int: From 752b0489051cff267c212c363d2ea20e75c7b630 Mon Sep 17 00:00:00 2001 From: alya Date: Tue, 21 Apr 2026 23:51:46 +0200 Subject: [PATCH 29/51] move the creation of zeek dir to zeek_input_utils --- .../core/input/zeek/interface_input.py | 2 +- slips_files/core/input/zeek/pcap_input.py | 2 +- .../core/input/zeek/utils/zeek_input_utils.py | 23 ++++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/slips_files/core/input/zeek/interface_input.py b/slips_files/core/input/zeek/interface_input.py index 9b8eedc389..7f542da98c 100644 --- a/slips_files/core/input/zeek/interface_input.py +++ b/slips_files/core/input/zeek/interface_input.py @@ -21,7 +21,7 @@ def run(self): """ runs when slips is given an interface with -i or 2 interfaces with -ap """ - self.input.zeek_utils.ensure_zeek_dir() + self.input.zeek_utils.create_zeek_output_dir() self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() diff --git a/slips_files/core/input/zeek/pcap_input.py b/slips_files/core/input/zeek/pcap_input.py index 9e1e1833ae..f23563c0e8 100644 --- a/slips_files/core/input/zeek/pcap_input.py +++ b/slips_files/core/input/zeek/pcap_input.py @@ -17,7 +17,7 @@ def run(self): """ runs when slips is given a pcap with -f """ - self.input.zeek_utils.ensure_zeek_dir() + self.input.zeek_utils.create_zeek_output_dir() self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 6bce258e73..522ff98fcd 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -8,6 +8,7 @@ import subprocess import threading import time +from pathlib import Path from typing import List, Tuple from re import split @@ -417,9 +418,25 @@ def read_zeek_files(self) -> int: return self.input.lines - def ensure_zeek_dir(self): - if not os.path.exists(self.input.zeek_dir): - os.makedirs(self.input.zeek_dir) + def create_zeek_output_dir(self) -> str: + """ + Return the Zeek output directory, create it if needed, + and store its path in the DB. + + :return: Directory where Zeek should write log files. + """ + if not self.input.zeek_dir: + without_ext = Path(self.input.given_path).stem + if self.input.conf.store_zeek_files_in_the_output_dir(): + zeek_dir = Path(self.input.args.output) / "zeek_files" + else: + zeek_dir = Path(f"zeek_files_{without_ext}") + + self.input.zeek_dir = str(zeek_dir) + + Path(self.input.zeek_dir).mkdir(parents=True, exist_ok=True) + self.input.db.set_input_metadata({"zeek_dir": self.input.zeek_dir}) + return self.input.zeek_dir def init_zeek( self, From 2a300c8bb01aeeb23a5ea7ae1b2edb648fde107c Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 00:21:03 +0200 Subject: [PATCH 30/51] each updated slips version now creates its own zeek dir inside output/zeek_files/slips_vx.y.z --- slips_files/core/input/input.py | 2 -- .../core/input/zeek/interface_input.py | 10 +++--- slips_files/core/input/zeek/pcap_input.py | 6 ++-- .../core/input/zeek/utils/zeek_input_utils.py | 33 +++++++++++++------ slips_files/core/input/zeek/zeek_dir_input.py | 4 +-- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/slips_files/core/input/input.py b/slips_files/core/input/input.py index a5204fce50..6d923bd64e 100644 --- a/slips_files/core/input/input.py +++ b/slips_files/core/input/input.py @@ -55,7 +55,6 @@ def init( input_information=None, cli_packet_filter=None, zeek_or_bro=None, - zeek_dir=None, line_type=None, is_profiler_done_event: multiprocessing.Event = None, is_input_done_event: multiprocessing.Event = None, @@ -68,7 +67,6 @@ def init( self.line_type: str = line_type # entire path self.given_path: str = input_information - self.zeek_dir: str = zeek_dir self.zeek_or_bro: str = zeek_or_bro self.read_lines_delay = 0 # when input is done processing, it reeleases this semaphore, that's h diff --git a/slips_files/core/input/zeek/interface_input.py b/slips_files/core/input/zeek/interface_input.py index 7f542da98c..dedb5cfa17 100644 --- a/slips_files/core/input/zeek/interface_input.py +++ b/slips_files/core/input/zeek/interface_input.py @@ -21,8 +21,8 @@ def run(self): """ runs when slips is given an interface with -i or 2 interfaces with -ap """ - self.input.zeek_utils.create_zeek_output_dir() - self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") + zeek_dir: str = self.input.zeek_utils.create_zeek_output_dir() + self.input.print(f"Storing zeek log files in {zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() @@ -35,7 +35,7 @@ def run(self): interfaces_to_monitor.update( { self.input.args.interface: { - "dir": self.input.zeek_dir, + "dir": zeek_dir, "type": "main_interface", } } @@ -46,9 +46,7 @@ def run(self): # interfaces, wifi and eth. for _type, interface in self.db.get_ap_info().items(): # _type can be 'wifi_interface' or "ethernet_interface" - dir_to_store_interface_logs = os.path.join( - self.input.zeek_dir, interface - ) + dir_to_store_interface_logs = os.path.join(zeek_dir, interface) interfaces_to_monitor.update( { interface: { diff --git a/slips_files/core/input/zeek/pcap_input.py b/slips_files/core/input/zeek/pcap_input.py index f23563c0e8..b66ae29f33 100644 --- a/slips_files/core/input/zeek/pcap_input.py +++ b/slips_files/core/input/zeek/pcap_input.py @@ -17,8 +17,8 @@ def run(self): """ runs when slips is given a pcap with -f """ - self.input.zeek_utils.create_zeek_output_dir() - self.input.print(f"Storing zeek log files in {self.input.zeek_dir}") + zeek_dir: str = self.input.zeek_utils.create_zeek_output_dir() + self.input.print(f"Storing zeek log files in {zeek_dir}") if self.input.is_running_non_stop: self.file_remover.start() @@ -26,7 +26,7 @@ def run(self): # if bro does not receive any new line while reading a pcap self.input.bro_timeout = 30 self.input.zeek_utils.init_zeek( - self.observer, self.input.zeek_dir, self.input.given_path + self.observer, zeek_dir, self.input.given_path ) self.input.lines = self.input.zeek_utils.read_zeek_files() diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index 522ff98fcd..c44bd70dd1 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -37,6 +37,7 @@ def __init__(self, input_process): self.args = self.input.args self.print = self.input.print self.update_msg_printed = False + self.is_running_non_stop = self.input.db.is_running_non_stop() def check_if_time_to_del_rotated_files(self): """ @@ -418,6 +419,13 @@ def read_zeek_files(self) -> int: return self.input.lines + def _is_auto_update_enabled(self) -> bool: + """ + returns true if slips is analyzing an interface (-i , -ap or -g) + and auto_update is enabled in slips.yaml + """ + return self.is_running_non_stop and self.input.conf.auto_update_slips() + def create_zeek_output_dir(self) -> str: """ Return the Zeek output directory, create it if needed, @@ -425,18 +433,23 @@ def create_zeek_output_dir(self) -> str: :return: Directory where Zeek should write log files. """ - if not self.input.zeek_dir: - without_ext = Path(self.input.given_path).stem - if self.input.conf.store_zeek_files_in_the_output_dir(): - zeek_dir = Path(self.input.args.output) / "zeek_files" - else: - zeek_dir = Path(f"zeek_files_{without_ext}") - self.input.zeek_dir = str(zeek_dir) + without_ext = Path(self.input.given_path).stem + if self.input.conf.store_zeek_files_in_the_output_dir(): + zeek_dir = Path(self.input.args.output) / "zeek_files" + else: + zeek_dir = Path(f"zeek_files_{without_ext}") + + if self._is_auto_update_enabled(): + # slips is gonna be auto updating for each new version, we need + # 1 zeek dir for each started version + zeek_dir = Path(zeek_dir) / f"slips_v{utils.get_current_version()}" + + zeek_dir = str(zeek_dir) - Path(self.input.zeek_dir).mkdir(parents=True, exist_ok=True) - self.input.db.set_input_metadata({"zeek_dir": self.input.zeek_dir}) - return self.input.zeek_dir + Path(zeek_dir).mkdir(parents=True, exist_ok=True) + self.input.db.set_input_metadata({"zeek_dir": zeek_dir}) + return zeek_dir def init_zeek( self, diff --git a/slips_files/core/input/zeek/zeek_dir_input.py b/slips_files/core/input/zeek/zeek_dir_input.py index 5efa851b05..b8acea1c21 100644 --- a/slips_files/core/input/zeek/zeek_dir_input.py +++ b/slips_files/core/input/zeek/zeek_dir_input.py @@ -38,9 +38,7 @@ def run(self): # know the interface, hence the "default" interface = "default" - self.input.zeek_dir = self.input.given_path - - self.observer.start(self.input.zeek_dir, interface) + self.observer.start(self.input.given_path, interface) if self.input.is_running_non_stop: self.file_remover.start() From 57d86152c7c6da9b2476529608b261885c32628f Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 00:28:37 +0200 Subject: [PATCH 31/51] remove the logic for creating and restoring offsets. each version of slips uses its own zeek instance and logs, there's no shared zeek files. --- managers/update_manager.py | 25 ++++- slips_files/core/database/database_manager.py | 6 -- .../core/database/redis_db/constants.py | 1 - .../core/database/redis_db/database.py | 22 ----- .../core/input/zeek/utils/zeek_input_utils.py | 97 +------------------ 5 files changed, 25 insertions(+), 126 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 46ea2ec02b..86f9c5f102 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -20,7 +20,10 @@ class UpdateManager: def __init__( - self, database: DBManager = None, is_slips_live_updating_event=None + self, + database: DBManager = None, + is_slips_live_updating_event=None, + print_func=None, ): self.db = database self.is_slips_live_updating_event = is_slips_live_updating_event @@ -36,6 +39,7 @@ def __init__( ) self._read_configuration() self.last_update_time = 0 + self.print = print_func def _read_configuration(self): self.auto_update_slips_enabled = self.conf.auto_update_slips() @@ -151,8 +155,8 @@ def update_slips(self): # # prep for handover. old version to the new one. # ... - # this event signals input.py to save the current zeek offsets in - # the db + # this event signals input.py to stop recving input and start + # draining self.is_slips_live_updating_event.set() ... @@ -172,7 +176,20 @@ def check_for_update_every_1_day(self) -> bool: should update itself """ if self._did_1d_pass_since_last_update(): - return self.should_update_slips() + should_update: bool = self.should_update_slips() + # @@@@@@@@@@@@@ ALYA DONOT COMMIT THIS + # should_update = True + if should_update: + self.print( + "A new version of Slips is available. " + "Updating slips now." + ) + else: + self.print( + "No new version of Slips is available. " + "Slips will check again after 1 day." + ) + return should_update return False def should_update_slips(self) -> bool: diff --git a/slips_files/core/database/database_manager.py b/slips_files/core/database/database_manager.py index 8343041e39..1c927a6768 100644 --- a/slips_files/core/database/database_manager.py +++ b/slips_files/core/database/database_manager.py @@ -647,12 +647,6 @@ def add_zeek_file(self, *args, **kwargs): def get_all_zeek_files(self, *args, **kwargs): return self.rdb.get_all_zeek_files(*args, **kwargs) - def store_open_zeek_files_offsets(self, *args, **kwargs): - return self.rdb.store_open_zeek_files_offsets(*args, **kwargs) - - def get_open_zeek_files_offsets(self, *args, **kwargs): - return self.rdb.get_open_zeek_files_offsets(*args, **kwargs) - def get_gateway_ip(self, *args, **kwargs): return self.rdb.get_gateway_ip(*args, **kwargs) diff --git a/slips_files/core/database/redis_db/constants.py b/slips_files/core/database/redis_db/constants.py index bffc9f65df..7855733242 100644 --- a/slips_files/core/database/redis_db/constants.py +++ b/slips_files/core/database/redis_db/constants.py @@ -42,7 +42,6 @@ class Constants: LABELS = "labels" MSGS_PUBLISHED_AT_RUNTIME = "msgs_published_at_runtime" ZEEK_FILES = "zeekfiles" - ZEEK_OPEN_FILES_OFFSETS = "zeek_open_files_offsets" DEFAULT_GATEWAY = "default_gateway" IS_CYST_ENABLED = "is_cyst_enabled" LOCAL_NETWORK = "local_network" diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 91d73db979..0dbe56f34c 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -1305,28 +1305,6 @@ def get_all_zeek_files(self) -> set: """Return all entries from the list of zeek files""" return self.r.hgetall(self.constants.ZEEK_FILES) - def store_open_zeek_files_offsets(self, zeek_files_offsets: dict): - """ - Store the current offsets for open Zeek files. - - :param zeek_files_offsets: Mapping of Zeek logfile path to offset. - :return: None - """ - self.r.delete(self.constants.ZEEK_OPEN_FILES_OFFSETS) - if zeek_files_offsets: - self.r.hset( - self.constants.ZEEK_OPEN_FILES_OFFSETS, - mapping=zeek_files_offsets, - ) - - def get_open_zeek_files_offsets(self) -> dict: - """ - Retrieve the current offsets for open Zeek files. - - :return: Mapping of Zeek logfile path to offset. - """ - return self.r.hgetall(self.constants.ZEEK_OPEN_FILES_OFFSETS) - def _get_gw_info(self, interface: str) -> Dict[str, str] | None: """ gets the gw of the given interface, when slips is runnuning on a diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index c44bd70dd1..e6373c9dd5 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -23,10 +23,6 @@ def __init__(self, input_process): self.open_file_handles = {} self.open_file_handlers_lock = threading.RLock() self.cache_lines = {} - # used to start reading from where slips left off after slips auto - # updates. - self.zeek_logs_offsets = {} - self.last_consumed_offsets = {} self.file_time = {} self.last_updated_file_time = None self.rotated_files_to_delete: List[Tuple[str, float]] = [] @@ -102,7 +98,6 @@ def get_file_handle(self, filename: str): try: # First time opening this file. file_handle = open(filename, "r") - self.restore_file_offset(filename, file_handle) self.open_file_handles[filename] = file_handle # now that we replaced the old handle with the newly created file handle # delete the old .log file, that has a timestamp in its name. @@ -119,55 +114,6 @@ def get_file_handle(self, filename: str): return False return file_handle - def restore_zeek_files_offsets_from_db(self): - """ - Restore previously stored Zeek logfile offsets from the database. - - :return: Mapping of Zeek logfile paths to integer offsets. - """ - self.print( - "Restoring zeek file offsets from the db to " - "continue analysis " - "from where the old slips version left off." - ) - restored_offsets = {} - stored_offsets = self.input.db.get_open_zeek_files_offsets() - - for filename, offset in stored_offsets.items(): - try: - print(f"@@@@@@@@@@@@@@@@ restored {filename}: {offset}") - restored_offsets[filename] = int(offset) - except (TypeError, ValueError): - print(f"@@@@@@@@@@@@@@@@ err??? {filename}") - self.input.print( - f"Ignoring invalid stored Zeek offset for {filename}: " - f"{offset}", - log_to_logfiles_only=True, - ) - - self.last_consumed_offsets = restored_offsets - self.zeek_logs_offsets = {} - return restored_offsets - - def restore_file_offset(self, filename: str, file_handle): - """ - Move a newly opened Zeek logfile handle to its restored offset. - This is useful when slips auto updates, to start reading from where - slips left off before updating - - :param filename: Zeek logfile path. - :param file_handle: Open file handle for the Zeek logfile. - :return: None - """ - offset = self.last_consumed_offsets.get(filename) - if offset is None: - return - - if offset > os.path.getsize(filename): - return - - file_handle.seek(offset) - def get_ts_from_line(self, zeek_line: str): """ used only by zeek log files @@ -223,7 +169,6 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): file_handle.readline() zeek_line = file_handle.readline() - line_end_offset = file_handle.tell() except ValueError: # remover thread just finished closing all old handles. @@ -255,22 +200,8 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): "data": nline, "interface": interface, } - self.zeek_logs_offsets[filename] = line_end_offset return True - def store_current_open_zeek_files_offsets_in_db(self): - """ - Store the last committed offset for each currently open Zeek file. - - :return: None - """ - with self.open_file_handlers_lock: - current_offsets = { - filename: str(self.last_consumed_offsets.get(filename, 0)) - for filename in self.open_file_handles - } - self.input.db.store_open_zeek_files_offsets(current_offsets) - def reached_timeout(self) -> bool: # If we don't have any cached lines to send, # it may mean that new lines are not arriving. Check @@ -325,28 +256,15 @@ def get_earliest_line(self): earliest_line = self.cache_lines[file_with_earliest_flow] return earliest_line, file_with_earliest_flow - def _get_newest_known_offset(self, file): - previously_saved_offset = self.last_consumed_offsets.get(file, 0) - return self.zeek_logs_offsets.get(file, previously_saved_offset) - def _print_update_msg(self): if not self.update_msg_printed: self.print( - "Slips is live updating. Storing last read zeek log " - "files offsets in the db. " + "Slips is live updating. Slips will stop receiving new " + "flows in this instance and start receiving new flows using " + "the updated version. " ) self.update_msg_printed = True - def _update_offsets(self, file): - newest_offset = self._get_newest_known_offset(file) - self.last_consumed_offsets[file] = newest_offset - # an old version of slips is shutting down, and a new one will - # start soon, save the offsets in the db so the new one can restore - # them and continue reading from where the old one left off - if self.input.is_slips_live_updating_event.is_set(): - self._print_update_msg() - self.store_current_open_zeek_files_offsets_in_db() - def read_zeek_files(self) -> int: """ Runs when slips is analyzing pcaps, interface, zeek dirs, and zeek @@ -359,10 +277,6 @@ def read_zeek_files(self) -> int: # that file self.file_time = {} self.cache_lines = {} - self.zeek_logs_offsets = {} - self.last_consumed_offsets = {} - if self.args.is_slips_started_by_an_update: - self.restore_zeek_files_offsets_from_db() # Try to keep track of when was the last update so we stop this # reading self.last_updated_file_time = datetime.datetime.now() @@ -395,18 +309,15 @@ def read_zeek_files(self) -> int: continue # self.print('\t> Sent Line: {}'.format(earliest_line), 0, 3) - + # print(f"@@@@@@@@@@@@@@@@ sent line {earliest_line}") self.input.give_profiler(earliest_line) self.input.lines += 1 - self._update_offsets(file_with_earliest_flow) - # when testing, no need to read the whole file! if self.input.lines == 10 and self.input.testing: break # Delete this line from the cache and the time list del self.cache_lines[file_with_earliest_flow] - self.zeek_logs_offsets.pop(file_with_earliest_flow, None) del self.file_time[file_with_earliest_flow] # Get the new list of files. Since new files may have been created by From 38ccd9a499f062e38955c2f34ec843f2529a40c9 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 22:07:08 +0200 Subject: [PATCH 32/51] Make sure any exports that happen at the end of the analysis don't happen when slips is being updated --- managers/process_manager.py | 68 +++++++++++++++++++++++-------------- slips/main.py | 3 ++ 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index a4d92cb77f..f8fb87e194 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -768,6 +768,21 @@ def get_print_function(self): else: return self.main.print + def _generate_plots(self): + if self.is_slips_live_updating_event: + # slips is updating and will start a new instance, plots + # should be done when slips is actually shutting down at the + # very end of the analysis. + return + + if self.main.conf.generate_performance_plots() is True: + self.plotter = Plotter(self.main.args.output, print) + self.plotter.plot_latency_csv() + self.plotter.plot_profiler_latency_csvs() + self.plotter.plot_throughput_csv() + self.plotter.write_throughput_metrics() + self.plotter.plot_flows_from_conn_log() + def shutdown_gracefully(self): """ Waits for all modules to confirm that they're done processing @@ -775,13 +790,8 @@ def shutdown_gracefully(self): """ try: print = self.get_print_function() - if self.main.conf.generate_performance_plots() is True: - self.plotter = Plotter(self.main.args.output, print) - self.plotter.plot_latency_csv() - self.plotter.plot_profiler_latency_csvs() - self.plotter.plot_throughput_csv() - self.plotter.write_throughput_metrics() - self.plotter.plot_flows_from_conn_log() + + self._generate_plots() if not self.main.args.stopdaemon: print("\n" + "-" * 27) @@ -790,17 +800,19 @@ def shutdown_gracefully(self): self.children: List[BaseProcess] = ( multiprocessing.active_children() ) - # by default, max 15 mins (taken from wait_for_modules_to_finish) - # from this time, all modules should be killed method_start_time = time.time() - # how long to wait for modules to finish in minutes + # how long to wait for modules to finish in minutes before + # killing them timeout: float = self.main.conf.wait_for_modules_to_finish() # convert to seconds timeout *= 60 - # close all tws - self.main.db.check_tw_to_close(close_all=True) + # dont close tws if we're updating, the next slips will continue + # from where this slips left off. + if not self.is_slips_live_updating_event.is_set(): + # close all tws + self.main.db.check_tw_to_close(close_all=True) graceful_shutdown = True if self.main.mode == "daemonized": @@ -856,20 +868,19 @@ def shutdown_gracefully(self): self.kill_all_children() - if self.main.args.save: - self.main.save_the_db() + if not self.is_slips_live_updating_event.is_set(): + if self.main.args.save: + self.main.save_the_db() + if self.main.conf.export_labeled_flows(): + format_ = self.main.conf.export_labeled_flows_to().lower() + self.main.db.export_labeled_flows(format_) - if self.main.conf.export_labeled_flows(): - format_ = self.main.conf.export_labeled_flows_to().lower() - self.main.db.export_labeled_flows(format_) - - # if store_a_copy_of_zeek_files is set to yes in slips.yaml - # copy the whole zeek_files dir to the output dir - self.main.store_zeek_dir_copy() - - # if delete_zeek_files is set to yes in slips.yaml, - # delete zeek_files/ dir - self.main.delete_zeek_files() + # if store_a_copy_of_zeek_files is set to yes in slips.yaml + # copy the whole zeek_files dir to the output dir + self.main.store_zeek_dir_copy() + # if delete_zeek_files is set to yes in slips.yaml, + # delete zeek_files/ dir + self.main.delete_zeek_files() analysis_time, end_date = self.get_analysis_time() self.main.metadata_man.set_analysis_end_date(end_date) @@ -888,6 +899,13 @@ def shutdown_gracefully(self): "[Process Manager] Slips shutdown gracefully\n", log_to_logfiles_only=True, ) + if self.is_slips_live_updating_event.is_set(): + print( + "[Process Manager] Slips is live updating, " + "Stopping this instance and starting the new " + "instance now.\n", + log_to_logfiles_only=True, + ) else: print( f"[Process Manager] Slips didn't " diff --git a/slips/main.py b/slips/main.py index fd5cf5521d..90d35b0b60 100644 --- a/slips/main.py +++ b/slips/main.py @@ -566,6 +566,9 @@ def start(self): self.conf, int(self.pid), start_redis_server=start_redis_server, + # if auto update is enabled, slips starts itself with + # -u, and continues using the same db as the old + # version without overwriting the keys there flush_db=not self.args.is_slips_started_by_an_update, ) From 7b6209a06673a27fe313191040d9372a71b1c9c0 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 23:04:05 +0200 Subject: [PATCH 33/51] Detect when slips is done updating, and start draining, and call shutdown_gracefully(). --- managers/process_manager.py | 5 + .../core/database/redis_db/database.py | 6 +- .../core/input/zeek/utils/zeek_input_utils.py | 96 +++++++++++++------ 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index f8fb87e194..63dd18f878 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -626,6 +626,11 @@ def should_stop_slips(self) -> bool: This function NEVER returns True if the input and profiler are still processing. """ + if self.is_slips_live_updating_event.is_set(): + # slips is auto updating this version of slips should stop and + # the updated one will start + return True + if self.should_run_non_stop(): return False diff --git a/slips_files/core/database/redis_db/database.py b/slips_files/core/database/redis_db/database.py index 0dbe56f34c..f661011d0e 100644 --- a/slips_files/core/database/redis_db/database.py +++ b/slips_files/core/database/redis_db/database.py @@ -196,7 +196,11 @@ def __init__(self, *args, **kwargs): self.call_mixins_setup() self._init_ttls() self.set_new_incoming_flows(True) - if self.logger and not self.flush_db: + if ( + self.logger + and not self.flush_db + and self.args.is_slips_started_by_an_update + ): self.print( "Continuing on previous data stored in redis.", log_to_logfiles_only=True, diff --git a/slips_files/core/input/zeek/utils/zeek_input_utils.py b/slips_files/core/input/zeek/utils/zeek_input_utils.py index e6373c9dd5..5b77c3f86f 100644 --- a/slips_files/core/input/zeek/utils/zeek_input_utils.py +++ b/slips_files/core/input/zeek/utils/zeek_input_utils.py @@ -144,8 +144,12 @@ def get_ts_from_line(self, zeek_line: str): def cache_nxt_line_in_file(self, filename: str, interface: str): """ - reads 1 line of the given file and stores in queue for sending to the profiler - :param: full path to the file. includes the .log extension + reads 1 line of the given file and stores in queue for sending to the + profiler + + :param filename: full path to the file. includes the .log extension + :param interface: interface that generated the Zeek file + :return: True if a line was cached, False otherwise """ file_handle = self.get_file_handle(filename) if not file_handle: @@ -168,7 +172,27 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): for _ in range(flows_to_skip_reading_if_under_heavy_load): file_handle.readline() - zeek_line = file_handle.readline() + while zeek_line := file_handle.readline(): + if zeek_line.startswith("#close"): + # We reached the end of one of the files that we were + # reading. + return False + + if zeek_line.startswith("#fields"): + # this line contains the zeek fields, we want to cache it + # and send it to the profiler normally + nline = zeek_line + # to send the line as early as possible + timestamp = -1 + break + + timestamp, nline = self.get_ts_from_line(zeek_line) + if timestamp: + break + else: + # We reached the end of one of the files that we were reading. + # Wait for more lines to come from another file. + return False except ValueError: # remover thread just finished closing all old handles. @@ -176,23 +200,6 @@ def cache_nxt_line_in_file(self, filename: str, interface: str): # to get the new dict of open handles. return False - # Did the file end? - if not zeek_line or zeek_line.startswith("#close"): - # We reached the end of one of the files that we were reading. - # Wait for more lines to come from another file - return False - - if zeek_line.startswith("#fields"): - # this line contains the zeek fields, we want to cache it and - # send it to the profiler normally - nline = zeek_line - # to send the line as early as possible - timestamp = -1 - else: - timestamp, nline = self.get_ts_from_line(zeek_line) - if not timestamp: - return False - self.file_time[filename] = timestamp # Store the line in the cache self.cache_lines[filename] = { @@ -280,8 +287,26 @@ def read_zeek_files(self) -> int: # Try to keep track of when was the last update so we stop this # reading self.last_updated_file_time = datetime.datetime.now() - while not self.input.should_stop(): + is_draining = False + while True: + is_live_updating = ( + self.input.is_slips_live_updating_event is not None + and self.input.is_slips_live_updating_event.is_set() + ) + if is_live_updating and not is_draining: + # Stop Zeek first so this instance has a finite set of + # generated logs to drain before the updated instance + # continues. + self._print_update_msg() + self.shutdown_zeek_runtime() + self.zeek_files = self.input.db.get_all_zeek_files() + is_draining = True + + if self.input.should_stop() and not is_draining: + break + self.check_if_time_to_del_rotated_files() + # implemented in icore.py self.input.store_flows_read_per_second() @@ -291,13 +316,27 @@ def read_zeek_files(self) -> int: # PS: self.zeek_files ties each zeek file to its interface ( # beacause slips supports reading multiple interfaces) + if is_draining: + self.zeek_files = self.input.db.get_all_zeek_files() + + cached_new_line = False for filename, interface in self.zeek_files.items(): if utils.is_ignored_zeek_log_file(filename): continue # reads 1 line from the given file and cache it # from in self.cache_lines - self.cache_nxt_line_in_file(filename, interface) + if self.cache_nxt_line_in_file(filename, interface): + self.last_updated_file_time = datetime.datetime.now() + cached_new_line = True + + if ( + is_draining + and not cached_new_line + and not self.cache_lines + ): + # done draining the flows left + break if self.reached_timeout(): break @@ -309,21 +348,21 @@ def read_zeek_files(self) -> int: continue # self.print('\t> Sent Line: {}'.format(earliest_line), 0, 3) - # print(f"@@@@@@@@@@@@@@@@ sent line {earliest_line}") self.input.give_profiler(earliest_line) self.input.lines += 1 - # when testing, no need to read the whole file! + # when testing, no need to read the whole file! #TODO this + # is bad practice, fix it if self.input.lines == 10 and self.input.testing: break + # Delete this line from the cache and the time list del self.cache_lines[file_with_earliest_flow] del self.file_time[file_with_earliest_flow] - # Get the new list of files. Since new files may have been created by - # Zeek while we were processing them. + # Get the new list of files. Since new files may have been + # created by Zeek while we were processing them. self.zeek_files = self.input.db.get_all_zeek_files() - self.close_all_handles() except KeyboardInterrupt: pass @@ -446,7 +485,8 @@ def run_zeek(self, zeek_logs_dir, pcap_or_interface, tcpdump_filter=None): print(f"Zeek: {out}") if error: self.input.print( - f"Zeek error. return code: {zeek.returncode} error:{error.strip()}" + f"Zeek error. return code: {zeek.returncode} " + f"error:{error.strip()}" ) def shutdown_zeek_runtime(self): From 5c4337da7fa8348ff27db71b15bae4a7897bff0b Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 23:10:29 +0200 Subject: [PATCH 34/51] git pull origin master before draining if slips detected that it should update itself. --- managers/update_manager.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 86f9c5f102..5b8fc3e24a 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -146,19 +146,24 @@ def _is_new_version_available(self) -> bool: return utils.get_current_version() == latest_version + def git_pull_master(self): + """ + Pull the latest origin/master changes and check them out. + + Returns: + The checked out origin/master commit. + """ + repo = Repo(".") + repo.remote("origin").fetch("master") + repo.git.checkout("origin/master") + return repo.head.commit + def update_slips(self): - # if self.is_first_run: - # # we're not live updating, there isnt going to be an older - # # version of slips draining in this case. - # ... - # else: - # # prep for handover. old version to the new one. - # ... - - # this event signals input.py to stop recving input and start - # draining + self.git_pull_master() + # this eventL + # - signals input.py to stop recving input and start draining flows + # - and signals the process_manager() to call shutdown_gracefully() self.is_slips_live_updating_event.set() - ... def _did_1d_pass_since_last_update(self) -> bool: """ @@ -177,8 +182,10 @@ def check_for_update_every_1_day(self) -> bool: """ if self._did_1d_pass_since_last_update(): should_update: bool = self.should_update_slips() + # @@@@@@@@@@@@@ ALYA DONOT COMMIT THIS - # should_update = True + should_update = True + if should_update: self.print( "A new version of Slips is available. " From c8d41707213c3543a3f299422fb8b571b03b2b44 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 23:20:47 +0200 Subject: [PATCH 35/51] start the new version of slips with -u before completely stopping the old one to ensure minimum lost zeek flows. --- managers/update_manager.py | 39 +++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 5b8fc3e24a..570a9f138c 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -8,10 +8,13 @@ import json import re +import subprocess +import sys import time -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from urllib import error, request +import psutil from git import InvalidGitRepositoryError, NoSuchPathError, Repo from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.slips_utils import utils @@ -158,8 +161,42 @@ def git_pull_master(self): repo.git.checkout("origin/master") return repo.head.commit + def _get_updated_slips_command(self) -> List[str]: + """ + Build the command used to start the updated Slips process. + + Returns: + The current Slips cmd plus (-u). + """ + try: + cmd = psutil.Process().cmdline() + except psutil.Error: + cmd = [] + + if not cmd: + cmd = [sys.executable, *sys.argv] + + return [*cmd, "-u"] + + def start_updated_slips_version(self) -> subprocess.Popen: + """ + Starts the updated Slips as an independent process. + + Returns: + The detached process handle for the updated Slips process. + """ + return subprocess.Popen( + self._get_updated_slips_command(), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + start_new_session=True, + ) + def update_slips(self): self.git_pull_master() + self.start_updated_slips_version() # this eventL # - signals input.py to stop recving input and start draining flows # - and signals the process_manager() to call shutdown_gracefully() From fdd3071c9b852450b526e43c90960ab4a386955c Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 23:21:05 +0200 Subject: [PATCH 36/51] update unit tests --- tests/unit/managers/test_process_manager.py | 10 +- tests/unit/managers/test_redis_manager.py | 18 +- tests/unit/managers/test_update_manager.py | 216 ++++++++++++++++++ tests/unit/slips/test_main.py | 47 ++-- .../slips_files/core/helpers/test_checker.py | 1 - .../core/input/test_dos_protector.py | 5 +- .../core/input/test_zeek_input_utils.py | 64 ++++++ 7 files changed, 319 insertions(+), 42 deletions(-) create mode 100644 tests/unit/managers/test_update_manager.py diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index dcf1d7a5c0..f118c5d882 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize( "input_type, input_information, cli_packet_filter, " - "zeek_or_bro, zeek_dir, line_type", + "zeek_or_bro, line_type", [ # Test case 1: pcap input ( @@ -19,13 +19,12 @@ "test.pcap", "tcp port 80", "zeek", - "/opt/zeek", "conn", ), # Test case 2: zeek input - (InputType.ZEEK, "test.log", "", "bro", "/opt/bro", "dns"), + (InputType.ZEEK, "test.log", "", "bro", "dns"), # Test case 3: stdin input - (InputType.STDIN, "-", "", "zeek", "/opt/zeek", "http"), + (InputType.STDIN, "-", "", "zeek", "http"), ], ) def test_start_input_process( @@ -33,7 +32,6 @@ def test_start_input_process( input_information, cli_packet_filter, zeek_or_bro, - zeek_dir, line_type, ): process_manager = ModuleFactory().create_process_manager_obj() @@ -41,7 +39,6 @@ def test_start_input_process( process_manager.main.input_information = input_information process_manager.main.args.pcapfilter = cli_packet_filter process_manager.main.zeek_bro = zeek_or_bro - process_manager.main.zeek_dir = zeek_dir process_manager.main.line_type = line_type process_manager.main.bloom_filters_man = Mock() @@ -68,7 +65,6 @@ def test_start_input_process( input_information=input_information, cli_packet_filter=cli_packet_filter, zeek_or_bro=zeek_or_bro, - zeek_dir=zeek_dir, line_type=line_type, is_profiler_done_event=process_manager.is_profiler_done_event, is_input_done_event=process_manager.is_input_done_event, diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index 55341c13fb..a67cbf99cf 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -48,7 +48,7 @@ def test_log_redis_server_pid_normal_ports( ): redis_manager = ModuleFactory().create_redis_manager_obj() redis_manager.main.input_information = "input_info" - redis_manager.main.zeek_dir = "zeek_dir" + redis_manager.main.db.get_zeek_output_dir.return_value = "zeek_dir" redis_manager.main.args.output = "output_dir" redis_manager.main.args.daemon = is_daemon redis_manager.main.args.save = save_db @@ -171,7 +171,9 @@ def test_check_redis_database(mock_db): mock_db.rcache.ping.return_value = True with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=False), + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=False + ), patch("managers.redis_manager.RedisDB", return_value=mock_db), ): result = redis_manager.start_redis_cache_if_not_running() @@ -188,7 +190,9 @@ def test_check_redis_database_failure(mock_db): mock_db.rcache.ping.side_effect = redis.exceptions.ConnectionError with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=False), + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=False + ), patch("managers.redis_manager.RedisDB", return_value=mock_db), pytest.raises(redis.exceptions.ConnectionError), ): @@ -203,8 +207,12 @@ def test_check_redis_database_uses_running_cache(mock_db): mock_db.rcache.ping.return_value = True with ( - patch("managers.redis_manager.utils.is_port_in_use", return_value=True), - patch("managers.redis_manager.RedisDB", return_value=mock_db) as mock_redis, + patch( + "managers.redis_manager.utils.is_port_in_use", return_value=True + ), + patch( + "managers.redis_manager.RedisDB", return_value=mock_db + ) as mock_redis, ): result = redis_manager.start_redis_cache_if_not_running() diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py new file mode 100644 index 0000000000..8a050604d2 --- /dev/null +++ b/tests/unit/managers/test_update_manager.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import subprocess +from unittest.mock import Mock, patch + +import pytest + +from managers.update_manager import UpdateManager + + +def create_update_manager(): + """ + Create an UpdateManager with mocked external dependencies. + + Returns: + UpdateManager instance for unit tests. + """ + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock(is_slips_started_by_an_update=False) + conf.auto_update_slips.return_value = True + + with patch("managers.update_manager.ConfigParser", return_value=conf): + return UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=Mock(), + ) + + +@pytest.mark.parametrize( + "remote_url, expected_link", + [ + ( + "https://github.com/stratosphereips/StratosphereLinuxIPS.git", + "https://raw.githubusercontent.com/stratosphereips/" + "StratosphereLinuxIPS/master/update.json", + ), + ( + "git@github.com:stratosphereips/StratosphereLinuxIPS.git", + "https://raw.githubusercontent.com/stratosphereips/" + "StratosphereLinuxIPS/master/update.json", + ), + ("https://example.com/stratosphereips/StratosphereLinuxIPS.git", None), + ], +) +def test_get_master_update_json_link(remote_url, expected_link): + update_manager = create_update_manager() + repo = Mock() + repo.remote.return_value.url = remote_url + + with patch("managers.update_manager.Repo", return_value=repo): + assert update_manager._get_master_update_json_link() == expected_link + + +@pytest.mark.parametrize( + "update_json, expected_dependencies, expected_compatibility", + [ + ( + '{"has_new_dependencies": true, "backwards_compatible": false}', + True, + False, + ), + ( + "{\n" + ' "has_new_dependencies": false,\n' + ' "backwards_compatible": true,\n' + "}\n", + False, + True, + ), + ], +) +def test_update_json_flags( + update_json, expected_dependencies, expected_compatibility +): + update_manager = create_update_manager() + response = Mock() + response.read.return_value = update_json.encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + + with patch.object( + update_manager, + "_get_master_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/master/update.json" + ), + ), patch( + "managers.update_manager.request.urlopen", return_value=response + ) as mock_urlopen: + assert ( + update_manager._new_version_has_new_dependencies() + == expected_dependencies + ) + assert ( + update_manager._is_new_version_backwards_compatible() + == expected_compatibility + ) + mock_urlopen.assert_called_once() + + +@pytest.mark.parametrize( + "update_payload, expected_dependencies, expected_compatibility", + [ + (None, True, False), + ("not-json", True, False), + ], +) +def test_update_json_fallbacks( + update_payload, + expected_dependencies, + expected_compatibility, +): + update_manager = create_update_manager() + + patches = [ + patch.object( + update_manager, + "_get_master_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/master/update.json" + ), + ) + ] + + if update_payload is None: + patches.append( + patch( + "managers.update_manager.request.urlopen", + side_effect=OSError("network error"), + ) + ) + else: + response = Mock() + response.read.return_value = update_payload.encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + patches.append( + patch( + "managers.update_manager.request.urlopen", + return_value=response, + ) + ) + + with patches[0], patches[1]: + assert ( + update_manager._new_version_has_new_dependencies() + == expected_dependencies + ) + assert ( + update_manager._is_new_version_backwards_compatible() + == expected_compatibility + ) + + +def test_get_updated_slips_command_appends_update_flag(): + update_manager = create_update_manager() + process = Mock() + process.cmdline.return_value = ["python3", "slips.py", "-i", "eth0"] + + with patch("managers.update_manager.psutil.Process", return_value=process): + assert update_manager._get_updated_slips_command() == [ + "python3", + "slips.py", + "-i", + "eth0", + "-u", + ] + + +def test_start_updated_slips_verison_starts_detached_process(): + update_manager = create_update_manager() + update_manager._get_updated_slips_command = Mock( + return_value=["python3", "slips.py", "-i", "eth0", "-u"] + ) + + with patch("managers.update_manager.subprocess.Popen") as popen: + process = update_manager.start_updated_slips_version() + + assert process == popen.return_value + popen.assert_called_once_with( + ["python3", "slips.py", "-i", "eth0", "-u"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + start_new_session=True, + ) + + +def test_update_slips_starts_updated_process_before_stopping_current_slips(): + update_manager = create_update_manager() + calls = [] + update_manager.git_pull_master = Mock( + side_effect=lambda: calls.append("git_pull_master") + ) + update_manager.start_updated_slips_version = Mock( + side_effect=lambda: calls.append("start_updated_slips_verison") + ) + update_manager.is_slips_live_updating_event.set = Mock( + side_effect=lambda: calls.append("set_update_event") + ) + + update_manager.update_slips() + + update_manager.git_pull_master.assert_called_once() + update_manager.start_updated_slips_version.assert_called_once() + update_manager.is_slips_live_updating_event.set.assert_called_once() + assert calls == [ + "git_pull_master", + "start_updated_slips_verison", + "set_update_event", + ] diff --git a/tests/unit/slips/test_main.py b/tests/unit/slips/test_main.py index ab84d62894..057ad084a5 100644 --- a/tests/unit/slips/test_main.py +++ b/tests/unit/slips/test_main.py @@ -301,7 +301,8 @@ def test_delete_zeek_files_enabled(): main = ModuleFactory().create_main_obj() main.conf = MagicMock() main.conf.delete_zeek_files.return_value = True - main.zeek_dir = "zeek_dir" + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" with patch("shutil.rmtree") as mock_rmtree: main.delete_zeek_files() @@ -312,13 +313,32 @@ def test_delete_zeek_files_disabled(): main = ModuleFactory().create_main_obj() main.conf = MagicMock() main.conf.delete_zeek_files.return_value = False - main.zeek_dir = "zeek_dir" + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" with patch("shutil.rmtree") as mock_rmtree: main.delete_zeek_files() mock_rmtree.assert_not_called() +def test_store_zeek_dir_copy_reads_zeek_dir_from_db(): + main = ModuleFactory().create_main_obj() + main.conf = MagicMock() + main.conf.store_a_copy_of_zeek_files.return_value = True + main.db = MagicMock() + main.db.get_zeek_output_dir.return_value = "zeek_dir" + main.args.output = "output" + + with ( + patch.object(main, "was_running_zeek", return_value=True), + patch("slips.main.copy_tree") as mock_copy_tree, + patch("builtins.print"), + ): + main.store_zeek_dir_copy() + + mock_copy_tree.assert_called_once_with("zeek_dir", "output/zeek_files") + + # TODO should be moved to utils unit tests after the PR is merged # def test_get_slips_version(): # main = ModuleFactory().create_main_obj() @@ -379,29 +399,6 @@ def test_check_zeek_or_bro_not_found(): mock_terminate.assert_called_once() -@pytest.mark.parametrize( - "store_in_output, expected_dir", - [ - # Test Case 1: Store Zeek files in the output directory - (True, "output/zeek_files"), - # Test Case 2: Use default directory for Zeek files - (False, "zeek_files_inputfile/"), - ], -) -def test_prepare_zeek_output_dir(store_in_output, expected_dir): - main = ModuleFactory().create_main_obj() - main.input_information = "/path/to/inputfile.pcap" - main.args = Mock() - main.args.output = "output" - main.conf = Mock() - main.conf.store_zeek_files_in_the_output_dir.return_value = store_in_output - - with patch("os.path.join", lambda *args: "/".join(args)): - main.prepare_zeek_output_dir() - - assert main.zeek_dir == expected_dir - - def test_terminate_slips_interactive(): main = ModuleFactory().create_main_obj() main.mode = "interactive" diff --git a/tests/unit/slips_files/core/helpers/test_checker.py b/tests/unit/slips_files/core/helpers/test_checker.py index e24c1a6c5b..ce2e45fc4c 100644 --- a/tests/unit/slips_files/core/helpers/test_checker.py +++ b/tests/unit/slips_files/core/helpers/test_checker.py @@ -14,7 +14,6 @@ def test_clear_redis_cache(): checker.clear_redis_cache() checker.main.redis_man.clear_redis_cache_database.assert_called_once() assert checker.main.input_information == "" - assert checker.main.zeek_dir == "" checker.main.redis_man.log_redis_server_pid.assert_called_once_with( 6379, mock.ANY ) diff --git a/tests/unit/slips_files/core/input/test_dos_protector.py b/tests/unit/slips_files/core/input/test_dos_protector.py index 84923bc875..b406ccd8e2 100644 --- a/tests/unit/slips_files/core/input/test_dos_protector.py +++ b/tests/unit/slips_files/core/input/test_dos_protector.py @@ -83,10 +83,7 @@ def test_get_number_of_flows_to_skip_updates_sampling_window_and_prints(): return_value="formatted-ts", ), ): - assert ( - protector.get_number_of_flows_to_skip_and_time_to_stop_sampling() - == 449 - ) + assert protector.get_number_of_flows_to_skip() == 449 assert protector.flow_sampling_stop_time == 160 assert protector._is_now_sampling is True diff --git a/tests/unit/slips_files/core/input/test_zeek_input_utils.py b/tests/unit/slips_files/core/input/test_zeek_input_utils.py index 61e0a92290..ac081e8a8c 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_utils.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_utils.py @@ -2,8 +2,11 @@ # SPDX-License-Identifier: GPL-2.0-only import json import pytest +from types import SimpleNamespace +from unittest.mock import Mock from tests.module_factory import ModuleFactory from slips_files.common.input_type import InputType +from slips_files.core.input.zeek.utils.zeek_input_utils import ZeekInputUtils @pytest.mark.parametrize( @@ -35,3 +38,64 @@ def test_get_ts_from_line_returns_timestamp_for_tabs(): ts, line = input_process.zeek_utils.get_ts_from_line("1.5\tfield\n") assert ts == 1.5 assert line == "1.5\tfield\n" + + +def test_read_zeek_files_drains_generated_lines_during_live_update(tmp_path): + test_file = tmp_path / "conn.log" + test_file.write_text( + "\n".join( + [ + json.dumps({"ts": 1, "uid": "flow-1"}), + json.dumps({"ts": 2, "uid": "flow-2"}), + "", + ] + ), + encoding="utf-8", + ) + live_update_event = Mock() + live_update_event.is_set.return_value = True + db = Mock() + db.is_running_non_stop.return_value = False + db.get_all_zeek_files.return_value = {str(test_file): "eth0"} + input_process = SimpleNamespace( + args=Mock(), + bro_timeout=100, + conf=Mock(), + db=db, + give_profiler=Mock(), + is_slips_live_updating_event=live_update_event, + is_zeek_tabs=False, + lines=0, + print=Mock(), + should_stop=Mock(return_value=True), + store_flows_read_per_second=Mock(), + ) + zeek_utils = ZeekInputUtils(input_process) + zeek_utils.shutdown_zeek_runtime = Mock() + get_flows_to_skip = Mock(return_value=0) + zeek_utils.dos_protector.get_number_of_flows_to_skip = get_flows_to_skip + + assert zeek_utils.read_zeek_files() == 2 + assert input_process.give_profiler.call_count == 2 + zeek_utils.shutdown_zeek_runtime.assert_called_once() + assert get_flows_to_skip.call_count >= 2 + + +@pytest.mark.parametrize( + "store_in_output, expected_dir", + [ + (True, "output/zeek_files"), + (False, "zeek_files_inputfile/"), + ], +) +def test_get_zeek_output_dir(store_in_output, expected_dir): + input_process = ModuleFactory().create_input_obj( + "pcaps/inputfile.pcap", InputType.PCAP + ) + input_process.zeek_dir = None + input_process.args.output = "output" + input_process.conf.store_zeek_files_in_the_output_dir.return_value = ( + store_in_output + ) + + assert input_process.zeek_utils.get_zeek_output_dir() == expected_dir From 0e580dc2eeedbae51f02dac894d59db5291473d2 Mon Sep 17 00:00:00 2001 From: alya Date: Wed, 22 Apr 2026 23:30:32 +0200 Subject: [PATCH 37/51] abort update if uncommitted changes were detected during a git pull --- managers/update_manager.py | 93 ++++++++++++++++++++-- tests/unit/managers/test_update_manager.py | 59 ++++++++++++++ 2 files changed, 146 insertions(+), 6 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 570a9f138c..73dcb555bf 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -15,7 +15,12 @@ from urllib import error, request import psutil -from git import InvalidGitRepositoryError, NoSuchPathError, Repo +from git import ( + GitCommandError, + InvalidGitRepositoryError, + NoSuchPathError, + Repo, +) from slips_files.common.parsers.config_parser import ConfigParser from slips_files.common.slips_utils import utils from slips_files.core.database.database_manager import DBManager @@ -161,6 +166,62 @@ def git_pull_master(self): repo.git.checkout("origin/master") return repo.head.commit + def _get_checkout_overwritten_files( + self, git_error: GitCommandError + ) -> List[str]: + """ + Extract files Git says would be overwritten by checkout. + we'll just print them to the user. + + Parameters: + git_error: The GitPython checkout failure. + + Returns: + The local paths reported by Git, or an empty list if this is not + a local-change checkout conflict. + """ + stderr = getattr(git_error, "stderr", "") or str(git_error) + if ( + "Your local changes to the following files would be " + "overwritten by checkout" not in stderr + ): + return [] + + files = [] + is_file_list = False + for line in stderr.splitlines(): + stripped_line = line.strip().strip("'") + if ( + "Your local changes to the following files would be " + "overwritten by checkout" in stripped_line + ): + is_file_list = True + continue + + if not is_file_list: + continue + + if stripped_line.startswith(("Please commit", "Aborting")): + break + + if stripped_line: + files.append(stripped_line) + + return files + + def _get_target_update_version(self) -> Optional[str]: + """ + Get the target Slips version from cached update metadata. + + Returns: + The update version if known, otherwise None. + """ + update_data = ( + self.cached_update_info or self._read_master_update_json() + ) + version = update_data.get("version") + return version if isinstance(version, str) and version else None + def _get_updated_slips_command(self) -> List[str]: """ Build the command used to start the updated Slips process. @@ -194,8 +255,32 @@ def start_updated_slips_version(self) -> subprocess.Popen: start_new_session=True, ) + def _warn_about_aborted_update( + self, git_error: Optional[GitCommandError] = None + ): + overwritten_files = self._get_checkout_overwritten_files(git_error) + if not overwritten_files: + raise + + target_version = self._get_target_update_version() + update_target = ( + f"Slips v{target_version}" + if target_version + else "the new Slips version" + ) + self.print( + f"Warning: Uncommitted changes to {overwritten_files} detected. " + f"Aborting update to {update_target}, please update Slips " + "manually." + ) + def update_slips(self): - self.git_pull_master() + try: + self.git_pull_master() + except GitCommandError as git_error: + self._warn_about_aborted_update(git_error) + return + self.start_updated_slips_version() # this eventL # - signals input.py to stop recving input and start draining flows @@ -219,10 +304,6 @@ def check_for_update_every_1_day(self) -> bool: """ if self._did_1d_pass_since_last_update(): should_update: bool = self.should_update_slips() - - # @@@@@@@@@@@@@ ALYA DONOT COMMIT THIS - should_update = True - if should_update: self.print( "A new version of Slips is available. " diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py index 8a050604d2..edbe9e3686 100644 --- a/tests/unit/managers/test_update_manager.py +++ b/tests/unit/managers/test_update_manager.py @@ -4,6 +4,7 @@ from unittest.mock import Mock, patch import pytest +from git import GitCommandError from managers.update_manager import UpdateManager @@ -214,3 +215,61 @@ def test_update_slips_starts_updated_process_before_stopping_current_slips(): "start_updated_slips_verison", "set_update_event", ] + + +def test_update_slips_aborts_when_local_changes_block_checkout(): + """ + Ensure local checkout conflicts abort the update without stopping Slips. + + Returns: + None. + """ + update_manager = create_update_manager() + update_manager.cached_update_info = {"version": "1.2.3"} + git_error = GitCommandError( + "git checkout origin/master", + 1, + stderr=( + "error: Your local changes to the following files would be " + "overwritten by checkout:\n" + "\tconfig/slips.yaml\n" + "Please commit your changes or stash them before you switch " + "branches.\n" + "Aborting" + ), + ) + update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.start_updated_slips_version = Mock() + + update_manager.update_slips() + + update_manager.git_pull_master.assert_called_once() + update_manager.start_updated_slips_version.assert_not_called() + update_manager.is_slips_live_updating_event.set.assert_not_called() + update_manager.print.assert_called_once_with( + "Uncommitted changes to ['config/slips.yaml'] detected. " + "Aborting update to Slips v1.2.3, please update Slips manually." + ) + + +def test_update_slips_reraises_unrelated_git_errors(): + """ + Ensure unexpected git failures are not hidden by the update manager. + + Returns: + None. + """ + update_manager = create_update_manager() + git_error = GitCommandError( + "git checkout origin/master", + 128, + stderr="fatal: not a git repository", + ) + update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.start_updated_slips_version = Mock() + + with pytest.raises(GitCommandError): + update_manager.update_slips() + + update_manager.start_updated_slips_version.assert_not_called() + update_manager.is_slips_live_updating_event.set.assert_not_called() From 3c4048cf06e0f34c13e29281efe776daab4f5893 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 16:57:17 +0200 Subject: [PATCH 38/51] update_manager.py: fixthe func checking for slips new version --- managers/update_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 73dcb555bf..6d47752ef3 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -147,12 +147,12 @@ def _is_new_version_backwards_compatible(self) -> bool: def _is_new_version_available(self) -> bool: update_data = self._read_master_update_json() - latest_version = bool(update_data.get("version", False)) + latest_version = update_data.get("version", False) if not latest_version: return False - return utils.get_current_version() == latest_version + return utils.get_current_version() != latest_version def git_pull_master(self): """ From 416f7d3696396b57cd67d672039193db3f24d3f9 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 17:38:20 +0200 Subject: [PATCH 39/51] Make the new updated slips use the same CLI as the old slips. the old shuts down and the new one continues --- managers/process_manager.py | 9 +++++---- managers/update_manager.py | 23 ++++++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/managers/process_manager.py b/managers/process_manager.py index 63dd18f878..3d82c7eed6 100644 --- a/managers/process_manager.py +++ b/managers/process_manager.py @@ -900,10 +900,6 @@ def shutdown_gracefully(self): self.main.db.close_all_dbs() if graceful_shutdown: - print( - "[Process Manager] Slips shutdown gracefully\n", - log_to_logfiles_only=True, - ) if self.is_slips_live_updating_event.is_set(): print( "[Process Manager] Slips is live updating, " @@ -911,6 +907,11 @@ def shutdown_gracefully(self): "instance now.\n", log_to_logfiles_only=True, ) + + print( + "[Process Manager] Slips shutdown gracefully\n", + log_to_logfiles_only=True, + ) else: print( f"[Process Manager] Slips didn't " diff --git a/managers/update_manager.py b/managers/update_manager.py index 6d47752ef3..8895615046 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -246,15 +246,21 @@ def start_updated_slips_version(self) -> subprocess.Popen: Returns: The detached process handle for the updated Slips process. """ - return subprocess.Popen( - self._get_updated_slips_command(), - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + cmd: List[str] = self._get_updated_slips_command() + + str_cmd = " ".join(cmd) + self.print(f"Starting updated Slips version using command: {str_cmd}") + + # without dev/null redirection, the new updated slips will use the + # same cli as the old slips. so this is intentional. + process = subprocess.Popen( + cmd, close_fds=True, - start_new_session=True, ) + self.print("Done starting the updated Slips version.") + return process + def _warn_about_aborted_update( self, git_error: Optional[GitCommandError] = None ): @@ -282,7 +288,7 @@ def update_slips(self): return self.start_updated_slips_version() - # this eventL + # this event # - signals input.py to stop recving input and start draining flows # - and signals the process_manager() to call shutdown_gracefully() self.is_slips_live_updating_event.set() @@ -304,6 +310,9 @@ def check_for_update_every_1_day(self) -> bool: """ if self._did_1d_pass_since_last_update(): should_update: bool = self.should_update_slips() + # @@@@@@@@@@@@@@@@@@@@@@ + should_update = True + if should_update: self.print( "A new version of Slips is available. " From 5532a78722cd804edff9aa3a407f6cf63508da3c Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 17:49:28 +0200 Subject: [PATCH 40/51] if old slips is started by -m, make sure the new slips knows which port the old slips was using. --- managers/redis_manager.py | 8 ++++++++ managers/update_manager.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/managers/redis_manager.py b/managers/redis_manager.py index 2216037625..ae0903f27e 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -382,6 +382,14 @@ def get_redis_port(self) -> int: -m or the default port if all ports are unavailable, this function terminates slips """ + # when slips is started by another slips during an update + # handover, make sure the new slips + # continues using the same old redis db and port. + if self.main.args.is_slips_started_by_an_update: + if self.main.args.port: + return int(self.main.args.port) + return DEFAULT_REDIS_PORT + if self.main.args.port: redis_port = int(self.main.args.port) # if the default port is already in use, slips should override it diff --git a/managers/update_manager.py b/managers/update_manager.py index 8895615046..bccb349277 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -227,7 +227,9 @@ def _get_updated_slips_command(self) -> List[str]: Build the command used to start the updated Slips process. Returns: - The current Slips cmd plus (-u). + The current Slips cmd plus (-u). If the current Slips was + started with -m, pass the current Redis port explicitly so the + updated process reuses it. """ try: cmd = psutil.Process().cmdline() @@ -237,7 +239,14 @@ def _get_updated_slips_command(self) -> List[str]: if not cmd: cmd = [sys.executable, *sys.argv] - return [*cmd, "-u"] + cmd = [*cmd, "-u"] + + if self.args.multiinstance: + cmd.remove("-m") + redis_port = self.db.get_used_redis_port() + cmd.extend(["-P", str(redis_port)]) + + return cmd def start_updated_slips_version(self) -> subprocess.Popen: """ From f6cd346bd233114149752678300e7d2e2859ea38 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 17:49:47 +0200 Subject: [PATCH 41/51] update unit tests --- tests/unit/managers/test_redis_manager.py | 36 +++++++++++++++++++++- tests/unit/managers/test_update_manager.py | 31 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index a67cbf99cf..002e4bc2e5 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -7,7 +7,11 @@ import pytest from tests.module_factory import ModuleFactory -from managers.redis_manager import UserCancelledErr +from managers.redis_manager import ( + DEFAULT_REDIS_PORT, + RedisManager, + UserCancelledErr, +) from slips_files.common.input_type import InputType @@ -584,6 +588,36 @@ def test_get_redis_port( mock_terminate.assert_called() +@pytest.mark.parametrize( + "args_port, expected_port", + [ + ("32768", 32768), + (None, DEFAULT_REDIS_PORT), + ], +) +def test_get_redis_port_started_by_update(args_port, expected_port, mock_db): + redis_manager = RedisManager(Mock()) + redis_manager.main.args = Mock() + redis_manager.main.args.is_slips_started_by_an_update = True + redis_manager.main.args.port = args_port + + with ( + patch.object( + redis_manager, "_get_dbmanager_without_starting_a_new_server" + ) as mock_db_mgr, + patch.object(redis_manager, "confirm_server_altering") as mock_confirm, + patch.object( + redis_manager, "get_random_redis_port" + ) as mock_random_port, + ): + result = redis_manager.get_redis_port() + + assert result == expected_port + mock_db_mgr.assert_not_called() + mock_confirm.assert_not_called() + mock_random_port.assert_not_called() + + @pytest.mark.parametrize( "total1, total2, expected", [ diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py index edbe9e3686..4de4edb1b0 100644 --- a/tests/unit/managers/test_update_manager.py +++ b/tests/unit/managers/test_update_manager.py @@ -20,7 +20,10 @@ def create_update_manager(): db.is_running_non_stop.return_value = True conf = Mock() - conf.get_args.return_value = Mock(is_slips_started_by_an_update=False) + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) conf.auto_update_slips.return_value = True with patch("managers.update_manager.ConfigParser", return_value=conf): @@ -172,6 +175,32 @@ def test_get_updated_slips_command_appends_update_flag(): ] +def test_get_updated_slips_command_passes_multiinstance_redis_port(): + update_manager = create_update_manager() + update_manager.args.multiinstance = True + update_manager.db.get_used_redis_port.return_value = 32768 + process = Mock() + process.cmdline.return_value = [ + "python3", + "slips.py", + "-m", + "-i", + "eth0", + ] + + with patch("managers.update_manager.psutil.Process", return_value=process): + assert update_manager._get_updated_slips_command() == [ + "python3", + "slips.py", + "-m", + "-i", + "eth0", + "-u", + "-P", + "32768", + ] + + def test_start_updated_slips_verison_starts_detached_process(): update_manager = create_update_manager() update_manager._get_updated_slips_command = Mock( From e623d5035d6f39cc2190c5c2b3a6e9fdb3672929 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 23:15:28 +0200 Subject: [PATCH 42/51] update unit tests and docs --- CHANGELOG.md | 3 +++ docs/usage.md | 6 +++--- managers/update_manager.py | 4 +++- tests/unit/slips/test_main.py | 3 +++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1347ce71b9..a2ea80c65a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* Add optional live Slips auto-update support controlled by `update.auto_update_slips`. + + 1.1.19 (Mar 31st, 2026) * Add SSH bruteforce detection based on Zeek SSH, software, and notice logs. diff --git a/docs/usage.md b/docs/usage.md index 8d9c39b88e..654e097fa8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -531,14 +531,14 @@ This includes persistent artifacts such as ```p2p_trust_runtime/``` and shared m **Live Slips auto update** -Use ```update.auto_update``` to enable or disable automatic live updates of the installed Slips version. +Use ```update.auto_update_slips``` to enable or disable automatic live updates of the installed Slips version. ```yaml update: - auto_update: false + auto_update_slips: false ``` -This setting is separate from the runtime ```update_manager``` module, which only updates TI feeds and related files. +This setting is separate from the runtime ```feeds_update_manager``` module, which only updates TI feeds and related files. Automatic Slips updates may overwrite the default config files shipped with Slips. If you want to keep local config changes safe, do not modify the default config files. Create and use your own config files with different names instead. diff --git a/managers/update_manager.py b/managers/update_manager.py index bccb349277..4124c18239 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -164,7 +164,9 @@ def git_pull_master(self): repo = Repo(".") repo.remote("origin").fetch("master") repo.git.checkout("origin/master") - return repo.head.commit + self.print( + "Done pulling new version and checking out master " "branch." + ) def _get_checkout_overwritten_files( self, git_error: GitCommandError diff --git a/tests/unit/slips/test_main.py b/tests/unit/slips/test_main.py index 057ad084a5..cfba3566fe 100644 --- a/tests/unit/slips/test_main.py +++ b/tests/unit/slips/test_main.py @@ -442,6 +442,7 @@ def main_obj(tmp_path): main = ModuleFactory().create_main_obj() main.args = MagicMock() main.args.output = str(tmp_path / "test_output") + main.args.is_slips_started_by_an_update = False return main @@ -485,6 +486,8 @@ def test_prepare_output_dir_without_o_flag( ): main = ModuleFactory().create_main_obj() main.args = MagicMock() + main.args.is_slips_started_by_an_update = False + main.args.output = None main.parent_output_dir = str(tmp_path) main.input_information = "/fake/input/wlp3s0" monkeypatch.setattr(sys, "argv", ["script.py"]) # No -o From 49f944579cbf352988ae312b786a5877db2df5f6 Mon Sep 17 00:00:00 2001 From: alya Date: Thu, 23 Apr 2026 23:56:50 +0200 Subject: [PATCH 43/51] add auto update docs --- docs/immune/auto_update.md | 146 +++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 docs/immune/auto_update.md diff --git a/docs/immune/auto_update.md b/docs/immune/auto_update.md new file mode 100644 index 0000000000..e1c862ce8a --- /dev/null +++ b/docs/immune/auto_update.md @@ -0,0 +1,146 @@ +# Slips Auto Update + +## Table of Contents + +- [Overview](#overview) +- [How Auto-Update Works](#how-auto-update-works) + * [New version checks](#new-version-checks) + * [Updating](#updating) + + [Redis handling](#redis-handling) + + [Zeek log handling](#zeek-log-handling) +- [Draining and Shutdown](#draining-and-shutdown) +- [How to use it](#how-to-use-it) + +## Overview + +Slips auto update functionality was designed to allow a running instance of +Slips to update itself with no downtime during the transition between versions. + +Updates usually consist of: + +A full application stop -> update -> restart sequence. +rather than simple restart. + +That sequence would lead to downtime and temporarily missing of flows during the upgrade. + +The implemented update mechanism was designed around "handover" instead of "restarts". +Where slips checks periodically for new compatible versions, pulls the update, +starts the new version, and orchestrates a controlled handover from the old +version to the new one. + + +## How Auto-Update Works + +```text +Old Slips running + ↓ +Check for compatible update + ↓ +git pull origin master + ↓ +Start new Slips with -u + ↓ +New Slips restores state and starts processing + ↓ +Old Slips drains + ↓ +Old Slips graceful shutdown + ↓ +New Slips continues normally +``` + + +### New version checks + +Slips checks for updates once per day. + +This is handled by the UpdateManager, which: + +- checks whether auto-update is enabled in the config file +- checks whether a new version exists +- checks compatibility before attempting update. + +Compatibility is determined using an `update.json` file hosted with the +deployed version. + +This file includes metadata about the new version such as: + +- latest version, +- backwards compatibility, +- whether new dependencies are needed. + + +The compatibility parser was added so we avoid updating to incompatible new releases. + + +**The update is aborted if:** + +- `auto_update_slips` is disabled in slips config. +- Slips is running on offline input instead of interface +- no newer version exists +- update is incompatible according to `update.json` +- local uncommitted changes are detected during `git pull` +- startup of the new version fails + + +### Updating Logic + + +When designing this, our main goal was zero downtime and zero missed flows during the update. This has the cost of maybe reading a very few duplicate flows, and this was done by +Starting the new Slips before stopping the old one. + +How this is done is: +- The old version starts the new one with the undocumented `-u` flag. +- The `-u` flag tells the new Slips instance that: 1. this is not a fresh run and 2. Slips should continue existing analysis and handle database migrations. +- do not overwrite: output dir, log files, and previous analysis artifacts/metrics. + + +#### Redis handling + +The new Slips does not flush Redis on startup. + +Instead, it appends to the existing Redis state as if the old process never +stopped. + +Without this, ongoing detections, states, and evidence state would be lost. + + +Now what happen when in the very few seconds during handover, a msg from the old slips' pub/sub is published, and the new updated slips receives it? + +To avoid this we added Pub/Sub message versioning, now each pub/sub message includes +the Slips version and consumers ignore messages that belong to the updated version and only read msgs intended +for them. + + +#### Zeek log handling + +The new Slips starts a new zeek process that uses new zeek log files in ```output/zeek_files/slips_vx.y.z```. +This ensures that the old zeek logs are not modified, re-read or overwritten during the update + +This was a major simplification because sharing Zeek logs between versions +introduced complexity and race conditions. + + +### Draining and Shutdown of the old Slips + +Once the updated Slips is confirmed to be running: + +the old Slips begins draining. + +Draining means: + +- stop ingesting new flows. +- finish processing pending flows. + + +PS: the new updated slips version starts reading flows before the old one starts draining to ensure 0 downtime. + +## How to use it + +enable ```auto_update_slips``` in ```config/slips.yaml``` and run slips on your interface. + +now whenever a new version of Slips is available, it will update itself and the new slips will use the same CLI as the old one. + +## PR + +https://github.com/stratosphereips/StratosphereLinuxIPS/pull/1915 From c0d4e192bcef36d4e3922dca84b61f7a13f1a035 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 24 Apr 2026 01:22:35 +0300 Subject: [PATCH 44/51] update unit tests --- docs/immune/auto_update.md | 5 ++- slips/daemon.py | 2 +- tests/module_factory.py | 45 +++++++++++++++---- tests/unit/managers/test_process_manager.py | 7 +-- tests/unit/managers/test_redis_manager.py | 1 + tests/unit/managers/test_slips.py | 9 +++- tests/unit/managers/test_update_manager.py | 8 +--- tests/unit/slips/test_main.py | 2 +- .../core/database/test_database.py | 11 +++-- .../unit/slips_files/core/input/test_input.py | 15 +++---- 10 files changed, 67 insertions(+), 38 deletions(-) diff --git a/docs/immune/auto_update.md b/docs/immune/auto_update.md index e1c862ce8a..e4c4815aa6 100644 --- a/docs/immune/auto_update.md +++ b/docs/immune/auto_update.md @@ -5,11 +5,12 @@ - [Overview](#overview) - [How Auto-Update Works](#how-auto-update-works) * [New version checks](#new-version-checks) - * [Updating](#updating) + * [Updating Logic](#updating-logic) + [Redis handling](#redis-handling) + [Zeek log handling](#zeek-log-handling) -- [Draining and Shutdown](#draining-and-shutdown) + * [Draining and Shutdown of the old Slips](#draining-and-shutdown-of-the-old-slips) - [How to use it](#how-to-use-it) +- [PR](#pr) ## Overview diff --git a/slips/daemon.py b/slips/daemon.py index 653e64bcb6..3b8630fc7e 100644 --- a/slips/daemon.py +++ b/slips/daemon.py @@ -49,7 +49,7 @@ def print(self, text, **kwargs): def create_std_streams(self): """Create standard steam files and dirs and clear them""" - if self.args.is_slips_started_by_an_update: + if self.slips.args.is_slips_started_by_an_update: return std_streams = [self.stderr, self.stdout, self.logsfile] diff --git a/tests/module_factory.py b/tests/module_factory.py index 6851169b55..9f76c5a859 100644 --- a/tests/module_factory.py +++ b/tests/module_factory.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only import shutil +import importlib.util +import sys from contextlib import contextmanager from unittest.mock import ( patch, @@ -108,7 +110,16 @@ def create_db_manager_obj( return db def create_main_obj(self): - from slips.main import Main + try: + from slips.main import Main + except ModuleNotFoundError: + module_spec = importlib.util.spec_from_file_location( + "slips_main_for_tests", os.path.join("slips", "main.py") + ) + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = module + module_spec.loader.exec_module(module) + Main = module.Main """returns an instance of Main() class in slips.py""" main = Main(testing=True) @@ -240,14 +251,30 @@ def create_go_director_obj(self, mock_db): @patch(DB_MANAGER, name="mock_db") def create_daemon_object(self, mock_db): - from slips.daemon import Daemon + try: + from slips.daemon import Daemon + + daemon_module = "slips.daemon" + except ModuleNotFoundError: + module_spec = importlib.util.spec_from_file_location( + "slips_daemon_for_tests", os.path.join("slips", "daemon.py") + ) + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = module + module_spec.loader.exec_module(module) + Daemon = module.Daemon + daemon_module = module_spec.name with ( - patch("slips.daemon.Daemon.read_pidfile", return_type=None), - patch("slips.daemon.Daemon.read_configuration"), + patch(f"{daemon_module}.Daemon.read_pidfile", return_value=None), + patch(f"{daemon_module}.Daemon.read_configuration"), patch("builtins.open", mock_open(read_data=None)), ): - daemon = Daemon(MagicMock()) + slips = MagicMock() + slips.args.stopdaemon = True + slips.args.is_slips_started_by_an_update = False + slips.args.output = "output" + daemon = Daemon(slips) daemon.stderr = "errors.log" daemon.stdout = "slips.log" daemon.stdin = "/dev/null" @@ -495,13 +522,12 @@ def create_input_obj( from slips_files.core.input import Input from slips_files.core.output import Output - zeek_tmp_dir = os.path.join(os.getcwd(), "zeek_dir_for_testing") input = Input( logger=Output(), output_dir="dummy_output_dir", redis_port=6379, termination_event=Mock(), - slips_args=Mock(), + slips_args=Mock(output="dummy_output_dir"), conf=Mock(), ppid=Mock(), bloom_filters_manager=Mock(), @@ -511,7 +537,6 @@ def create_input_obj( input_information=input_information, cli_packet_filter=None, zeek_or_bro=check_zeek_or_bro(), - zeek_dir=zeek_tmp_dir, line_type=line_type, is_profiler_done_event=Mock(), ) @@ -659,7 +684,9 @@ def create_spamhaus_obj(self, mock_db): @patch(MODULE_DB_MANAGER, name="mock_db") def create_update_manager_obj(self, mock_db): - from modules.update_manager.update_manager import FeedsUpdateManager + from modules.feeds_update_manager.feeds_update_manager import ( + FeedsUpdateManager, + ) update_manager = FeedsUpdateManager( logger=self.logger, diff --git a/tests/unit/managers/test_process_manager.py b/tests/unit/managers/test_process_manager.py index f118c5d882..a5f0b4cbeb 100644 --- a/tests/unit/managers/test_process_manager.py +++ b/tests/unit/managers/test_process_manager.py @@ -81,13 +81,13 @@ def test_start_input_process( @pytest.mark.parametrize( "is_slips_started_by_an_update,expected_is_set", - [(True, True), (False, False)], + [(True, False), (False, False)], ) def test_init_sets_live_update_event_for_update_start( is_slips_started_by_an_update, expected_is_set, ): - """Test that -u startup marks Slips as live-updating.""" + """Test that ProcessManager starts with an unset live-update event.""" main_mock = Mock() main_mock.args.is_slips_started_by_an_update = ( is_slips_started_by_an_update @@ -553,12 +553,13 @@ def test_start_update_manager( asyncio_called, ): process_manager = ModuleFactory().create_process_manager_obj() + process_manager.main.args.output = "output" mock_lock_instance = Mock() mock_lock.return_value.__enter__.return_value = mock_lock_instance mock_update_manager = Mock() with patch( - "managers.process_manager.UpdateManager", + "managers.process_manager.FeedsUpdateManager", return_value=mock_update_manager, ): process_manager.start_update_manager( diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index 002e4bc2e5..49d61ffd09 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -561,6 +561,7 @@ def test_get_redis_port( redis_manager = ModuleFactory().create_redis_manager_obj() redis_manager.main.args.port = args_port redis_manager.main.args.multiinstance = multiinstance + redis_manager.main.args.is_slips_started_by_an_update = False with ( patch.object( diff --git a/tests/unit/managers/test_slips.py b/tests/unit/managers/test_slips.py index 2ba204b07a..a6e383bb66 100644 --- a/tests/unit/managers/test_slips.py +++ b/tests/unit/managers/test_slips.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-only """Unit test for ../slips.py""" +from unittest.mock import patch + from tests.module_factory import ModuleFactory @@ -49,4 +51,9 @@ def test_load_modules(): def test_clear_redis_cache_database(): redis_manager = ModuleFactory().create_redis_manager_obj() - assert redis_manager.clear_redis_cache_database() + + with patch("redis.StrictRedis") as mock_redis: + mock_redis_instance = mock_redis.return_value + + assert redis_manager.clear_redis_cache_database() + mock_redis_instance.flushdb.assert_called_once() diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py index 4de4edb1b0..0a46f1ac03 100644 --- a/tests/unit/managers/test_update_manager.py +++ b/tests/unit/managers/test_update_manager.py @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only -import subprocess from unittest.mock import Mock, patch import pytest @@ -192,7 +191,6 @@ def test_get_updated_slips_command_passes_multiinstance_redis_port(): assert update_manager._get_updated_slips_command() == [ "python3", "slips.py", - "-m", "-i", "eth0", "-u", @@ -213,11 +211,7 @@ def test_start_updated_slips_verison_starts_detached_process(): assert process == popen.return_value popen.assert_called_once_with( ["python3", "slips.py", "-i", "eth0", "-u"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, close_fds=True, - start_new_session=True, ) @@ -276,7 +270,7 @@ def test_update_slips_aborts_when_local_changes_block_checkout(): update_manager.start_updated_slips_version.assert_not_called() update_manager.is_slips_live_updating_event.set.assert_not_called() update_manager.print.assert_called_once_with( - "Uncommitted changes to ['config/slips.yaml'] detected. " + "Warning: Uncommitted changes to ['config/slips.yaml'] detected. " "Aborting update to Slips v1.2.3, please update Slips manually." ) diff --git a/tests/unit/slips/test_main.py b/tests/unit/slips/test_main.py index cfba3566fe..8c095e0a15 100644 --- a/tests/unit/slips/test_main.py +++ b/tests/unit/slips/test_main.py @@ -331,7 +331,7 @@ def test_store_zeek_dir_copy_reads_zeek_dir_from_db(): with ( patch.object(main, "was_running_zeek", return_value=True), - patch("slips.main.copy_tree") as mock_copy_tree, + patch(f"{main.__class__.__module__}.copy_tree") as mock_copy_tree, patch("builtins.print"), ): main.store_zeek_dir_copy() diff --git a/tests/unit/slips_files/core/database/test_database.py b/tests/unit/slips_files/core/database/test_database.py index 9c54ff69d5..c7343aa964 100644 --- a/tests/unit/slips_files/core/database/test_database.py +++ b/tests/unit/slips_files/core/database/test_database.py @@ -147,11 +147,14 @@ def test_add_mac_addr_with_ipv6_association(): def test_get_the_other_ip_version(): db = ModuleFactory().create_db_manager_obj(6379, flush_db=True) - # profileid is ipv4 + profileid_ipv4 = "profile_192.168.250.250" ipv6 = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - db.set_ipv6_of_profile(profileid, ipv6) - # the other ip version is ipv6 - other_ip = json.loads(db.get_the_other_ip_version(profileid)) + + db.rdb.get_ipv6_from_profile = Mock(return_value=json.dumps(ipv6)) + + other_ip = json.loads(db.get_the_other_ip_version(profileid_ipv4)) + + db.rdb.get_ipv6_from_profile.assert_called_once_with(profileid_ipv4) assert other_ip == ipv6 diff --git a/tests/unit/slips_files/core/input/test_input.py b/tests/unit/slips_files/core/input/test_input.py index 4db7e39cfb..9ec2a15a82 100644 --- a/tests/unit/slips_files/core/input/test_input.py +++ b/tests/unit/slips_files/core/input/test_input.py @@ -21,30 +21,25 @@ def test_handle_pcap_input(tmp_path, input_type, input_information): input_process = ModuleFactory().create_input_obj( input_information, input_type ) - input_process.zeek_dir = tmp_path handler = input_process.input_handlers[input_type] input_process.is_running_non_stop = False handler.file_remover.start = Mock() - # Mock attributes and methods used inside the function - input_process.zeek_utils.ensure_zeek_dir = Mock() + input_process.zeek_utils.create_zeek_output_dir = Mock( + return_value=str(tmp_path) + ) input_process.zeek_utils.init_zeek = Mock() input_process.zeek_utils.read_zeek_files = Mock(return_value=7) assert handler.run() is True - # Assert that the expected methods were called - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once_with( - handler.observer, input_process.zeek_dir, input_process.given_path + handler.observer, str(tmp_path), input_process.given_path ) input_process.zeek_utils.read_zeek_files.assert_called_once() handler.file_remover.start.assert_not_called() assert input_process.lines == 7 - # Clean up any directories created (safe guard) - if os.path.exists(input_process.zeek_dir): - shutil.rmtree(input_process.zeek_dir, ignore_errors=True) - @pytest.mark.parametrize( "zeek_dir, is_tabs", From 0e3e89e5ec68bc27dd0bdbe6fdfe1deb23d80b43 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 24 Apr 2026 01:30:28 +0300 Subject: [PATCH 45/51] abort update on any git err --- managers/update_manager.py | 11 +++++++---- tests/unit/managers/test_update_manager.py | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/managers/update_manager.py b/managers/update_manager.py index 4124c18239..5a526b2444 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -17,6 +17,7 @@ import psutil from git import ( GitCommandError, + GitError, InvalidGitRepositoryError, NoSuchPathError, Repo, @@ -277,7 +278,11 @@ def _warn_about_aborted_update( ): overwritten_files = self._get_checkout_overwritten_files(git_error) if not overwritten_files: - raise + self.print( + "Warning: Aborting Slips update because a git error " + f"occurred: {git_error}" + ) + return target_version = self._get_target_update_version() update_target = ( @@ -294,7 +299,7 @@ def _warn_about_aborted_update( def update_slips(self): try: self.git_pull_master() - except GitCommandError as git_error: + except GitError as git_error: self._warn_about_aborted_update(git_error) return @@ -321,8 +326,6 @@ def check_for_update_every_1_day(self) -> bool: """ if self._did_1d_pass_since_last_update(): should_update: bool = self.should_update_slips() - # @@@@@@@@@@@@@@@@@@@@@@ - should_update = True if should_update: self.print( diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py index 0a46f1ac03..3a243c6d9e 100644 --- a/tests/unit/managers/test_update_manager.py +++ b/tests/unit/managers/test_update_manager.py @@ -275,9 +275,9 @@ def test_update_slips_aborts_when_local_changes_block_checkout(): ) -def test_update_slips_reraises_unrelated_git_errors(): +def test_update_slips_aborts_on_unrelated_git_errors(): """ - Ensure unexpected git failures are not hidden by the update manager. + Ensure unexpected git failures abort the update without crashing. Returns: None. @@ -291,8 +291,11 @@ def test_update_slips_reraises_unrelated_git_errors(): update_manager.git_pull_master = Mock(side_effect=git_error) update_manager.start_updated_slips_version = Mock() - with pytest.raises(GitCommandError): - update_manager.update_slips() + update_manager.update_slips() update_manager.start_updated_slips_version.assert_not_called() update_manager.is_slips_live_updating_event.set.assert_not_called() + update_manager.print.assert_called_once_with( + "Warning: Aborting Slips update because a git error occurred: " + f"{git_error}" + ) From 7eb7b76172c0b154d3d898145aae25305cbff5f8 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 24 Apr 2026 01:34:25 +0300 Subject: [PATCH 46/51] update unit tests --- .../unit/slips_files/core/input/test_input.py | 20 ++++++++++++------- .../core/input/test_interface_input.py | 4 ++-- .../slips_files/core/input/test_pcap_input.py | 4 ++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/unit/slips_files/core/input/test_input.py b/tests/unit/slips_files/core/input/test_input.py index 9ec2a15a82..fbaf9f1d30 100644 --- a/tests/unit/slips_files/core/input/test_input.py +++ b/tests/unit/slips_files/core/input/test_input.py @@ -511,17 +511,23 @@ def test_get_file_handle_non_existing_file(): assert file_handle is False -def test_ensure_zeek_dir_creates_dir(tmp_path): - """Test that ensure_zeek_dir creates the directory if missing.""" +def test_create_zeek_output_dir_creates_dir(tmp_path): + """Test that create_zeek_output_dir creates the directory if missing.""" input_process = ModuleFactory().create_input_obj( "", InputType.ZEEK_LOG_FILE ) - zeek_dir = tmp_path / "zeek_logs" - input_process.zeek_dir = str(zeek_dir) + input_process.given_path = str(tmp_path / "capture.pcap") + input_process.args.output = str(tmp_path / "output") + input_process.zeek_utils.is_running_non_stop = False + input_process.conf.store_zeek_files_in_the_output_dir.return_value = True - assert not os.path.exists(input_process.zeek_dir) - input_process.zeek_utils.ensure_zeek_dir() - assert os.path.exists(input_process.zeek_dir) + zeek_dir = input_process.zeek_utils.create_zeek_output_dir() + + assert zeek_dir == os.path.join(input_process.args.output, "zeek_files") + assert os.path.exists(zeek_dir) + input_process.db.set_input_metadata.assert_called_once_with( + {"zeek_dir": zeek_dir} + ) def test_check_if_time_to_del_rotated_files_deletes_old_files(): diff --git a/tests/unit/slips_files/core/input/test_interface_input.py b/tests/unit/slips_files/core/input/test_interface_input.py index b87f64eb74..f3a4c945d1 100644 --- a/tests/unit/slips_files/core/input/test_interface_input.py +++ b/tests/unit/slips_files/core/input/test_interface_input.py @@ -10,7 +10,7 @@ def test_interface_input_runs_for_single_interface(tmp_path): input_process.args.interface = "eth0" input_process.args.access_point = False input_process.is_running_non_stop = False - input_process.zeek_utils.ensure_zeek_dir = MagicMock() + input_process.zeek_utils.create_zeek_output_dir = MagicMock() input_process.zeek_utils.init_zeek = MagicMock() input_process.zeek_utils.read_zeek_files = MagicMock(return_value=4) @@ -18,7 +18,7 @@ def test_interface_input_runs_for_single_interface(tmp_path): with patch("os.path.exists", return_value=True): assert handler.run() is True - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once() input_process.zeek_utils.read_zeek_files.assert_called_once() assert input_process.lines == 4 diff --git a/tests/unit/slips_files/core/input/test_pcap_input.py b/tests/unit/slips_files/core/input/test_pcap_input.py index b56ab41a91..536923040a 100644 --- a/tests/unit/slips_files/core/input/test_pcap_input.py +++ b/tests/unit/slips_files/core/input/test_pcap_input.py @@ -11,14 +11,14 @@ def test_pcap_input_runs_through_zeek_utils(tmp_path): ) input_process.zeek_dir = str(tmp_path) input_process.is_running_non_stop = False - input_process.zeek_utils.ensure_zeek_dir = MagicMock() + input_process.zeek_utils.create_zeek_output_dir = MagicMock() input_process.zeek_utils.init_zeek = MagicMock() input_process.zeek_utils.read_zeek_files = MagicMock(return_value=3) handler = input_process.input_handlers[InputType.PCAP] assert handler.run() is True - input_process.zeek_utils.ensure_zeek_dir.assert_called_once() + input_process.zeek_utils.create_zeek_output_dir.assert_called_once() input_process.zeek_utils.init_zeek.assert_called_once() input_process.zeek_utils.read_zeek_files.assert_called_once() assert input_process.lines == 3 From 9eca1330d473cb9f34ba915ce5422f288432ae99 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 24 Apr 2026 01:46:06 +0300 Subject: [PATCH 47/51] update unit tests --- tests/unit/slips_files/core/input/test_zeek_input_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/slips_files/core/input/test_zeek_input_utils.py b/tests/unit/slips_files/core/input/test_zeek_input_utils.py index ac081e8a8c..03fb56e8b6 100644 --- a/tests/unit/slips_files/core/input/test_zeek_input_utils.py +++ b/tests/unit/slips_files/core/input/test_zeek_input_utils.py @@ -85,17 +85,18 @@ def test_read_zeek_files_drains_generated_lines_during_live_update(tmp_path): "store_in_output, expected_dir", [ (True, "output/zeek_files"), - (False, "zeek_files_inputfile/"), + (False, "zeek_files_inputfile"), ], ) -def test_get_zeek_output_dir(store_in_output, expected_dir): +def test_create_zeek_output_dir(store_in_output, expected_dir): input_process = ModuleFactory().create_input_obj( "pcaps/inputfile.pcap", InputType.PCAP ) input_process.zeek_dir = None input_process.args.output = "output" + input_process.zeek_utils.is_running_non_stop = False input_process.conf.store_zeek_files_in_the_output_dir.return_value = ( store_in_output ) - assert input_process.zeek_utils.get_zeek_output_dir() == expected_dir + assert input_process.zeek_utils.create_zeek_output_dir() == expected_dir From 2ea4ff1116f0e4db3cf94b58065cb64221b39823 Mon Sep 17 00:00:00 2001 From: alya Date: Fri, 24 Apr 2026 16:01:28 +0300 Subject: [PATCH 48/51] update iris main config for integration tests --- tests/integration/config/slips_iris_main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/config/slips_iris_main.yaml b/tests/integration/config/slips_iris_main.yaml index d095cc84b4..6bc3dcbd22 100644 --- a/tests/integration/config/slips_iris_main.yaml +++ b/tests/integration/config/slips_iris_main.yaml @@ -97,7 +97,7 @@ parameters: analysis_direction: out client_ips: [] debug: 0 - default_rotation_interval: 30sec + default_rotation_interval: 1day deletePrevdb: true delete_zeek_files: false export_format: json @@ -127,6 +127,8 @@ threatintelligence: ssl_feeds: config/SSL_feeds.csv ti_files: config/TI_feeds.csv wait_for_TI_to_finish: false +update: + auto_update_slips: false virustotal: api_key_file: config/vt_api_key virustotal_update_period: 259200 From d15c3c1fc3e0875506539f69c8a8423677ede485 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 25 Apr 2026 10:39:15 +0300 Subject: [PATCH 49/51] organize integration_tests dir to be able to group config files with their tests --- modules/fides/fides.py | 2 +- tests/integration/conftest.py | 11 ++++++++ .../integration/test_config_files/__init__.py | 0 .../config_test.yaml} | 0 .../{ => test_config_files}/test.yaml | 0 .../{ => test_config_files}/test2.yaml | 0 .../test_config_files.py | 25 ++++++------------- tests/integration/test_dataset/__init__.py | 0 .../{ => test_dataset}/test_dataset.py | 0 tests/integration/test_fides/__init__.py | 0 .../{config => test_fides}/fides.conf.yml | 0 .../{config => test_fides}/fides_config.yaml | 0 .../{ => test_fides}/test_fides.py | 8 +++--- tests/integration/test_iris/__init__.py | 0 .../integration/test_pcap_dataset/__init__.py | 0 .../test_pcap_dataset.py | 0 tests/integration/test_portscans/__init__.py | 0 .../{ => test_portscans}/test_portscans.py | 0 .../integration/test_zeek_dataset/__init__.py | 0 .../test_zeek_dataset.py | 0 tests/run_all_tests.sh | 6 ++--- 21 files changed, 27 insertions(+), 25 deletions(-) create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_config_files/__init__.py rename tests/integration/{config/test.yaml => test_config_files/config_test.yaml} (100%) rename tests/integration/{ => test_config_files}/test.yaml (100%) rename tests/integration/{ => test_config_files}/test2.yaml (100%) rename tests/integration/{ => test_config_files}/test_config_files.py (91%) create mode 100644 tests/integration/test_dataset/__init__.py rename tests/integration/{ => test_dataset}/test_dataset.py (100%) create mode 100644 tests/integration/test_fides/__init__.py rename tests/integration/{config => test_fides}/fides.conf.yml (100%) rename tests/integration/{config => test_fides}/fides_config.yaml (100%) rename tests/integration/{ => test_fides}/test_fides.py (98%) create mode 100644 tests/integration/test_iris/__init__.py create mode 100644 tests/integration/test_pcap_dataset/__init__.py rename tests/integration/{ => test_pcap_dataset}/test_pcap_dataset.py (100%) create mode 100644 tests/integration/test_portscans/__init__.py rename tests/integration/{ => test_portscans}/test_portscans.py (100%) create mode 100644 tests/integration/test_zeek_dataset/__init__.py rename tests/integration/{ => test_zeek_dataset}/test_zeek_dataset.py (100%) diff --git a/modules/fides/fides.py b/modules/fides/fides.py index 3837573070..80867f7efd 100644 --- a/modules/fides/fides.py +++ b/modules/fides/fides.py @@ -231,6 +231,6 @@ def main(self): self.__intelligence.request_data(ip) # TODO: the code below exists for testing purposes for - # tests/integration_tests/test_fides.py + # tests/integration/test_fides/test_fides.py if msg := self.get_msg("fides2network"): pass diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000000..ac91ed8e2d --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only +import importlib.util +import pytest + + +if importlib.util.find_spec("termcolor") is None: + pytest.skip( + "termcolor is required to run integration tests that invoke slips", + allow_module_level=True, + ) diff --git a/tests/integration/test_config_files/__init__.py b/tests/integration/test_config_files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/config/test.yaml b/tests/integration/test_config_files/config_test.yaml similarity index 100% rename from tests/integration/config/test.yaml rename to tests/integration/test_config_files/config_test.yaml diff --git a/tests/integration/test.yaml b/tests/integration/test_config_files/test.yaml similarity index 100% rename from tests/integration/test.yaml rename to tests/integration/test_config_files/test.yaml diff --git a/tests/integration/test2.yaml b/tests/integration/test_config_files/test2.yaml similarity index 100% rename from tests/integration/test2.yaml rename to tests/integration/test_config_files/test2.yaml diff --git a/tests/integration/test_config_files.py b/tests/integration/test_config_files/test_config_files.py similarity index 91% rename from tests/integration/test_config_files.py rename to tests/integration/test_config_files/test_config_files.py index fc1da3c6c4..83fc3fbbc0 100644 --- a/tests/integration/test_config_files.py +++ b/tests/integration/test_config_files/test_config_files.py @@ -5,8 +5,6 @@ test/test.yaml and tests/test2.yaml """ -from slips.main import Main -from slips_files.common.input_type import InputType from tests.common_test_utils import ( is_evidence_present, create_output_dir, @@ -22,17 +20,10 @@ import pytest import shutil import os +from pathlib import Path alerts_file = "alerts.log" - - -def create_main_instance(input_information): - """returns an instance of Main() class in slips.py""" - main = Main(testing=True) - main.input_information = input_information - main.input_type = InputType.PCAP - main.line_type = False - return main +TEST_DIR = Path(__file__).resolve().parent @pytest.mark.parametrize( @@ -55,10 +46,10 @@ def test_conf_file(pcap_path, expected_profiles, output_dir, redis_port): binaries=("redis-server",), require_zeek_or_bro=True, ) - config_file = "tests/integration/test.yaml" + config_file = TEST_DIR / "test.yaml" modify_yaml_config( - output_filename=config_file, - output_dir=os.getcwd(), + output_filename=config_file.name, + output_dir=TEST_DIR, changes={ "DisabledAlerts": { "disabled_detections": ["ConnectionWithoutDNS"] @@ -157,10 +148,10 @@ def test_conf_file2(pcap_path, expected_profiles, output_dir, redis_port): binaries=("redis-server",), require_zeek_or_bro=True, ) - config_file = "tests/integration/test2.yaml" + config_file = TEST_DIR / "test2.yaml" modify_yaml_config( - output_filename=config_file, - output_dir=os.getcwd(), + output_filename=config_file.name, + output_dir=TEST_DIR, changes={ "detection": {"evidence_detection_threshold": 0.1}, "parameters": { diff --git a/tests/integration/test_dataset/__init__.py b/tests/integration/test_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset/test_dataset.py similarity index 100% rename from tests/integration/test_dataset.py rename to tests/integration/test_dataset/test_dataset.py diff --git a/tests/integration/test_fides/__init__.py b/tests/integration/test_fides/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/config/fides.conf.yml b/tests/integration/test_fides/fides.conf.yml similarity index 100% rename from tests/integration/config/fides.conf.yml rename to tests/integration/test_fides/fides.conf.yml diff --git a/tests/integration/config/fides_config.yaml b/tests/integration/test_fides/fides_config.yaml similarity index 100% rename from tests/integration/config/fides_config.yaml rename to tests/integration/test_fides/fides_config.yaml diff --git a/tests/integration/test_fides.py b/tests/integration/test_fides/test_fides.py similarity index 98% rename from tests/integration/test_fides.py rename to tests/integration/test_fides/test_fides.py index 242db07289..4eeeab3f3b 100644 --- a/tests/integration/test_fides.py +++ b/tests/integration/test_fides/test_fides.py @@ -30,6 +30,7 @@ alerts_file = "alerts.log" +TEST_DIR = Path(__file__).resolve().parent def ensure_redis_is_running(port): @@ -166,8 +167,7 @@ def get_main_interface(): ) def test_conf_file2(path, output_dir, redis_port): """ - In this test we're using tests/integration/config/fides_config.yaml as fides - config file + In this test we're using the local fides integration config file. """ ensure_redis_is_running(redis_port) output_dir: PosixPath = create_output_dir(output_dir) @@ -185,7 +185,7 @@ def test_conf_file2(path, output_dir, redis_port): "-o", str(output_dir), "-c", - "tests/integration/config/fides_config.yaml", + str(TEST_DIR / "fides_config.yaml"), "-P", str(redis_port), ] @@ -277,7 +277,7 @@ def test_trust_recommendation_response(path, output_dir, redis_port): "-o", str(output_dir), "-c", - "tests/integration/config/fides_config.yaml", + str(TEST_DIR / "fides_config.yaml"), "-P", str(redis_port), ] diff --git a/tests/integration/test_iris/__init__.py b/tests/integration/test_iris/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_pcap_dataset/__init__.py b/tests/integration/test_pcap_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_pcap_dataset.py b/tests/integration/test_pcap_dataset/test_pcap_dataset.py similarity index 100% rename from tests/integration/test_pcap_dataset.py rename to tests/integration/test_pcap_dataset/test_pcap_dataset.py diff --git a/tests/integration/test_portscans/__init__.py b/tests/integration/test_portscans/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_portscans.py b/tests/integration/test_portscans/test_portscans.py similarity index 100% rename from tests/integration/test_portscans.py rename to tests/integration/test_portscans/test_portscans.py diff --git a/tests/integration/test_zeek_dataset/__init__.py b/tests/integration/test_zeek_dataset/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/test_zeek_dataset.py b/tests/integration/test_zeek_dataset/test_zeek_dataset.py similarity index 100% rename from tests/integration/test_zeek_dataset.py rename to tests/integration/test_zeek_dataset/test_zeek_dataset.py diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 73fc26e829..1d64bca694 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -22,9 +22,9 @@ printf "0" | ./slips.py -k # command before running the dataset tests # distribute on 3 workers only because every worker will be spawning 10+ processes -python3 -m pytest -s tests/integration/test_portscans.py -p no:warnings -vv -python3 -m pytest -s tests/integration/test_dataset.py -p no:warnings -vv -python3 -m pytest -s tests/integration/test_config_files.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_portscans/test_portscans.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_dataset/test_dataset.py -p no:warnings -vv +python3 -m pytest -s tests/integration/test_config_files/test_config_files.py -p no:warnings -vv printf "0" | ./slips.py -k From d48e6e697f4b64774699bcdc8c0f2ceec79046c9 Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 25 Apr 2026 10:39:55 +0300 Subject: [PATCH 50/51] change the names of iris and slips config files used in test_iris() --- config/iris_config.yaml | 4 +- docs/iris_module.md | 2 +- .../integration/config/iris_peer_config.yaml | 15 - tests/integration/test_iris.py | 345 ------------------ .../test_iris/peer1_config/iris.yaml | 13 + .../peer1_config/slips.yaml} | 2 +- .../peer2_config/iris.yaml} | 17 +- .../test_iris/peer2_config/slips.yaml | 142 +++++++ 8 files changed, 168 insertions(+), 372 deletions(-) delete mode 100644 tests/integration/config/iris_peer_config.yaml delete mode 100644 tests/integration/test_iris.py create mode 100644 tests/integration/test_iris/peer1_config/iris.yaml rename tests/integration/{config/slips_iris_main.yaml => test_iris/peer1_config/slips.yaml} (98%) rename tests/integration/{config/iris_config.yaml => test_iris/peer2_config/iris.yaml} (86%) create mode 100644 tests/integration/test_iris/peer2_config/slips.yaml diff --git a/config/iris_config.yaml b/config/iris_config.yaml index 76625208fc..2e5e1b6111 100644 --- a/config/iris_config.yaml +++ b/config/iris_config.yaml @@ -2,11 +2,11 @@ Identity: GenerateNewKey: true Server: Port: 9010 - Host: 0.0.0.0 + Host: null DhtServerMode: 'true' Redis: Host: 127.0.0.1 - Port: 6379 + Port: 6644 Tl2NlChannel: iris_internal PeerDiscovery: DisableBootstrappingNodes: false diff --git a/docs/iris_module.md b/docs/iris_module.md index dc15dd30db..ab2308f16e 100644 --- a/docs/iris_module.md +++ b/docs/iris_module.md @@ -98,7 +98,7 @@ will be left upon the future developers. * ```go test ./...``` ### Integration Testing -Integration tests are located in ```tests/integration/test_iris.py```. +Integration tests are located in ```tests/integration/test_iris/test_iris.py```. ### Test Messaging The scenario that was modeled in this test refers to a common use case. diff --git a/tests/integration/config/iris_peer_config.yaml b/tests/integration/config/iris_peer_config.yaml deleted file mode 100644 index d1bf3e4777..0000000000 --- a/tests/integration/config/iris_peer_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -Identity: - GenerateNewKey: true - KeyFile: second.priv -PeerDiscovery: - DisableBootstrappingNodes: false - ListOfMultiAddresses: - - /ip4/127.0.0.1/udp/9010/quic 12D3KooWNKe9xFM85HkWsoGAy4pM3kXrZHGumJwQBBr9T4VzHtNR -Redis: - Host: 127.0.0.1 - Port: 6655 - Tl2NlChannel: iris_internal -Server: - DhtServerMode: 'true' - Host: null - Port: 9006 diff --git a/tests/integration/test_iris.py b/tests/integration/test_iris.py deleted file mode 100644 index 332cfc2f0e..0000000000 --- a/tests/integration/test_iris.py +++ /dev/null @@ -1,345 +0,0 @@ -""" -This file tests 2 different config files other than slips' default config/slips.yaml -test/test.yaml and tests/test2.yaml -""" - -import re -import shutil -from pathlib import PosixPath - -import redis - -from tests.common_test_utils import ( - create_output_dir, - assert_no_errors, - modify_yaml_config, -) -import pytest -import os -import subprocess -import time -import sys - -alerts_file = "alerts.log" - - -def countdown(seconds, message): - """ - counts down from the given number of seconds, printing a message each second. - """ - while seconds > 0: - sys.stdout.write( - f"\rSending {message} in {seconds} " - ) # overwrite the line - sys.stdout.flush() # ensures immediate output - time.sleep(1) # waits for 1 second - seconds -= 1 - sys.stdout.write(f"\rSending {message} now! \n") - - -def message_send(port, channel, message): - # connect to redis database 0 - redis_client = redis.StrictRedis(host="localhost", port=port, db=0) - - # publish the message to the "network2fides" channel - redis_client.publish(channel, message) - - print(f"Test message published to channel '{channel}'.") - - -message_alert_TL_NL = """{ - "type": "tl2nl_alert", - "version": 1, - "data": { - "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - } -}""" - - -message_alert_NL_S = """{ - "type": "nl2tl_alert", - "version": 1, - "data": - "sender": "" - "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" -}""" - - -def check_strings_in_file(string_list, file_path): - # Check if the file exists - if not os.path.exists(file_path): - print(f"File {file_path} does not exist.") - return False - - # Open the file and read its content - try: - with open(file_path, "r") as file: - file_content = file.read() - - # Check if all strings in the list are present in the file content - for string in string_list: - if string not in file_content: - return False - return True - - except Exception as e: - print(f"Error reading file: {e}") - return False - - -def wait_for_file(file_path, timeout_seconds): - """ - Wait until a file exists or the timeout elapses. - - Parameters: - file_path: Path to the expected file. - timeout_seconds: Maximum number of seconds to wait. - - Returns: - True if the file exists before the timeout, otherwise False. - """ - deadline = time.time() + timeout_seconds - while time.time() < deadline: - if os.path.exists(file_path): - return True - time.sleep(1) - return os.path.exists(file_path) - - -def get_default_interface(): - with open("/proc/net/route") as f: - for line in f.readlines()[1:]: - fields = line.strip().split() - if fields[1] == "00000000": # default route - return fields[0] - - -@pytest.mark.parametrize( - "zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port", - [ - ( - "dataset/test13-malicious-dhcpscan-zeek-dir", - "iris_integration_test/", - "peer_iris_integration_test/", - 6644, - 6655, - ) - ], -) -def test_messaging( - zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port -): - """ - Tests whether Iris properly distributes an alert message generated by - Slips to the network (~other peers). - - First Slips instance is a general node in the network, its connection - string is generated and extracted from logs as a normal user would do, - in a very standard use case. - - The second instance of Slips acts as a normal-user-peer that joins the - network via the aforementioned Slips instance, - which extends the standard use case of connecting to such P2P network. - """ - # Two Slips instances are necessary to be run in this test. - default_interface = get_default_interface() - - # Prepare output dir for the main Slips instance. - # The logs of both beers will be clearly separated and kept intact. - output_dir: PosixPath = create_output_dir(output_dir) - output_file = os.path.join(output_dir, "slips_output.txt") - - # Prepare output dir for the main Slips instance. - # The logs of both beers will be clearly separated and kept intact. - output_dir_peer: PosixPath = create_output_dir(peer_output_dir) - output_file_peer = os.path.join(output_dir_peer, "slips_output.txt") - - # this will be used for the extraction of the connection string form the - # logs of iris running under the main Slips - log_file_first_iris = output_dir / "iris/iris_logs.txt" - log_file_second_iris = output_dir_peer / "iris/iris_logs.txt" - - # generate config of first peer - slips_iris_main_config_file = ( - "tests/integration/config/slips_iris_main.yaml" - ) - modify_yaml_config( - input_path="config/slips.yaml", - output_dir=os.path.dirname(slips_iris_main_config_file), - output_filename=os.path.basename(slips_iris_main_config_file), - changes={ - "global_p2p": {"use_global_p2p": True}, - "modules": {"disable": ["template", "updatemanager"]}, - }, - ) - - # that config file will be generated later to be able to add the first - # peer's id to it - iris_peer_config_file = "tests/integration/config/iris_peer_config.yaml" - - # generate config of second peer - iris_config_file = "tests/integration/config/iris_config.yaml" - modify_yaml_config( - input_path="config/slips.yaml", - output_dir=os.path.dirname(iris_config_file), - output_filename=os.path.basename(iris_config_file), - changes={ - "global_p2p": { - "use_global_p2p": True, - "iris_conf": iris_peer_config_file, - }, - "modules": {"disable": ["template", "updatemanager"]}, - }, - ) - - print("running slips ...") - with open(output_file, "w") as log_file: - with open(output_file_peer, "w") as iris_log_file: - # Start the subprocess, redirecting stdout and stderr - # to the same file - # command for the main Slips instance - command = [ - sys.executable, - "./slips.py", - "-t", - "-g", - str(zeek_dir_path), - # dummy interface required by -g - "-i", - default_interface, - "-e", - "1", - "-o", - str(output_dir), - "-c", - slips_iris_main_config_file, # we're using the dafult peer - # config here - "-P", - str(redis_port), - ] - process = subprocess.Popen( - command, - stdout=log_file, - stderr=log_file, - ) - - # First peer (its Iris) needs to be ready and available for - # connections when the second peer tries to reach out to it. - countdown(20, "second peer") - # get the connection string from the first peer and give it - # to the second one so it is reachable - assert wait_for_file( - log_file_first_iris, 30 - ), f"Expected Iris log file was not created: {log_file_first_iris}" - with open(log_file_first_iris, "r") as log: - for line in log: - match = re.search(r"connection string:\s+'(.+)'", line) - if match: - original_conn_string = match.group(1) - break - else: - # if it comes here make sure that port 9010 used by - # iris is free - # sudo lsof -i :9010 - # sudo kill -9 - print("No connection string found in log file.") - exit(1) - - # put the PeerID in the config file of the second peer's Iris - # the goal is for the second iris to be able to find the first - # iris/slips - modify_yaml_config( - input_path="config/iris_config.yaml", - output_dir=os.path.dirname(iris_peer_config_file), - output_filename=os.path.basename(iris_peer_config_file), - changes={ - "Redis": {"Port": 6655}, - "Server": {"Port": 9006}, - "PeerDiscovery": { - "ListOfMultiAddresses": [original_conn_string] - }, - "Identity": {"KeyFile": "second.priv"}, - }, - ) - # generate a second command for the second peer - peer_command = [ - sys.executable, - "./slips.py", - "-t", - "-g", - str(zeek_dir_path), - # dummy interface required by -g - "-i", - default_interface, - "-e", - "1", - "-o", - str(output_dir_peer), - "-c", - iris_config_file, # we're not using the dafult peer config - # here - "-P", - str(peer_redis_port), - ] - peer_process = subprocess.Popen( - peer_command, stdout=iris_log_file, stderr=iris_log_file - ) - - print( - f"Output and errors of first peer are logged in" - f" {output_file}" - ) - - # let Slips properly and fully star with all of its parts and modules. - countdown(80, "Sending msg in fides2network") - # Sending a manual message to make sure there is an alert generated, because - # is is highly probable that both slips have covered their network captures - # before the infrastructure of P2P network was fully up and running - message_send( - redis_port, - message=message_alert_TL_NL, - channel="fides2network", - ) - - # these seconds are the time we give slips to process the msg - countdown(30, "Sending SIGTERM to the 2 peers") - # Kill em with kindness. - os.kill(process.pid, 15) - os.kill(peer_process.pid, 15) - print("SIGTERM sent.") - - print("Sending SIGKILL to the 2 instances of Slips + iris") - # Kill em. Without kindness. - os.kill(process.pid, 9) - print(f"Slips with PID {process.pid} was killed.") - - os.kill(peer_process.pid, 9) - print(f"Slips peer with PID {peer_process.pid} was killed.") - - print("Slips is done, checking for errors in the 2 output dirs.") - assert_no_errors(output_dir) - assert_no_errors(output_dir_peer) - - print("Checking for iris expected logs in the generated log file") - # make sure this string is there in the generated iris logs - # this is how we ensure that iris ran correctly - expected_log_entry = [ - "INFO iris protocols/alert.go:111 received p2p alert message" - ] - assert check_strings_in_file(expected_log_entry, log_file_second_iris) - - print("Deleting the output directories") - shutil.rmtree(output_dir) - shutil.rmtree(output_dir_peer) - os.remove("modules/iris/second.priv") - modify_yaml_config( - input_path="config/iris_config.yaml", - output_dir=os.path.dirname(iris_peer_config_file), - output_filename=os.path.basename(iris_peer_config_file), - changes={ - "Redis": {"Port": 6644}, - "Server": {"Port": 9010}, - "PeerDiscovery": {}, - "Identity": {"KeyFile": "private.key"}, - }, - ) diff --git a/tests/integration/test_iris/peer1_config/iris.yaml b/tests/integration/test_iris/peer1_config/iris.yaml new file mode 100644 index 0000000000..e623c04475 --- /dev/null +++ b/tests/integration/test_iris/peer1_config/iris.yaml @@ -0,0 +1,13 @@ +Identity: + GenerateNewKey: true +PeerDiscovery: + DisableBootstrappingNodes: true + ListOfMultiAddresses: [] +Redis: + Host: 127.0.0.1 + Port: 6644 + Tl2NlChannel: iris_internal +Server: + DhtServerMode: 'true' + Host: null + Port: 9010 diff --git a/tests/integration/config/slips_iris_main.yaml b/tests/integration/test_iris/peer1_config/slips.yaml similarity index 98% rename from tests/integration/config/slips_iris_main.yaml rename to tests/integration/test_iris/peer1_config/slips.yaml index 6bc3dcbd22..81977b53a2 100644 --- a/tests/integration/config/slips_iris_main.yaml +++ b/tests/integration/test_iris/peer1_config/slips.yaml @@ -79,7 +79,7 @@ global_p2p: - fides - iris bootstrapping_node: false - iris_conf: config/iris_config.yaml + iris_conf: tests/integration/test_iris/peer1_config/iris.yaml use_global_p2p: true local_p2p: create_p2p_logfile: false diff --git a/tests/integration/config/iris_config.yaml b/tests/integration/test_iris/peer2_config/iris.yaml similarity index 86% rename from tests/integration/config/iris_config.yaml rename to tests/integration/test_iris/peer2_config/iris.yaml index 76625208fc..04dbe81985 100644 --- a/tests/integration/config/iris_config.yaml +++ b/tests/integration/test_iris/peer2_config/iris.yaml @@ -1,14 +1,15 @@ Identity: GenerateNewKey: true -Server: - Port: 9010 - Host: 0.0.0.0 - DhtServerMode: 'true' -Redis: - Host: 127.0.0.1 - Port: 6379 - Tl2NlChannel: iris_internal + KeyFile: private.key PeerDiscovery: DisableBootstrappingNodes: false ListOfMultiAddresses: - /dns/melchior.slips.stratosphere.fel.cvut.cz/udp/6437/quic 12D3KooWJJa9PpMFVP7s3TQs2sedypJXxtMVkphRhgkjGH9EYMfM +Redis: + Host: 127.0.0.1 + Port: 6644 + Tl2NlChannel: iris_internal +Server: + DhtServerMode: 'true' + Host: null + Port: 9010 diff --git a/tests/integration/test_iris/peer2_config/slips.yaml b/tests/integration/test_iris/peer2_config/slips.yaml new file mode 100644 index 0000000000..0a03b12de7 --- /dev/null +++ b/tests/integration/test_iris/peer2_config/slips.yaml @@ -0,0 +1,142 @@ +Debug: + generate_performance_plots: false +DisabledAlerts: + disabled_detections: [] +Docker: + GID: 0 + UID: 0 +Profiling: + cpu_profiler_dev_mode_entries: 500000 + cpu_profiler_enable: false + cpu_profiler_mode: dev + cpu_profiler_multiprocess: true + cpu_profiler_output_limit: 20 + cpu_profiler_sampling_interval: 20 + memory_profiler_enable: false + memory_profiler_mode: live + memory_profiler_multiprocess: true +anomaly_detection_https: + adaptation_score_threshold: 2.0 + adwin_clock: 1 + adwin_delta: 0.01 + adwin_grace_period: 5 + adwin_min_window_length: 5 + baseline_alpha: 0.5 + drift_alpha: 0.05 + empirical_threshold_quantile: 0.995 + flow_zscore_threshold: 3.5 + hourly_zscore_threshold: 3.0 + ja3_min_variants_per_server: 3 + log_verbosity: 3 + max_small_flow_anomalies: 1 + min_baseline_points: 6 + suspicious_alpha: 0.005 + training_alpha: 1 + training_fit_method: welford + training_hours: 2 + use_adwin_drift: true +brute_force_detector: + ssh_attempt_threshold: 9 +cesnet: + configuration_file: config/warden.conf + receive_alerts: false + receive_delay: 86400 + send_alerts: false +detection: + evidence_detection_threshold: 0.25 + popup_alerts: false +exporting_alerts: + TAXII_server: localhost + collection_name: Alerts + direct_export: true + direct_export_max_workers: 12 + direct_export_retry_backoff: 0.5 + direct_export_retry_max: 0 + direct_export_retry_max_delay: 5.0 + direct_export_workers: 4 + discovery_path: /taxii2/ + export_to: [] + port: 1234 + push_delay: 3600 + sensor_name: sensor1 + slack_api_path: config/slack_bot_token_secret + slack_channel_name: proj_slips_alerting_module + taxii_password: changeme_before_installing_a_medallion_server + taxii_timeout: 10 + taxii_username: admin + taxii_version: 2 + use_https: false +flow_alerts: + data_exfiltration_threshold: 500 + entropy_threshold: 5 + long_connection_threshold: 1500 + pastebin_download_threshold: 700 + ssh_succesful_detection_threshold: 4290 +flow_ml_detection: + mode: test +global_p2p: + bootstrapping_modules: + - fides + - iris + bootstrapping_node: false + iris_conf: tests/integration/test_iris/peer2_config/iris.yaml + use_global_p2p: true +local_p2p: + create_p2p_logfile: false + use_p2p: false +modules: + disable: + - template + - updatemanager + timeline_human_timestamp: true +output: + logs: slips.log + stderr: errors.log + stdout: slips.log +parameters: + analysis_direction: out + client_ips: [] + debug: 0 + default_rotation_interval: 1day + deletePrevdb: true + delete_zeek_files: false + export_format: json + export_labeled_flows: false + export_strato_letters: false + keep_rotated_files_for: 1 day + label: normal + metadata_dir: true + pcapfilter: false + permanent_dir: permanent + rotation: true + store_a_copy_of_zeek_files: false + store_zeek_files_in_the_output_dir: true + tcp_inactivity_timeout: 60 + time_window_width: 3600 + verbose: 1 + wait_for_modules_to_finish: 10080 mins +threatintelligence: + RiskIQ_credentials_path: config/RiskIQ_credentials + TI_files_update_period: 86400 + download_path_for_remote_threat_intelligence: modules/threat_intelligence/remote_data_files/ + ja3_feeds: config/JA3_feeds.csv + local_threat_intelligence_files: config/local_ti_files/ + mac_db: https://maclookup.app/downloads/json-database/get-db + mac_db_update: 1209600 + riskiq_update_period: 604800 + ssl_feeds: config/SSL_feeds.csv + ti_files: config/TI_feeds.csv + wait_for_TI_to_finish: false +update: + auto_update_slips: false +virustotal: + api_key_file: config/vt_api_key + virustotal_update_period: 259200 +web_interface: + port: 55000 +whitelists: + enable_local_whitelist: true + enable_online_whitelist: true + local_whitelist_path: config/whitelist.conf + online_whitelist: https://tranco-list.eu/download/X5QNN/10000 + online_whitelist_update_period: 86400 From 75bacdc64b894ca5beba27218a99707670f0440e Mon Sep 17 00:00:00 2001 From: alya Date: Sat, 25 Apr 2026 10:40:48 +0300 Subject: [PATCH 51/51] test_iris(): split peer1 and peer2 handling into separate funcs --- tests/integration/test_iris/test_iris.py | 528 +++++++++++++++++++++++ 1 file changed, 528 insertions(+) create mode 100644 tests/integration/test_iris/test_iris.py diff --git a/tests/integration/test_iris/test_iris.py b/tests/integration/test_iris/test_iris.py new file mode 100644 index 0000000000..2ba978b4c9 --- /dev/null +++ b/tests/integration/test_iris/test_iris.py @@ -0,0 +1,528 @@ +""" +This file tests 2 different config files other than slips' default config/slips.yaml +test/test.yaml and tests/test2.yaml +""" + +import re +import shutil +from pathlib import PosixPath + +import redis + +from tests.common_test_utils import ( + create_output_dir, + assert_no_errors, + modify_yaml_config, +) +import pytest +import os +import subprocess +import time +import sys +from pathlib import Path + +alerts_file = "alerts.log" +TEST_DIR = Path(__file__).resolve().parent +PEER1_CONFIG_DIR = TEST_DIR / "peer1_config" +PEER2_CONFIG_DIR = TEST_DIR / "peer2_config" + + +def countdown(seconds, message): + """ + counts down from the given number of seconds, printing a message each second. + """ + while seconds > 0: + sys.stdout.write( + f"\rSending {message} in {seconds} " + ) # overwrite the line + sys.stdout.flush() # ensures immediate output + time.sleep(1) # waits for 1 second + seconds -= 1 + sys.stdout.write(f"\rSending {message} now! \n") + + +def message_send(port, channel, message): + # connect to redis database 0 + redis_client = redis.StrictRedis(host="localhost", port=port, db=0) + + # publish the message to the "network2fides" channel + redis_client.publish(channel, message) + + print(f"Test message published to channel '{channel}'.") + + +message_alert_TL_NL = """{ + "type": "tl2nl_alert", + "version": 1, + "data": { + "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + } +}""" + + +message_alert_NL_S = """{ + "type": "nl2tl_alert", + "version": 1, + "data": + "sender": "" + "payload": "https://www.youtube.com/watch?v=dQw4w9WgXcQ" +}""" + + +def check_strings_in_file(string_list, file_path): + # Check if the file exists + if not os.path.exists(file_path): + print(f"File {file_path} does not exist.") + return False + + # Open the file and read its content + try: + with open(file_path, "r") as file: + file_content = file.read() + + # Check if all strings in the list are present in the file content + for string in string_list: + if string not in file_content: + return False + return True + + except Exception as e: + print(f"Error reading file: {e}") + return False + + +def wait_for_file(file_path, timeout_seconds): + """ + Wait until a file exists or the timeout elapses. + + Parameters: + file_path: Path to the expected file. + timeout_seconds: Maximum number of seconds to wait. + + Returns: + True if the file exists before the timeout, otherwise False. + """ + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if os.path.exists(file_path): + return True + time.sleep(1) + return os.path.exists(file_path) + + +def get_default_interface(): + """ + Get the default network interface. + + Returns: + str: Name of the default network interface. + """ + with open("/proc/net/route") as f: + for line in f.readlines()[1:]: + fields = line.strip().split() + if fields[1] == "00000000": # default route + return fields[0] + + +def extract_connection_string(log_file_first_iris): + """ + Extract the first peer connection string from the Iris log file. + + Parameters: + log_file_first_iris: Path to the first peer Iris log file. + + Returns: + str: The extracted connection string. + """ + with open(log_file_first_iris, "r") as log: + for line in log: + match = re.search(r"connection string:\s+'(.+)'", line) + if match: + return match.group(1) + + print("No connection string found in log file.") + exit(1) + + +def assert_peer1_setup(peer1_slips_config): + """ + Assert that the first peer config was generated as expected. + + Parameters: + peer1_slips_config: Path to the first peer Slips config. + + Returns: + None + """ + assert check_strings_in_file( + ["iris_conf: tests/integration/test_iris/peer1_config/iris.yaml"], + peer1_slips_config, + ) + + +def assert_peer2_setup( + peer2_slips_config, + peer2_iris_config, + connection_string, + peer_redis_port, +): + """ + Assert that the second peer config was generated as expected. + + Parameters: + peer2_slips_config: Path to the second peer Slips config. + peer2_iris_config: Path to the second peer Iris config. + connection_string: Multiaddress used to connect to the first peer. + peer_redis_port: Redis port used by the second peer. + + Returns: + None + """ + assert check_strings_in_file( + ["iris_conf: tests/integration/test_iris/peer2_config/iris.yaml"], + peer2_slips_config, + ) + assert check_strings_in_file( + [ + f"Port: {peer_redis_port}", + "Port: 9006", + "DisableBootstrappingNodes: true", + "KeyFile: second.priv", + connection_string, + ], + peer2_iris_config, + ) + + +def assert_peer1_results(output_dir): + """ + Assert the first peer completed without runtime errors. + + Parameters: + output_dir: Output directory of the first peer. + + Returns: + None + """ + assert_no_errors(output_dir) + + +def assert_peer2_results(output_dir_peer, log_file_second_iris): + """ + Assert the second peer completed without runtime errors and started Iris. + + Parameters: + output_dir_peer: Output directory of the second peer. + log_file_second_iris: Path to the second peer Iris log file. + + Returns: + None + """ + assert_no_errors(output_dir_peer) + assert check_strings_in_file( + ["connection string:"], + log_file_second_iris, + ) + + +def prepare_and_start_peer1( + zeek_dir_path, + output_dir, + redis_port, + default_interface, + log_file, +): + """ + Generate config for peer1, start it, and return startup metadata. + + Parameters: + zeek_dir_path: Zeek dataset path used by the test. + output_dir: Output directory for peer1. + redis_port: Redis port used by peer1. + default_interface: Interface required by Slips when using `-g`. + log_file: Open file handle used to capture peer1 output. + + Returns: + tuple: Peer1 process, Iris log path, and Slips config path. + """ + peer1_slips_config = PEER1_CONFIG_DIR / "slips.yaml" + peer1_iris_config = PEER1_CONFIG_DIR / "iris.yaml" + peer1_iris_config_path = Path( + "tests/integration/test_iris/peer1_config" + ) / (peer1_iris_config.name) + + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER1_CONFIG_DIR, + output_filename=peer1_iris_config.name, + changes={ + "Redis": {"Port": redis_port}, + "PeerDiscovery": { + "DisableBootstrappingNodes": True, + "ListOfMultiAddresses": [], + }, + }, + ) + modify_yaml_config( + input_path="config/slips.yaml", + output_dir=PEER1_CONFIG_DIR, + output_filename=peer1_slips_config.name, + changes={ + "global_p2p": { + "use_global_p2p": True, + "iris_conf": str(peer1_iris_config_path), + }, + "modules": {"disable": ["template", "updatemanager"]}, + }, + ) + + command = [ + sys.executable, + "./slips.py", + "-t", + "-g", + str(zeek_dir_path), + "-i", + default_interface, + "-e", + "1", + "-o", + str(output_dir), + "-c", + str(peer1_slips_config), + "-P", + str(redis_port), + ] + print(f"@@@@@@@@@@@@@@@@ {' '.join(command)}") + + process = subprocess.Popen( + command, + stdout=log_file, + stderr=log_file, + ) + log_file_first_iris = output_dir / "iris/iris_logs.txt" + return process, log_file_first_iris, peer1_slips_config + + +def prepare_and_start_peer2( + zeek_dir_path, + output_dir_peer, + peer_redis_port, + default_interface, + connection_string, + log_file, +): + """ + Generate config for peer2, start it, and return startup metadata. + + Parameters: + zeek_dir_path: Zeek dataset path used by the test. + output_dir_peer: Output directory for peer2. + peer_redis_port: Redis port used by peer2. + default_interface: Interface required by Slips when using `-g`. + connection_string: Multiaddress of peer1 used by peer2. + log_file: Open file handle used to capture peer2 output. + + Returns: + tuple: Peer2 process, Iris log path, Slips config path, and Iris config path. + """ + peer2_iris_config = PEER2_CONFIG_DIR / "iris.yaml" + peer2_iris_config_path = Path( + "tests/integration/test_iris/peer2_config" + ) / (peer2_iris_config.name) + peer2_slips_config = PEER2_CONFIG_DIR / "slips.yaml" + + modify_yaml_config( + input_path="config/slips.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_slips_config.name, + changes={ + "global_p2p": { + "use_global_p2p": True, + "iris_conf": str(peer2_iris_config_path), + }, + "modules": {"disable": ["template", "updatemanager"]}, + }, + ) + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_iris_config.name, + changes={ + "Redis": {"Port": peer_redis_port}, + "Server": {"Port": 9006}, + "PeerDiscovery": { + "DisableBootstrappingNodes": True, + "ListOfMultiAddresses": [connection_string], + }, + "Identity": {"KeyFile": "second.priv"}, + }, + ) + + peer_command = [ + sys.executable, + "./slips.py", + "-t", + "-g", + str(zeek_dir_path), + "-i", + default_interface, + "-e", + "1", + "-o", + str(output_dir_peer), + "-c", + str(peer2_slips_config), + "-P", + str(peer_redis_port), + ] + peer_process = subprocess.Popen( + peer_command, stdout=log_file, stderr=log_file + ) + log_file_second_iris = output_dir_peer / "iris/iris_logs.txt" + return ( + peer_process, + log_file_second_iris, + peer2_slips_config, + peer2_iris_config, + ) + + +@pytest.mark.parametrize( + "zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port", + [ + ( + "dataset/test13-malicious-dhcpscan-zeek-dir", + "iris_integration_test/", + "peer_iris_integration_test/", + 6644, + 6655, + ) + ], +) +def test_messaging( + zeek_dir_path, output_dir, peer_output_dir, redis_port, peer_redis_port +): + """ + Tests whether Iris properly distributes an alert message generated by + Slips to the network (~other peers). + + First Slips instance is a general node in the network, its connection + string is generated and extracted from logs as a normal user would do, + in a very standard use case. + + The second instance of Slips acts as a normal-user-peer that joins the + network via the aforementioned Slips instance, + which extends the standard use case of connecting to such P2P network. + """ + default_interface = get_default_interface() + + # Two Slips instances are necessary to be run in this test. + + # Prepare output dir for peer1 + output_dir: PosixPath = create_output_dir(output_dir) + output_file = os.path.join(output_dir, "slips_output.txt") + + # Prepare output dir for the peer2 + output_dir_peer: PosixPath = create_output_dir(peer_output_dir) + output_file_peer = os.path.join(output_dir_peer, "slips_output.txt") + + print("running slips ...") + with open(output_file, "w") as log_file: + with open(output_file_peer, "w") as iris_log_file: + process, log_file_first_iris, peer1_slips_config = ( + prepare_and_start_peer1( + zeek_dir_path=zeek_dir_path, + output_dir=output_dir, + redis_port=redis_port, + default_interface=default_interface, + log_file=log_file, + ) + ) + assert_peer1_setup(peer1_slips_config) + + # First peer (its Iris) needs to be ready and available for + # connections when the second peer tries to reach out to it. + countdown(20, "second peer") + # get the connection string from the first peer and give it + # to the second one so it is reachable + assert wait_for_file( + log_file_first_iris, 30 + ), f"Expected Iris log file was not created: {log_file_first_iris}" + original_conn_string = extract_connection_string( + log_file_first_iris + ) + + ( + peer_process, + log_file_second_iris, + peer2_slips_config, + peer2_iris_config, + ) = prepare_and_start_peer2( + zeek_dir_path=zeek_dir_path, + output_dir_peer=output_dir_peer, + peer_redis_port=peer_redis_port, + default_interface=default_interface, + connection_string=original_conn_string, + log_file=iris_log_file, + ) + assert_peer2_setup( + peer2_slips_config=peer2_slips_config, + peer2_iris_config=peer2_iris_config, + connection_string=original_conn_string, + peer_redis_port=peer_redis_port, + ) + + print( + f"Output and errors of first peer are logged in" + f" {output_file}" + ) + + # let Slips properly and fully star with all of its parts and modules. + countdown(80, "Sending msg in fides2network") + # Sending a manual message to make sure there is an alert generated, because + # is is highly probable that both slips have covered their network captures + # before the infrastructure of P2P network was fully up and running + message_send( + redis_port, + message=message_alert_TL_NL, + channel="fides2network", + ) + + # these seconds are the time we give slips to process the msg + countdown(30, "Sending SIGTERM to the 2 peers") + # Kill em with kindness. + os.kill(process.pid, 15) + os.kill(peer_process.pid, 15) + print("SIGTERM sent.") + + print("Sending SIGKILL to the 2 instances of Slips + iris") + # Kill em. Without kindness. + os.kill(process.pid, 9) + print(f"Slips with PID {process.pid} was killed.") + + os.kill(peer_process.pid, 9) + print(f"Slips peer with PID {peer_process.pid} was killed.") + + print("Slips is done, checking for errors in the 2 output dirs.") + assert_peer1_results(output_dir) + assert_peer2_results(output_dir_peer, log_file_second_iris) + + print("Deleting the output directories") + shutil.rmtree(output_dir) + shutil.rmtree(output_dir_peer) + os.remove("modules/iris/second.priv") + + # reset the generated peer2 Iris config back to its default values + # after the test finishes. + modify_yaml_config( + input_path="config/iris_config.yaml", + output_dir=PEER2_CONFIG_DIR, + output_filename=peer2_iris_config.name, + changes={ + "Redis": {"Port": 6644}, + "Server": {"Port": 9010}, + "PeerDiscovery": {}, + "Identity": {"KeyFile": "private.key"}, + }, + )