diff --git a/.secrets.baseline b/.secrets.baseline index 78f90d921a..8609229b8b 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -149,7 +149,7 @@ "filename": "config/slips.yaml", "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016", "is_verified": false, - "line_number": 304 + "line_number": 322 } ], "dataset/test14-malicious-zeek-dir/http.log": [ @@ -7176,5 +7176,5 @@ } ] }, - "generated_at": "2026-04-27T14:39:21Z" + "generated_at": "2026-05-04T20:22:30Z" } diff --git a/README.md b/README.md index 9cc2e54bc2..3b870e3a57 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,11 @@ Slips can be run on different platforms, the easiest and most recommended way if # Configuration Slips has a [config/slips.yaml](https://github.com/stratosphereips/StratosphereLinuxIPS/blob/develop/config/slips.yaml) that contains user configurations for different modules and general execution. +Do not edit the default `config/slips.yaml` directly. Create a copy for your local configuration changes and run Slips with `-c`, for example `./slips.py -c config/my_slips.yaml -f dataset/test7-malicious.pcap`. + +The same applies to `config/whitelist.conf`: keep the default file unchanged, create a copy, and set `whitelists.local_whitelist_path` in your copied Slips config file to point to your copied whitelist file. +Slips aborts updating to new versions when there are changes to Slips local config files. + * You can change the timewindow width by modifying the ```time_window_width``` parameter * You can change the analysis direction to ```all``` if you want to see the attacks from and to your computer * You can also specify whether to ```train``` or ```test``` the ML models diff --git a/config/slips.yaml b/config/slips.yaml index ef22d6f1fb..c83ac5618b 100644 --- a/config/slips.yaml +++ b/config/slips.yaml @@ -1,6 +1,11 @@ # This configuration file controls several aspects of the working of Slips. +################ 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 ######################### +## DO NOT MODIFY THIS FILE. EVER. # +## IF YOU NEED TO CHANGE ANYTHING, MAKE A COPY AND USE THAT COPY WITH slips.py -c # +################ 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 ######################### --- + update: # Enable automatic live updates of the installed Slips version. # This setting is separate from the feeds_update_manager module, which only @@ -10,6 +15,19 @@ update: # Instead, create separate config files with different names and use those. auto_update_slips: false + # Select which update channel Slips should auto-update from. + # Available update channels: + # - stable: uses origin/master + # - unstable: uses origin/develop + # - testing: specify your branch name in the testing_branch_to_update_slips_from parameter below. + channel_to_update_slips_from: stable + + # This branch is used only when channel_to_update_slips_from is set to + # testing. + # Must be a remote origin/ ref. + # Example: origin/your_branch_here + testing_branch_to_update_slips_from: origin/your_branch_here + output: # Define the file names for the default output. stdout: slips.log diff --git a/config/whitelist.conf b/config/whitelist.conf index be554887d5..e35f4e51a5 100644 --- a/config/whitelist.conf +++ b/config/whitelist.conf @@ -1,6 +1,12 @@ +;;;;;;;;;;;;;;;; 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 ;;;;;;;;;;;;;;;;;;;;;;;;; +;; DO NOT MODIFY THIS FILE. EVER. ; +;; IF YOU NEED TO CHANGE ANYTHING, MAKE A COPY AND USE THAT COPY WITH slips.py -c ; +;;;;;;;;;;;;;;;; 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 ;;;;;;;;;;;;;;;;;;;;;;;;; + + ; NOTE: -; USER COMMENTS START WITH ; -; COMMENTED OUT WHITELIST LINES START WITH # +; USER COMMENTS START WITH ; (aka any english comment like this one that doesnt contain TI) +; COMMENTED OUT WHITELIST LINES (lines with TI that you don't want slips to use ) START WITH # ; FOR SLIPS TO BE ABLE TO REMOVE THEM FROM THE CACHE DATABASE ; A whitelist of IPs, domains, organisations or mac addresses diff --git a/docs/contributing.md b/docs/contributing.md index 93086233e0..67b119acd2 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -62,6 +62,36 @@ Here's a very simple beginner-level steps on how to create your PR in Slips 11. Open a PR in Slips, remember to set the base branch as develop. 12. List your changes in the PR description +## How to test the auto update functionality + +To test Slips auto update using the `testing` channel, follow these steps: + +1. Create a new temporary branch from the branch you want to test, for example `origin/yourname-autoupdate-test`. +2. Edit [update.json] in your new branch and update the `testing` channel entry: + - increase the `version` value so this branch appears newer than the version currently available + - keep `"backwards_compatible": true` + - keep `"has_new_dependencies": false` + - set the testing branch entry to your new branch name e.g `origin/yourname-autoupdate-test` +3. Commit the `update.json` change in that temporary branch and push the branch to your remote. +4. Check out your original old branch again. Before continuing, make sure there are no merge conflicts and no modified Slips files in this branch. +5. Create a copy of [config/slips.yaml](config/slips.yaml) . +6. In your copied config file, enable auto update and configure the testing channel: + + ```yaml + update: + auto_update_slips: true + channel_to_update_slips_from: testing + testing_branch_to_update_slips_from: origin/ + ``` + +7. Run Slips on an interface and point it to your copied config file using `-c`, for example: + + ```bash + ./slips.py -i -c config/.yaml + ``` + +8. While Slips is running, it should detect the higher version in the `testing` channel and auto update to the branch you pushed in step 3. + ## Rejected PRs diff --git a/docs/features.md b/docs/features.md index 3b2335d155..0e886291c4 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1068,7 +1068,7 @@ to one of the above alerts, slips does not detect it assuming it's a false posit internally by slips. -You can change this behaviour by updating ```config/whitelist.conf```. +You can change this behaviour by copying ```config/whitelist.conf```, updating the copy, and pointing ```whitelists.local_whitelist_path``` in your copied ```slips.yaml``` file to that copied whitelist. ## Ensembling diff --git a/docs/immune/updating_slips.md b/docs/immune/updating_slips.md index c861d8ad7d..3364b02b63 100644 --- a/docs/immune/updating_slips.md +++ b/docs/immune/updating_slips.md @@ -40,7 +40,7 @@ Old Slips running ↓ Check for compatible update ↓ -git pull origin master +git fetch && git checkout ↓ Start new Slips with -u ↓ @@ -61,6 +61,8 @@ Slips checks for updates once per day. This is handled by the UpdateManager, which: - checks whether auto-update is enabled in the config file +- resolves the configured update channel +- resolves the configured testing branch when the channel is `testing` - checks whether a new version exists - checks compatibility before attempting update. @@ -72,10 +74,51 @@ This file includes metadata about the new version such as: - latest version, - backwards compatibility, - whether new dependencies are needed. - +- update branch/channel metadata. The compatibility parser was added so we avoid updating to incompatible new releases. +`update.json` now supports either a single object or a list of objects. When a +list is used, each entry should define both: + +- `branch`: the git branch name without the `origin/` prefix +- `channel`: the Slips update channel name (`stable`, `unstable`, `testing`) + +Slips first tries to match the configured branch, then falls back to the +configured channel. For backward compatibility, older list entries that only +use `branch` with channel aliases are still accepted. + +Example: + +```json +[ + { + "version": "1.1.20", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": false, + "branch": "master", + "channel": "stable" + }, + { + "version": "1.1.19", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": false, + "branch": "develop", + "channel": "unstable" + }, + { + "version": "1.1.19", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": true, + "branch": "feature/your_branch_here", + "channel": "testing" + } +] +``` + **The update is aborted if:** @@ -141,10 +184,38 @@ PS: the new updated slips version starts reading flows before the old one starts ## How to use it -enable ```auto_update_slips``` in ```config/slips.yaml``` and run slips on your interface. +Enable `auto_update_slips` in `config/slips.yaml`, set +`channel_to_update_slips_from`, and run slips on your interface. + +Available channel mappings: + +- `stable` -> `origin/master` +- `unstable` -> `origin/develop` +- `testing` -> use `testing_branch_to_update_slips_from` + +Examples: + +```yaml +update: + auto_update_slips: true + channel_to_update_slips_from: stable +``` + +```yaml +update: + auto_update_slips: true + channel_to_update_slips_from: testing + testing_branch_to_update_slips_from: origin/feature_branch +``` + +When the channel is `testing`, Slips sanitizes +`testing_branch_to_update_slips_from` before using it in git operations. If the +channel or testing branch is invalid, it falls back to `stable`. now whenever a new version of Slips is available, it will update itself and the new slips will use the same CLI as the old one. +During this live update, Slips also updates its git submodules. If local P2P is enabled, the restarted updated process rebuilds the `p2p4slips` binary before the P2P module starts. + ## Manual update If you do not use `auto_update_slips`, update Slips manually using the method @@ -166,7 +237,7 @@ repository, then rebuild the image so the new code and dependencies are availabl into the container: ```bash -git pull --recurse-submodules && git submodule update --init --recursive +git pull --recurse-submodules origin && git submodule update --init --recursive docker build --target amd --no-cache -t slips -f docker/Dockerfile . ``` @@ -183,7 +254,7 @@ Then start a new container from the rebuilt image. For native installations, first update the repository and all submodules: ```bash -git pull --recurse-submodules && git submodule update --init --recursive +git pull --recurse-submodules origin && git submodule update --init --recursive ``` Then run the installer script: diff --git a/docs/installation.md b/docs/installation.md index 2fdd22831f..2458d63431 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -179,7 +179,8 @@ If you cloned Slips in '~/StratosphereLinuxIPS', then you can build the Docker i cd ~/StratosphereLinuxIPS docker build --target amd --no-cache -t slips -f docker/Dockerfile . docker run -it --rm --net=host slips - ./slips.py -c config/slips.yaml -f dataset/test3-mixed.binetflow + cp config/slips.yaml config/my_slips.yaml + ./slips.py -c config/my_slips.yaml -f dataset/test3-mixed.binetflow If you don't have Internet connection from inside your Docker image while building, you may have another set of networks defined in your Docker. For that try: diff --git a/docs/usage.md b/docs/usage.md index 2c1ec0c226..9762e043aa 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -335,6 +335,21 @@ request to the DNS server 1.2.3.4 asking for slack.com will still be shown. This whitelist can be enabled or disabled by changing the ```enable_local_whitelist``` key in `config/slips.yaml`. +Do not modify the default ```config/whitelist.conf``` in place. Create a copy, update your copy, and set ```whitelists.local_whitelist_path``` in the Slips config file you are using to point to that copy. + +Example: + +```bash +cp config/whitelist.conf config/my_whitelist.conf +cp config/slips.yaml config/my_slips.yaml +``` + +Then set ```local_whitelist_path: config/my_whitelist.conf``` in ```config/my_slips.yaml``` and run Slips with: + +```bash +./slips.py -c config/my_slips.yaml -f dataset/test7-malicious.pcap +``` + The attacker and victim of every evidence are checked against the whitelist. In addition to all the related IPs, DNS resolutions, SNI, and CNAMEs of the attacker and teh victim. If any of them are whitelisted, the flow/evidence is discarded. Whitelists now use bloom filters to speed up the process of checking if an IoC is whitelisted or not. @@ -393,7 +408,9 @@ The tranco list is updated daily by default in Slips, but you can change how oft Tranco whitelist can be enabled or disabled by changing the ```enable_online_whitelist``` key in `config/slips.yaml`. ### Whitelisting Example -You can modify the file ```config/whitelist.conf``` file with this content: +Do not edit the default ```config/whitelist.conf``` directly. Copy it, set ```local_whitelist_path``` in your copied Slips config file to the copied whitelist file, and modify that copied whitelist instead. + +For example, your copied whitelist file can contain: "IoCType","IoCValue","Direction","IgnoreType" @@ -458,6 +475,15 @@ Even when Slips is run using sudo, it drops root privileges in modules that don Slips has a ```config/slips.yaml``` the contains user configurations for different modules and general execution. Below are some of Slips features that can be modifie with respect to the user preferences. +Do not modify the default ```config/slips.yaml``` in place. Keep it as the shipped baseline, create a copy for your local changes, and run Slips with that copy using ```-c```. + +Example: + +```bash +cp config/slips.yaml config/my_slips.yaml +./slips.py -c config/my_slips.yaml -f dataset/test7-malicious.pcap +``` + ### Generic configuration **Time window width.** @@ -497,6 +523,12 @@ This setting is separate from the runtime ```feeds_update_manager``` module, whi Automatic Slips updates may overwrite the default config files shipped with Slips. If you want to keep local config changes safe, do not modify the default config files. Create and use your own config files with different names instead. +Use ```update.channel_to_update_slips_from``` in slips.yaml to choose the update channel: + +- ```stable``` -> ```origin/master``` +- ```unstable``` -> ```origin/develop``` +- ```testing``` -> uses the branch specified in ```update.testing_branch_to_update_slips_from``` +
@@ -746,7 +778,7 @@ this file can be used for training Slips RNN module. ## Slips parameters -- ```-c``` or ```--config``` Used for changing then path to the Slips config file. default is config/slips.yaml +- ```-c``` or ```--config``` Used for changing then path to the Slips config file. default is config/slips.yaml. It is recommended to copy ```config/slips.yaml``` and pass your copy with ```-c``` instead of editing the default file. - ```-v``` or ```--verbose``` Verbosity level. This logs more info about Slips. - ```-e``` or ```--debug``` Debugging level. This shows more detailed errors. - ```-f``` or ```--filepath``` Read and automatically recognize a Zeek dir, a Zeek conn.log file, a Suricata JSON file, Argus, PCAP. diff --git a/managers/redis_manager.py b/managers/redis_manager.py index 36c1b653e8..26484eddaa 100644 --- a/managers/redis_manager.py +++ b/managers/redis_manager.py @@ -66,7 +66,7 @@ def log_redis_server_pid(self, redis_port: int, redis_pid: int): "Save the DB\n" ) - zeek_dir = self.main.db.get_zeek_output_dir() + zeek_dir = self.main.args.output f.write( f"{now},{self.main.input_information},{redis_port}," diff --git a/managers/update_manager.py b/managers/update_manager.py index 5a526b2444..5c0e463d38 100644 --- a/managers/update_manager.py +++ b/managers/update_manager.py @@ -28,6 +28,11 @@ class UpdateManager: + UPDATE_BRANCH_MAP = { + "stable": "origin/master", + "unstable": "origin/develop", + } + def __init__( self, database: DBManager = None, @@ -40,6 +45,7 @@ def __init__( self.cached_update_info: Optional[Dict[str, Any]] = None self.conf = ConfigParser() self.args = self.conf.get_args() + self.print = print_func or (lambda *args, **kwargs: None) # The very first time, slips is started by the user via CLI. then # for each new update, it's started by this update manager. # this func returns true if the user just started slips from cli. @@ -48,14 +54,48 @@ def __init__( ) self._read_configuration() self.last_update_time = 0 - self.print = print_func def _read_configuration(self): + """ + Read update channel and branch configuration. + + Returns: + None. + """ self.auto_update_slips_enabled = self.conf.auto_update_slips() + self.update_channel = self.conf.channel_to_update_slips_from() + self.testing_branch_to_update_slips_from = ( + self.conf.testing_branch_to_update_slips_from() + ) - def _get_master_update_json_link(self) -> Optional[str]: + if self.update_channel == "testing": + self.update_branch = self.testing_branch_to_update_slips_from + if not self.update_branch: + # use stable instead + self.update_branch = self.UPDATE_BRANCH_MAP["stable"] + self.update_channel = "stable" + self.print( + "Warning: Invalid " + "update.testing_branch_to_update_slips_from value " + f"{self.testing_branch_to_update_slips_from!r}. Falling " + f"back to stable." + ) + return + + self.update_branch = self.UPDATE_BRANCH_MAP[self.update_channel] + + def _get_remote_branch_name(self) -> str: """ - Build the raw GitHub URL for update.json on the master branch. + Get the branch name without the origin/ prefix. + + Returns: + The configured remote branch name. + """ + return self.update_branch.removeprefix("origin/") + + def _get_update_json_link(self) -> Optional[str]: + """ + Build the raw GitHub URL for update.json on the configured branch. Returns: The raw update.json URL if the origin remote is supported, @@ -86,12 +126,58 @@ def _get_master_update_json_link(self) -> Optional[str]: return None return ( - f"https://raw.githubusercontent.com/{repo_path}/master/update.json" + "https://raw.githubusercontent.com/" + f"{repo_path}/{self._get_remote_branch_name()}/update.json" ) - def _read_master_update_json(self) -> Dict[str, Any]: + def _get_matching_update_entry( + self, update_data: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """ + Which entry of update.json matches the user selection in slips + config file? + + Parameters: + update_data: Parsed list of update.json entries. + + Returns: + The matching update metadata entry, or an empty dictionary. + """ + if not update_data: + return {} + + short_branch_name = self._get_remote_branch_name() + normalized_channel = self.update_channel.lower() + + for entry in update_data: + entry_branch = entry.get("branch") + if ( + isinstance(entry_branch, str) + and entry_branch.strip().lower() == short_branch_name.lower() + ): + return entry + + for entry in update_data: + entry_channel = entry.get("channel") + if ( + isinstance(entry_channel, str) + and entry_channel.strip().lower() == normalized_channel + ): + return entry + + for entry in update_data: + entry_branch = entry.get("branch") + if ( + isinstance(entry_branch, str) + and entry_branch.strip().lower() == normalized_channel + ): + return entry + + return {} + + def _read_update_json(self) -> Dict[str, Any]: """ - Read the update.json file from the origin/master branch of slips repo. + Read the update.json file from the configured Slips branch. Returns: Parsed update metadata if it can be fetched and decoded, @@ -100,7 +186,7 @@ def _read_master_update_json(self) -> Dict[str, Any]: if self.cached_update_info is not None: return self.cached_update_info - update_json_link = self._get_master_update_json_link() + update_json_link = self._get_update_json_link() if not update_json_link: self.cached_update_info = {} return self.cached_update_info @@ -119,35 +205,43 @@ def _read_master_update_json(self) -> Dict[str, Any]: self.cached_update_info = {} return self.cached_update_info - self.cached_update_info = ( - update_data if isinstance(update_data, dict) else {} - ) + if isinstance(update_data, dict): + self.cached_update_info = update_data + return self.cached_update_info + + if isinstance(update_data, list): + self.cached_update_info = self._get_matching_update_entry( + [entry for entry in update_data if isinstance(entry, dict)] + ) + return self.cached_update_info + + self.cached_update_info = {} return self.cached_update_info def _new_version_has_new_dependencies(self) -> bool: """ - Check whether the version on master introduces new dependencies. + Check whether the configured update target introduces dependencies. Returns: True if update.json reports new dependencies or the metadata cannot be read safely, otherwise False. """ - update_data = self._read_master_update_json() + update_data = self._read_update_json() return bool(update_data.get("has_new_dependencies", True)) def _is_new_version_backwards_compatible(self) -> bool: """ - Check whether the version on master is backwards compatible. + Check whether the configured update target is backwards compatible. Returns: True if update.json marks the update as backwards compatible, otherwise False. """ - update_data = self._read_master_update_json() + update_data = self._read_update_json() return bool(update_data.get("backwards_compatible", False)) def _is_new_version_available(self) -> bool: - update_data = self._read_master_update_json() + update_data = self._read_update_json() latest_version = update_data.get("version", False) if not latest_version: @@ -155,20 +249,34 @@ def _is_new_version_available(self) -> bool: return utils.get_current_version() != latest_version - def git_pull_master(self): + def git_pull_branch(self): """ - Pull the latest origin/master changes and check them out. + Pull the latest configured branch changes and check them out. Returns: - The checked out origin/master commit. + None. """ repo = Repo(".") - repo.remote("origin").fetch("master") - repo.git.checkout("origin/master") + remote_branch_name = self._get_remote_branch_name() + repo.remote("origin").fetch(remote_branch_name) + repo.git.checkout(self.update_branch) self.print( - "Done pulling new version and checking out master " "branch." + "Done pulling new version and checking out " + f"{self.update_branch} branch." ) + def update_submodules(self): + """ + Sync and update git submodules after the main repository update. + + Returns: + None. + """ + repo = Repo(".") + repo.git.submodule("sync", "--recursive") + repo.git.submodule("update", "--init", "--recursive") + self.print("Done updating Slips submodules.") + def _get_checkout_overwritten_files( self, git_error: GitCommandError ) -> List[str]: @@ -219,9 +327,7 @@ def _get_target_update_version(self) -> Optional[str]: Returns: The update version if known, otherwise None. """ - update_data = ( - self.cached_update_info or self._read_master_update_json() - ) + update_data = self.cached_update_info or self._read_update_json() version = update_data.get("version") return version if isinstance(version, str) and version else None @@ -298,7 +404,8 @@ def _warn_about_aborted_update( def update_slips(self): try: - self.git_pull_master() + self.git_pull_branch() + self.update_submodules() except GitError as git_error: self._warn_about_aborted_update(git_error) return @@ -324,6 +431,9 @@ def check_for_update_every_1_day(self) -> bool: return sTrue if a new compatible version is available and slips should update itself """ + if not self.auto_update_slips_enabled: + return False + if self._did_1d_pass_since_last_update(): should_update: bool = self.should_update_slips() diff --git a/modules/p2p_trust/p2p_trust.py b/modules/p2p_trust/p2p_trust.py index bdebcaff6f..179d262134 100644 --- a/modules/p2p_trust/p2p_trust.py +++ b/modules/p2p_trust/p2p_trust.py @@ -80,7 +80,8 @@ class Trust(IModule): pygo_channel_raw = "p2p_pygo" start_pigeon = True # or make sure the binary is in $PATH - pigeon_binary = os.path.join(os.getcwd(), "p2p4slips/p2p4slips") + pigeon_binary_dir = "p2p4slips" + pigeon_binary = os.path.join(pigeon_binary_dir, "p2p4slips") pigeon_key_file = "pigeon.keys" rename_redis_ip_info = False override_p2p = False @@ -200,6 +201,9 @@ def _configure(self): self.pigeon = None if self.start_pigeon: + if not self._rebuild_pigeon_binary_after_slips_update(): + return + if not shutil.which(self.pigeon_binary): self.print( f"Warning: P2p4slips binary not found in " @@ -230,6 +234,60 @@ def _configure(self): executable, cwd=self.p2p_trust_runtime_dir, stdout=outfile ) + def _should_rebuild_pigeon_binary(self) -> bool: + """ + Check whether the p2p binary should be rebuilt after a Slips update. + + Returns: + True when Slips was started by the update manager and local p2p + is enabled, otherwise False. + """ + return bool( + self.start_pigeon + and getattr(self.args, "is_slips_started_by_an_update", False) + and self.conf.use_local_p2p() + ) + + def _rebuild_pigeon_binary_after_slips_update(self) -> bool: + """ + Rebuild the p2p4slips binary after a live Slips update when needed. + + Returns: + True when no rebuild is needed or the rebuild succeeds, + otherwise False. + """ + if not self._should_rebuild_pigeon_binary(): + return True + + self.print( + "Rebuilding p2p4slips after Slips update. This can take " + "some time." + ) + try: + subprocess.run( + ["go", "build"], + cwd=self.pigeon_binary_dir, + check=True, + capture_output=True, + text=True, + ) + except OSError as error: + self.print( + "Warning: Failed to rebuild p2p4slips after Slips update. " + f"Error: {error}" + ) + return False + except subprocess.CalledProcessError as error: + error_output = (error.stderr or error.stdout or str(error)).strip() + self.print( + "Warning: Failed to rebuild p2p4slips after Slips update. " + f"Error: {error_output}" + ) + return False + + self.print("Done rebuilding p2p4slips after Slips update.") + return True + def extract_confidence(self, evidence: Evidence) -> Optional[float]: """ returns the confidence of the given evidence or None if no diff --git a/slips_files/common/parsers/config_parser.py b/slips_files/common/parsers/config_parser.py index c7eba4682f..0dc9e548d3 100644 --- a/slips_files/common/parsers/config_parser.py +++ b/slips_files/common/parsers/config_parser.py @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: 2021 Sebastian Garcia # SPDX-License-Identifier: GPL-2.0-only +import re from datetime import timedelta import os import sys @@ -8,6 +9,7 @@ from typing import ( List, Union, + Optional, ) import yaml from ipaddress import IPv4Network, IPv6Network, IPv4Address, IPv6Address @@ -20,6 +22,10 @@ class ConfigParser(object): name = "ConfigParser" description = "Parse and sanitize slips.yaml values. used by all modules" authors = ["Alya Gomaa"] + UPDATE_BRANCH_ALIASES = { + "stable": "origin/master", + "unstable": "origin/develop", + } def __init__(self): configfile: str = self.get_config_file() @@ -152,6 +158,108 @@ def auto_update_slips(self) -> bool: """ return self.read_configuration("update", "auto_update_slips", False) + def channel_to_update_slips_from(self) -> str: + """ + Read which update channel Slips should auto-update from. + + Returns: + The configured channel name, or "stable" when unset. + """ + return self._sanitize_update_channel( + self.read_configuration( + "update", "channel_to_update_slips_from", "stable" + ) + ) + + def _sanitize_update_branch(self, branch_name: str) -> Optional[str]: + """ + Resolve and sanitize the configured update branch. + + Parameters: + branch_name: Raw branch selector from the configuration file. + + Returns: + A safe origin/ reference, or None when the value + cannot be sanitized safely. + """ + if not isinstance(branch_name, str): + return None + + normalized_branch_name = branch_name.strip() + # is it master or develop?? + branch_alias = self.UPDATE_BRANCH_ALIASES.get( + normalized_branch_name.lower() + ) + if branch_alias: + return branch_alias + + if not normalized_branch_name.startswith("origin/"): + return None + + if ( + ".." in normalized_branch_name + or "@{" in normalized_branch_name + or "\\" in normalized_branch_name + or normalized_branch_name.endswith("/") + or "//" in normalized_branch_name + or normalized_branch_name.endswith(".lock") + ): + return None + + if not re.fullmatch( + r"origin/[A-Za-z0-9._/-]+", normalized_branch_name + ): + return None + + branch_without_remote = normalized_branch_name.removeprefix("origin/") + if branch_without_remote.startswith(("-", "/", ".")): + return None + + return normalized_branch_name + + def _sanitize_update_channel(self, channel_name: str) -> str: + """ + Resolve and sanitize the configured update channel. + + Parameters: + channel_name: Raw channel name from the configuration file. + + Returns: + A supported update channel name. + """ + if not isinstance(channel_name, str): + return "stable" + + normalized_channel_name = channel_name.strip().lower() + if normalized_channel_name in ( + "stable", + "unstable", + "testing", + ): + return normalized_channel_name + + self.print( + "Warning: Invalid update.channel_to_update_slips_from " + f"value {channel_name!r}. Falling back to stable." + ) + return "stable" + + def testing_branch_to_update_slips_from(self) -> str: + """ + Read which testing branch Slips should auto-update from. + + Returns: + The configured testing branch ref, or + "origin/your_branch_here" when unset. + """ + return self._sanitize_update_branch( + self.read_configuration( + "update", + "testing_branch_to_update_slips_from", + "origin/your_branch_here", + ) + ) + def online_whitelist(self): return self.read_configuration("whitelists", "online_whitelist", False) diff --git a/slips_files/core/helpers/checker.py b/slips_files/core/helpers/checker.py index 16ba8b438d..a9293ed904 100644 --- a/slips_files/core/helpers/checker.py +++ b/slips_files/core/helpers/checker.py @@ -256,12 +256,16 @@ def clear_redis_cache(self): redis_cache_server_pid = self.main.redis_man.get_pid_of_redis_server( redis_cache_default_server_port ) - print("Deleting Cache DB in Redis.") + print( + f"\nDeleting the cache database in the Redis server running on " + f"port {redis_cache_default_server_port}." + ) self.main.redis_man.clear_redis_cache_database() self.main.input_information = "" self.main.redis_man.log_redis_server_pid( redis_cache_default_server_port, redis_cache_server_pid ) + print("Done deleting the cache database.") self.main.terminate_slips() def input_module_exists(self, module): diff --git a/tests/unit/managers/test_redis_manager.py b/tests/unit/managers/test_redis_manager.py index 3e20eb59c6..2f9f9dfecb 100644 --- a/tests/unit/managers/test_redis_manager.py +++ b/tests/unit/managers/test_redis_manager.py @@ -24,7 +24,7 @@ 1234, False, False, - "Date,input_info,32768,1234,zeek_dir," + "Date,input_info,32768,1234,output_dir," "output_dir,os_pid,False,False\n", ), # Testcase 2: Daemon mode @@ -33,7 +33,7 @@ 9101, True, False, - "Date,input_info,32769,9101,zeek_dir," + "Date,input_info,32769,9101,output_dir," "output_dir,os_pid,True,False\n", ), # Testcase 3: Save DB @@ -42,7 +42,7 @@ 1122, False, True, - "Date,input_info,32770,1122,zeek_dir," + "Date,input_info,32770,1122,output_dir," "output_dir,os_pid,False,True\n", ), ], @@ -52,7 +52,6 @@ def test_log_redis_server_pid_normal_ports( ): redis_manager = ModuleFactory().create_redis_manager_obj() redis_manager.main.input_information = "input_info" - redis_manager.main.db.get_zeek_output_dir.return_value = "zeek_dir" redis_manager.main.args.output = "output_dir" redis_manager.main.args.daemon = is_daemon redis_manager.main.args.save = save_db diff --git a/tests/unit/managers/test_update_manager.py b/tests/unit/managers/test_update_manager.py index 3a243c6d9e..cbe5e98ca1 100644 --- a/tests/unit/managers/test_update_manager.py +++ b/tests/unit/managers/test_update_manager.py @@ -24,6 +24,10 @@ def create_update_manager(): multiinstance=False, ) conf.auto_update_slips.return_value = True + conf.channel_to_update_slips_from.return_value = "stable" + conf.testing_branch_to_update_slips_from.return_value = ( + "origin/feature/test" + ) with patch("managers.update_manager.ConfigParser", return_value=conf): return UpdateManager( @@ -34,28 +38,171 @@ def create_update_manager(): @pytest.mark.parametrize( - "remote_url, expected_link", + "configured_branch, remote_url, expected_link", [ ( + "origin/master", "https://github.com/stratosphereips/StratosphereLinuxIPS.git", "https://raw.githubusercontent.com/stratosphereips/" "StratosphereLinuxIPS/master/update.json", ), ( + "origin/develop", "git@github.com:stratosphereips/StratosphereLinuxIPS.git", "https://raw.githubusercontent.com/stratosphereips/" - "StratosphereLinuxIPS/master/update.json", + "StratosphereLinuxIPS/develop/update.json", + ), + ( + "origin/feature/test", + "git@github.com:stratosphereips/StratosphereLinuxIPS.git", + "https://raw.githubusercontent.com/stratosphereips/" + "StratosphereLinuxIPS/feature/test/update.json", + ), + ( + "origin/master", + "https://example.com/stratosphereips/StratosphereLinuxIPS.git", + None, ), - ("https://example.com/stratosphereips/StratosphereLinuxIPS.git", None), ], ) -def test_get_master_update_json_link(remote_url, expected_link): +def test_get_update_json_link(configured_branch, remote_url, expected_link): update_manager = create_update_manager() + update_manager.update_branch = configured_branch repo = Mock() repo.remote.return_value.url = remote_url with patch("managers.update_manager.Repo", return_value=repo): - assert update_manager._get_master_update_json_link() == expected_link + assert update_manager._get_update_json_link() == expected_link + + +@pytest.mark.parametrize( + "configured_channel, configured_testing_branch, expected_branch, expected_channel", + [ + ("stable", "origin/feature/test", "origin/master", "stable"), + ("unstable", "origin/feature/test", "origin/develop", "unstable"), + ("testing", "origin/feature/test", "origin/feature/test", "testing"), + ], +) +def test_read_configuration_resolves_update_branch( + configured_channel, + configured_testing_branch, + expected_branch, + expected_channel, +): + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) + conf.auto_update_slips.return_value = True + conf.channel_to_update_slips_from.return_value = configured_channel + conf.testing_branch_to_update_slips_from.return_value = ( + configured_testing_branch + ) + + with patch("managers.update_manager.ConfigParser", return_value=conf): + update_manager = UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=Mock(), + ) + + assert update_manager.update_branch == expected_branch + assert update_manager.update_channel == expected_channel + + +def test_invalid_update_channel_raises_key_error(): + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) + conf.auto_update_slips.return_value = True + conf.channel_to_update_slips_from.return_value = "branch with spaces" + conf.testing_branch_to_update_slips_from.return_value = ( + "origin/feature/test" + ) + + with ( + patch("managers.update_manager.ConfigParser", return_value=conf), + pytest.raises(KeyError, match="branch with spaces"), + ): + UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=Mock(), + ) + + +@pytest.mark.parametrize( + "configured_testing_branch", + [ + "origin/../master", + "origin/@{bad}", + "origin/-bad", + ], +) +def test_testing_update_channel_uses_configured_branch_as_is( + configured_testing_branch, +): + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) + conf.auto_update_slips.return_value = True + conf.channel_to_update_slips_from.return_value = "testing" + conf.testing_branch_to_update_slips_from.return_value = ( + configured_testing_branch + ) + + with patch("managers.update_manager.ConfigParser", return_value=conf): + update_manager = UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=Mock(), + ) + + assert update_manager.update_branch == configured_testing_branch + assert update_manager.update_channel == "testing" + + +def test_testing_update_channel_without_branch_falls_back_to_stable(): + db = Mock() + db.is_running_non_stop.return_value = True + + conf = Mock() + conf.get_args.return_value = Mock( + is_slips_started_by_an_update=False, + multiinstance=False, + ) + conf.auto_update_slips.return_value = True + conf.channel_to_update_slips_from.return_value = "testing" + conf.testing_branch_to_update_slips_from.return_value = "" + print_func = Mock() + + with patch("managers.update_manager.ConfigParser", return_value=conf): + update_manager = UpdateManager( + database=db, + is_slips_live_updating_event=Mock(), + print_func=print_func, + ) + + assert update_manager.update_branch == "origin/master" + assert update_manager.update_channel == "stable" + print_func.assert_called_once_with( + "Warning: Invalid update.testing_branch_to_update_slips_from " + "value ''. Falling back to stable." + ) @pytest.mark.parametrize( @@ -87,7 +234,7 @@ def test_update_json_flags( with patch.object( update_manager, - "_get_master_update_json_link", + "_get_update_json_link", return_value=( "https://raw.githubusercontent.com/org/repo/master/update.json" ), @@ -122,7 +269,7 @@ def test_update_json_fallbacks( patches = [ patch.object( update_manager, - "_get_master_update_json_link", + "_get_update_json_link", return_value=( "https://raw.githubusercontent.com/org/repo/master/update.json" ), @@ -159,6 +306,116 @@ def test_update_json_fallbacks( ) +def test_update_json_list_uses_matching_branch_entry(): + update_manager = create_update_manager() + update_manager.update_branch = "origin/develop" + update_manager.update_channel = "unstable" + response = Mock() + response.read.return_value = ( + "[" + '{"version": "1.1.20", "branch": "master", "channel": "stable",' + ' "has_new_dependencies": false,' + ' "backwards_compatible": true},' + '{"version": "1.1.21", "branch": "develop", "channel": "unstable",' + ' "has_new_dependencies": true,' + ' "backwards_compatible": false}' + "]" + ).encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + + with patch.object( + update_manager, + "_get_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/develop/update.json" + ), + ), patch( + "managers.update_manager.request.urlopen", + return_value=response, + ): + assert update_manager._read_update_json() == { + "version": "1.1.21", + "branch": "develop", + "channel": "unstable", + "has_new_dependencies": True, + "backwards_compatible": False, + } + + +def test_update_json_list_falls_back_to_matching_channel_entry(): + update_manager = create_update_manager() + update_manager.update_branch = "origin/feature/test" + update_manager.update_channel = "testing" + response = Mock() + response.read.return_value = ( + "[" + '{"version": "1.1.20", "branch": "master", "channel": "stable",' + ' "has_new_dependencies": false,' + ' "backwards_compatible": true},' + '{"version": "1.1.22", "branch": "feature/other", "channel": "testing",' + ' "has_new_dependencies": false,' + ' "backwards_compatible": true}' + "]" + ).encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + + with patch.object( + update_manager, + "_get_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/feature/test/update.json" + ), + ), patch( + "managers.update_manager.request.urlopen", + return_value=response, + ): + assert update_manager._read_update_json() == { + "version": "1.1.22", + "branch": "feature/other", + "channel": "testing", + "has_new_dependencies": False, + "backwards_compatible": True, + } + + +def test_update_json_list_supports_legacy_branch_only_channel_aliases(): + update_manager = create_update_manager() + update_manager.update_branch = "origin/develop" + update_manager.update_channel = "unstable" + response = Mock() + response.read.return_value = ( + "[" + '{"version": "1.1.20", "branch": "stable",' + ' "has_new_dependencies": false,' + ' "backwards_compatible": true},' + '{"version": "1.1.21", "branch": "unstable",' + ' "has_new_dependencies": true,' + ' "backwards_compatible": false}' + "]" + ).encode("utf-8") + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + + with patch.object( + update_manager, + "_get_update_json_link", + return_value=( + "https://raw.githubusercontent.com/org/repo/develop/update.json" + ), + ), patch( + "managers.update_manager.request.urlopen", + return_value=response, + ): + assert update_manager._read_update_json() == { + "version": "1.1.21", + "branch": "unstable", + "has_new_dependencies": True, + "backwards_compatible": False, + } + + def test_get_updated_slips_command_appends_update_flag(): update_manager = create_update_manager() process = Mock() @@ -216,10 +473,19 @@ def test_start_updated_slips_verison_starts_detached_process(): def test_update_slips_starts_updated_process_before_stopping_current_slips(): + """ + Ensure Slips updates submodules before starting the new process. + + Returns: + None. + """ update_manager = create_update_manager() calls = [] - update_manager.git_pull_master = Mock( - side_effect=lambda: calls.append("git_pull_master") + update_manager.git_pull_branch = Mock( + side_effect=lambda: calls.append("git_pull_branch") + ) + update_manager.update_submodules = Mock( + side_effect=lambda: calls.append("update_submodules") ) update_manager.start_updated_slips_version = Mock( side_effect=lambda: calls.append("start_updated_slips_verison") @@ -230,16 +496,47 @@ def test_update_slips_starts_updated_process_before_stopping_current_slips(): update_manager.update_slips() - update_manager.git_pull_master.assert_called_once() + update_manager.git_pull_branch.assert_called_once() + update_manager.update_submodules.assert_called_once() update_manager.start_updated_slips_version.assert_called_once() update_manager.is_slips_live_updating_event.set.assert_called_once() assert calls == [ - "git_pull_master", + "git_pull_branch", + "update_submodules", "start_updated_slips_verison", "set_update_event", ] +def test_update_slips_aborts_when_submodule_update_fails(): + """ + Ensure submodule update errors abort the live update. + + Returns: + None. + """ + update_manager = create_update_manager() + git_error = GitCommandError( + "git submodule update --init --recursive", + 1, + stderr="fatal: unable to update submodule", + ) + update_manager.git_pull_branch = Mock() + update_manager.update_submodules = Mock(side_effect=git_error) + update_manager.start_updated_slips_version = Mock() + + update_manager.update_slips() + + update_manager.git_pull_branch.assert_called_once() + update_manager.update_submodules.assert_called_once() + update_manager.start_updated_slips_version.assert_not_called() + update_manager.is_slips_live_updating_event.set.assert_not_called() + update_manager.print.assert_called_once_with( + "Warning: Aborting Slips update because a git error occurred: " + f"{git_error}" + ) + + def test_update_slips_aborts_when_local_changes_block_checkout(): """ Ensure local checkout conflicts abort the update without stopping Slips. @@ -261,12 +558,12 @@ def test_update_slips_aborts_when_local_changes_block_checkout(): "Aborting" ), ) - update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.git_pull_branch = Mock(side_effect=git_error) update_manager.start_updated_slips_version = Mock() update_manager.update_slips() - update_manager.git_pull_master.assert_called_once() + update_manager.git_pull_branch.assert_called_once() update_manager.start_updated_slips_version.assert_not_called() update_manager.is_slips_live_updating_event.set.assert_not_called() update_manager.print.assert_called_once_with( @@ -288,7 +585,7 @@ def test_update_slips_aborts_on_unrelated_git_errors(): 128, stderr="fatal: not a git repository", ) - update_manager.git_pull_master = Mock(side_effect=git_error) + update_manager.git_pull_branch = Mock(side_effect=git_error) update_manager.start_updated_slips_version = Mock() update_manager.update_slips() @@ -299,3 +596,18 @@ def test_update_slips_aborts_on_unrelated_git_errors(): "Warning: Aborting Slips update because a git error occurred: " f"{git_error}" ) + + +def test_git_pull_branch_fetches_and_checks_out_configured_branch(): + update_manager = create_update_manager() + update_manager.update_branch = "origin/develop" + repo = Mock() + + with patch("managers.update_manager.Repo", return_value=repo): + update_manager.git_pull_branch() + + repo.remote.return_value.fetch.assert_called_once_with("develop") + repo.git.checkout.assert_called_once_with("origin/develop") + update_manager.print.assert_called_once_with( + "Done pulling new version and checking out origin/develop branch." + ) diff --git a/tests/unit/modules/p2p_trust/test_p2p_trust.py b/tests/unit/modules/p2p_trust/test_p2p_trust.py new file mode 100644 index 0000000000..e879815f73 --- /dev/null +++ b/tests/unit/modules/p2p_trust/test_p2p_trust.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: 2021 Sebastian Garcia +# SPDX-License-Identifier: GPL-2.0-only + +from types import SimpleNamespace +from unittest.mock import Mock, call, patch + +import pytest + +from modules.p2p_trust.p2p_trust import Trust + + +def create_trust(): + """ + Create a minimal Trust object for unit tests. + + Returns: + A Trust instance with mocked dependencies. + """ + trust = Trust.__new__(Trust) + trust.start_pigeon = True + trust.args = SimpleNamespace(is_slips_started_by_an_update=False) + trust.conf = Mock() + trust.conf.use_local_p2p.return_value = False + trust.db = Mock() + trust.print = Mock() + trust.pigeon_binary_dir = "p2p4slips" + return trust + + +@pytest.mark.parametrize( + "is_slips_started_by_an_update,use_local_p2p,expected", + [ + (False, False, False), + (False, True, False), + (True, False, False), + (True, True, True), + ], +) +def test_should_rebuild_pigeon_binary( + is_slips_started_by_an_update, use_local_p2p, expected +): + """ + Ensure the p2p binary rebuild only runs for updated local p2p runs. + + Parameters: + is_slips_started_by_an_update: Whether Slips was restarted by update. + use_local_p2p: Whether local p2p is enabled in config. + expected: Expected rebuild decision. + + Returns: + None. + """ + trust = create_trust() + trust.args.is_slips_started_by_an_update = is_slips_started_by_an_update + trust.conf.use_local_p2p.return_value = use_local_p2p + + assert trust._should_rebuild_pigeon_binary() is expected + + +def test_rebuild_pigeon_binary_after_slips_update_runs_go_build(): + """ + Ensure the p2p module rebuilds p2p4slips after a live update. + + Returns: + None. + """ + trust = create_trust() + trust.args.is_slips_started_by_an_update = True + trust.conf.use_local_p2p.return_value = True + + with patch("modules.p2p_trust.p2p_trust.subprocess.run") as mock_run: + assert trust._rebuild_pigeon_binary_after_slips_update() is True + + mock_run.assert_called_once_with( + ["go", "build"], + cwd="p2p4slips", + check=True, + capture_output=True, + text=True, + ) + assert trust.print.call_args_list == [ + call( + "Rebuilding p2p4slips after Slips update. This can take " + "some time." + ), + call("Done rebuilding p2p4slips after Slips update."), + ] + + +def test_rebuild_pigeon_binary_after_slips_update_stops_on_build_error(): + """ + Ensure build failures are reported and stop p2p startup. + + Returns: + None. + """ + trust = create_trust() + trust.args.is_slips_started_by_an_update = True + trust.conf.use_local_p2p.return_value = True + + with patch( + "modules.p2p_trust.p2p_trust.subprocess.run", + side_effect=OSError("go not found"), + ): + assert trust._rebuild_pigeon_binary_after_slips_update() is False + + assert trust.print.call_args_list == [ + call( + "Rebuilding p2p4slips after Slips update. This can take " + "some time." + ), + call( + "Warning: Failed to rebuild p2p4slips after Slips update. " + "Error: go not found" + ), + ] diff --git a/update.json b/update.json index 1178023cc7..14deeb0c99 100644 --- a/update.json +++ b/update.json @@ -1,6 +1,26 @@ -{ -"version": "1.1.20", -"release_date": "2026-04-30T14:39:56+03:00", -"backwards_compatible": true, -"has_new_dependencies": false, -} +[ + { + "version": "1.1.20", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": false, + "branch": "master", + "channel": "stable" + }, + { + "version": "1.1.20", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": false, + "branch": "develop", + "channel": "unstable" + }, + { + "version": "1.1.19", + "release_date": "2026-04-30T14:39:56+03:00", + "backwards_compatible": true, + "has_new_dependencies": false, + "branch": "feature/your_branch_here", + "channel": "testing" + } +]