Improve integration tests

.github/workflows/integration-tests.yml

@@ -53,11 +53,18 @@ jobs:
         run: |
           python3 -m pytest tests/${{ matrix.test_file }} -p no:warnings -vv -s -n 3
 
+      - name: Build Artifact Name
+        # otherwise we get numeric names for the artifacts and we dont know which is which
+        id: artifact-name
+        run: |
+          sanitized_test_file="${{ matrix.test_file }}"
+          sanitized_test_file=$(printf '%s\n' "$sanitized_test_file" | tr '/' '_')
+          echo "name=${sanitized_test_file}-integration-output" >> "$GITHUB_OUTPUT"
+
       - name: Upload Artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          # Replaces slashes with underscores for valid artifact naming
-          name: ${{ github.run_id }}-${{ strategy.job-index }}-integration-output
+          name: ${{ steps.artifact-name.outputs.name }}
           path: |
-            output/integration
+            output/integration_tests

config/TI_feeds.csv

@@ -11,7 +11,7 @@ https://mcfp.felk.cvut.cz/publicDatasets/CTU-AIPP-BlackList/Todays-Blacklists/AI
 https://mcfp.felk.cvut.cz/publicDatasets/CTU-AIPP-BlackList/Todays-Blacklists/AIP_historical_blacklist_prioritized_by_newest_attackers.csv,medium, ['phishing','honeypot']
 https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/strangereallintel-cyberthreatintel.csv,medium, ['phishing']
 https://raw.githubusercontent.com/AssoEchap/stalkerware-indicators/master/generated/network.csv,medium, ['stalkerware']
-https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/adserversandtrackers.csv,medium, ['adtrackers']
+https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/adserversandtrackers.csv,info, ['adtrackers']
 https://raw.githubusercontent.com/stratosphereips/Civilsphere/main/threatintel/civilsphereindicators.csv,medium, ['apt']
 https://raw.githubusercontent.com/botherder/targetedthreats/master/targetedthreats.csv,medium, ['apt']
 https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt,medium, ['honeypot']
@@ -26,10 +26,10 @@ https://lists.blocklist.de/lists/mail.txt,medium, ['honeypot']
 https://lists.blocklist.de/lists/bruteforcelogin.txt,medium, ['honeypot']
 https://feodotracker.abuse.ch/downloads/ipblocklist.csv,medium, ['honeypot']
 https://reputation.alienvault.com/reputation.generic,medium, ['honeypot']
-https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,medium, ['adtrackers']
+https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,info, ['adtrackers']
 # bigdargon: Hosts block ads of Vietnamese
-https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/domain.txt,medium, ['adtrackers']
-https://raw.githubusercontent.com/SweetSophia/mifitxiaomipiholelist/master/mifitblocklist.txt,medium, ['xiaomi-trackers']
+https://raw.githubusercontent.com/bigdargon/hostsVN/master/option/domain.txt,info, ['adtrackers']
+https://raw.githubusercontent.com/SweetSophia/mifitxiaomipiholelist/master/mifitblocklist.txt,info, ['xiaomi-trackers']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/abuse-ch-ipblocklist.intel,medium, ['honeypot']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/alienvault.intel,medium, ['honeypot']
 https://raw.githubusercontent.com/CriticalPathSecurity/Zeek-Intelligence-Feeds/master/cobaltstrike_ips.intel,medium, ['honeypot']

docs/fides.md

@@ -25,7 +25,11 @@ To be able to use the fides module, you should use ```--cap-add=NET_ADMIN```
 If you plan on using the Fides Module, please be aware that it is used only if Slips is running on an interface OR on a growing Zeek directory. The `--use_fides=True` is ignored when Slips is run on a file.
 
 ## Configuration
-The evaluation model used, the evaluation thresholds, and other configurations are located in ```fides.conf.yml``` file
+The evaluation model used, the evaluation thresholds, and other configurations are located in ```modules/fides/config/fides.conf.yml```.
+
+If you need a Slips run to use a different Fides configuration file, set
+```global_p2p.fides_conf``` in Slips config to the relative path
+of that alternate YAML file.
 
 **Possible threat intelligence evaluation models**
 

modules/arp_poisoner/arp_poisoner.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2021 Sebastian Garcia <sebastian.garcia@agents.fel.cvut.cz>
 # SPDX-License-Identifier: GPL-2.0-only
 import logging
+import shutil
 import subprocess
 import time
 from threading import Lock
@@ -30,6 +31,8 @@ class ARPPoisoner(IModule):
     authors = ["Alya Gomaa"]
 
     def init(self):
+        self.arp_scan_path = shutil.which("arp-scan")
+        self.arp_scan_bin_available = self.arp_scan_path is not None
         self._time_since_last_repoison = {}
         self._time_since_last_internet_cut = {}
         self.log_file_path = self.get_module_specific_output_path(
@@ -56,13 +59,32 @@ def init(self):
         self.ip_interface_map = {}
 
     def subscribe_to_channels(self):
+        if not self.arp_scan_bin_available:
+            self.channels = {}
+            return
         self.c1 = self.db.subscribe("new_blocking")
         self.c2 = self.db.subscribe("tw_closed")
         self.channels = {
             "new_blocking": self.c1,
             "tw_closed": self.c2,
         }
 
+    def pre_main(self) -> bool:
+        """
+        Stop the module before entering the main loop when arp-scan is
+        unavailable.
+
+        :return: True when the module should shut down, otherwise False.
+        """
+        if self.arp_scan_bin_available:
+            return False
+
+        self.print(
+            "The arp-scan tool is not installed. ARP poisoner module is "
+            "stopping.",
+        )
+        return True
+
     def log(self, text):
         """Logs the given text to the blocking log file"""
         with self.blocking_logfile_lock:

modules/fides/fides.py

@@ -53,8 +53,11 @@ def init(self):
 
         # load trust model configuration
         current_dir = Path(__file__).resolve().parent
-        config_path = current_dir / "config" / "fides.conf.yml"
-        self.__trust_model_config = load_configuration(config_path.__str__())
+        default_config_path = current_dir / "config" / "fides.conf.yml"
+        config_path = self.conf.read_configuration(
+            "global_p2p", "fides_conf", str(default_config_path)
+        )
+        self.__trust_model_config = load_configuration(config_path)
 
         # prepare variables for global protocols
         self.__bridge: NetworkBridge

modules/fides/messaging/message_handler.py

@@ -1,6 +1,6 @@
 from typing import Dict, List, Callable, Optional, Union
 
-
+from slips_files.common.slips_utils import utils
 from ..messaging.dacite import from_dict
 
 from ..messaging.model import (
@@ -28,7 +28,7 @@ class MessageHandler:
     # def print(self, *args, **kwargs):
     #    return self.printer.print(*args, **kwargs)
 
-    version = 1
+    version = utils.get_current_version()
 
     def __init__(
         self,

modules/fides/messaging/model.py

@@ -13,7 +13,7 @@
 @dataclass
 class NetworkMessage:
     type: str
-    version: int
+    version: str
     data: Any
 
 

modules/fides/messaging/network_bridge.py

@@ -2,6 +2,7 @@
 from dataclasses import asdict
 from typing import Dict, List
 
+from slips_files.common.slips_utils import utils
 from .dacite import from_dict
 
 from .message_handler import MessageHandler
@@ -24,7 +25,7 @@ class NetworkBridge:
     execute "listen" method.
     """
 
-    version = 1
+    version = utils.get_current_version()
 
     def __init__(self, queue: Queue):
         self.__queue = queue
@@ -36,14 +37,15 @@ def listen(self, handler: MessageHandler, block: bool = False):
         """
 
         def message_received(message: str):
+            """this is the callback that executes every new msg"""
             try:
-                # with open("fides_nb.txt", "a") as f:
-                #     f.write(message)
+
                 logger.debug("New message received! Trying to parse.")
                 parsed = json.loads(message)
                 network_message = from_dict(
                     data_class=NetworkMessage, data=parsed
                 )
+
                 logger.debug("Message parsed. Executing handler.")
                 handler.on_message(network_message)
             except Exception as e:

modules/fides/model/peer_trust_data.py

@@ -3,8 +3,11 @@
 
 from ..model.aliases import PeerId, OrganisationId
 from ..model.peer import PeerInfo
-from ..model.recommendation_history import RecommendationHistory
-from ..model.service_history import ServiceHistory
+from ..model.recommendation_history import (
+    RecommendationHistory,
+    RecommendationHistoryRecord,
+)
+from ..model.service_history import ServiceHistory, ServiceHistoryRecord
 
 
 @dataclass
@@ -121,6 +124,7 @@ def to_dict(self, remove_histories: bool = False):
     # Method to create an object from a dictionary
     @classmethod
     def from_dict(cls, data):
+        """Create a PeerTrustData instance from a dictionary payload."""
         return cls(
             info=PeerInfo.from_dict(
                 data["info"]
@@ -135,14 +139,13 @@ def from_dict(cls, data):
                 "initial_reputation_provided_by_count"
             ],
             service_history=[
-                ServiceHistory.from_dict(sh) for sh in data["service_history"]
+                ServiceHistoryRecord.from_dict(sh)
+                for sh in data["service_history"]
             ],
-            # Assuming ServiceHistory has from_dict
             recommendation_history=[
-                RecommendationHistory.from_dict(rh)
+                RecommendationHistoryRecord.from_dict(rh)
                 for rh in data["recommendation_history"]
             ],
-            # Assuming RecommendationHistory has from_dict
         )
 
 

modules/fides/persistence/fides_sqlite_db.py

@@ -6,6 +6,7 @@
 
 import os
 import sqlite3
+from pathlib import Path
 from typing import List, Any, Optional
 
 from slips_files.core.output import Output
@@ -31,13 +32,23 @@ def __init__(self, logger: Output, db_path: str) -> None:
         """
         self.logger = logger
         self.db_path = db_path
-        with open(self.db_path, "a") as f:
-            f.close()
-        sqlite3.connect(self.db_path).close()
+        if not self.__is_in_memory_database():
+            Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+            with open(self.db_path, "a") as f:
+                f.close()
+            sqlite3.connect(self.db_path).close()
         self.connection: Optional[sqlite3.Connection] = None
         self.__connect()
         self.__create_tables()
 
+    def __is_in_memory_database(self) -> bool:
+        """
+        Determines whether the configured database path targets SQLite memory storage.
+
+        :return: True when the database should live only in memory, otherwise False.
+        """
+        return self.db_path == ":memory:"
+
     def __slips_log(self, txt: str) -> None:
         self.logger.output_line_to_cli_and_logfiles(
             {"verbose": 2, "debug": 0, "from": self.name, "txt": txt}

modules/fides/persistence/threat_intelligence_db.py

@@ -33,7 +33,12 @@ def get_for(self, target: Target) -> Optional[SlipsThreatIntelligence]:
         out = self.db.get_fides_ti(target)  # returns str containing dumped
         # dict of STI or None
         if out:
-            out = SlipsThreatIntelligence(**json.loads(out))
+            try:
+                out = SlipsThreatIntelligence(**json.loads(out))
+            except (TypeError, ValueError, json.JSONDecodeError):
+                out = self.sqldb.get_slips_threat_intelligence_by_target(
+                    target
+                )
         else:
             out = self.sqldb.get_slips_threat_intelligence_by_target(target)
         return out

modules/fides/persistence/trust_db.py

@@ -120,7 +120,7 @@ def get_peer_trust_data(
 
         td_json = self.db.get_peer_trust_data(peer_id)
         if td_json:  # Redis has available data
-            out = PeerTrustData(**json.loads(td_json))
+            out = PeerTrustData.from_dict(json.loads(td_json))
         else:  # if redis is empty, try SQLite
             out = self.sqldb.get_peer_trust_data(peer_id)
         return out
@@ -130,17 +130,20 @@ def get_peers_trust_data(
     ) -> TrustMatrix:
         """Return trust data for each peer from peer_ids."""
         out = {}
-        peer_id = None
 
         for peer in peer_ids:
             # get PeerID to properly create TrustMatrix
             if isinstance(peer, PeerId):
                 peer_id = peer
             elif isinstance(peer, PeerInfo):
                 peer_id = peer.id
+            else:
+                continue
 
             # TrustMatrix = Dict[PeerId, PeerTrustData]; here - peer_id: PeerId
-            out[peer_id] = self.get_peer_trust_data(peer_id)
+            trust_data = self.get_peer_trust_data(peer_id)
+            if trust_data is not None:
+                out[peer_id] = trust_data
         return out
 
     def cache_network_opinion(self, ti: SlipsThreatIntelligence):

modules/fides/protocols/threat_intelligence.py

@@ -81,9 +81,17 @@ def handle_intelligence_response(
         self, responses: List[PeerIntelligenceResponse]
     ):
         """Handles intelligence responses."""
-        trust_matrix = self._trust_db.get_peers_trust_data(
-            [r.sender.id for r in responses]
-        )
+        trust_matrix = {}
+        for response in responses:
+            peer_trust = self._trust_db.get_peer_trust_data(response.sender.id)
+            if peer_trust is None:
+                peer_trust = (
+                    self.__trust_protocol.determine_and_store_initial_trust(
+                        response.sender
+                    )
+                )
+            trust_matrix[response.sender.id] = peer_trust
+
         assert len(trust_matrix) == len(
             responses
         ), "We need to have trust data for all peers that sent the response."

modules/fides/utils/logger.py

@@ -1,5 +1,4 @@
 import json
-import threading
 from dataclasses import is_dataclass, asdict
 from typing import Optional, List, Callable
 
@@ -15,7 +14,7 @@
 ]
 
 # Set this to custom callback that should be executed when there's new log message.
-# First parameter is level ('DEBUG', 'INFO', 'WARN', 'ERROR'), second is message to be logged.
+# First parameter is message, second is level ('DEBUG', 'INFO', 'WARN', 'ERROR')
 
 
 class Logger:
@@ -54,23 +53,30 @@ def __try_to_guess_name() -> str:
         return name
 
     def debug(self, message: str, params=None):
-        return self.__print("DEBUG", message)
+        return self.__print("DEBUG", message, params)
 
     def info(self, message: str, params=None):
-        return self.__print("INFO", message)
+        return self.__print("INFO", message, params)
 
+    def warning(self, message: str, params=None):
+        return self.__print("WARN", message, params)
+
+    # keep for backward compatibility
     def warn(self, message: str, params=None):
-        return self.__print("WARN", message)
+        return self.warning(message, params)
 
     def error(self, message: str, params=None):
-        return self.__print("ERROR", message)
+        return self.__print("ERROR", message, params)
 
     def __format(self, message: str, params=None):
-        thread = threading.get_ident()
-        formatted_message = f"T{thread}: {self.__name} -  {message}"
+        formatted_message = f"{self.__name} - {message}"
         if params:
             params = asdict(params) if is_dataclass(params) else params
-            formatted_message = f"{formatted_message} {json.dumps(params)}"
+            try:
+                serialized_params = json.dumps(params)
+            except TypeError:
+                serialized_params = json.dumps(str(params))
+            formatted_message = f"{formatted_message} {serialized_params}"
         return formatted_message
 
     def __print(self, level: str, message: str, params=None):
@@ -81,6 +87,4 @@ def __print(self, level: str, message: str, params=None):
                     formatted_message, verbose=0
                 )  # automatically verbose = 1 - print, debug = 0 - do not print
             else:
-                print_callback(
-                    formatted_message, verbose=self.log_levels[level]
-                )
+                print_callback(formatted_message, debug=1)