From 5d683068640eb016cfda86166aa5dd72ba712dac Mon Sep 17 00:00:00 2001 From: Luis Augenstein Date: Tue, 31 Mar 2026 18:23:42 +0200 Subject: [PATCH] split monolithic savedcmd_parser.py file into separate files Signed-off-by: Luis Augenstein --- .../cmd_graph/savedcmd_parser/__init__.py | 6 + .../savedcmd_parser/command_splitter.py | 124 +++++++ .../savedcmd_parser/savedcmd_parser.py | 60 ++++ .../single_command_parsers.py} | 312 ++---------------- .../cmd_graph/savedcmd_parser/tokenizer.py | 94 ++++++ 5 files changed, 315 insertions(+), 281 deletions(-) create mode 100644 sbom/sbom/cmd_graph/savedcmd_parser/__init__.py create mode 100644 sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py create mode 100644 sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py rename sbom/sbom/cmd_graph/{savedcmd_parser.py => savedcmd_parser/single_command_parsers.py} (60%) create mode 100644 sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py diff --git a/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py b/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py new file mode 100644 index 00000000..d13876af --- /dev/null +++ b/sbom/sbom/cmd_graph/savedcmd_parser/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from sbom.cmd_graph.savedcmd_parser.savedcmd_parser import parse_inputs_from_commands + +__all__ = ["parse_inputs_from_commands"] diff --git a/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py b/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py new file mode 100644 index 00000000..f1059862 --- /dev/null +++ b/sbom/sbom/cmd_graph/savedcmd_parser/command_splitter.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +from dataclasses import dataclass + + +# If Block pattern to match a simple, single-level if-then-fi block. Nested If blocks are not supported. +IF_BLOCK_PATTERN = re.compile( + r""" + ^if(.*?);\s* # Match 'if ;' (non-greedy) + then(.*?);\s* # Match 'then ;' (non-greedy) + fi\b # Match 'fi' + """, + re.VERBOSE, +) + + +@dataclass +class IfBlock: + condition: str + then_statement: str + + +def _unwrap_outer_parentheses(s: str) -> str: + s = s.strip() + if not (s.startswith("(") and s.endswith(")")): + return s + + count = 0 + for i, char in enumerate(s): + if char == "(": + count += 1 + elif char == ")": + count -= 1 + # If count is 0 before the end, outer parentheses don't match + if count == 0 and i != len(s) - 1: + return s + + # outer parentheses do match, unwrap once + return _unwrap_outer_parentheses(s[1:-1]) + + +def _find_first_top_level_command_separator( + commands: str, separators: list[str] = [";", "&&"] +) -> tuple[int | None, int | None]: + in_single_quote = False + in_double_quote = False + in_curly_braces = 0 + in_braces = 0 + for i, char in enumerate(commands): + if char == "'" and not in_double_quote: + # Toggle single quote state (unless inside double quotes) + in_single_quote = not in_single_quote + elif char == '"' and not in_single_quote: + # Toggle double quote state (unless inside single quotes) + in_double_quote = not in_double_quote + + if in_single_quote or in_double_quote: + continue + + # Toggle braces state + if char == "{": + in_curly_braces += 1 + if char == "}": + in_curly_braces -= 1 + + if char == "(": + in_braces += 1 + if char == ")": + in_braces -= 1 + + if in_curly_braces > 0 or in_braces > 0: + continue + + # return found separator position and separator length + for separator in separators: + if commands[i : i + len(separator)] == separator: + return i, len(separator) + + return None, None + + +def split_commands(commands: str) -> list[str | IfBlock]: + """ + Splits a string of command-line commands into individual parts. + + This function handles: + - Top-level command separators (e.g., `;` and `&&`) to split multiple commands. + - Conditional if-blocks, returning them as `IfBlock` instances. + - Preserves the order of commands and trims whitespace. + + Args: + commands (str): The raw command string. + + Returns: + list[str | IfBlock]: A list of single commands or `IfBlock` objects. + """ + single_commands: list[str | IfBlock] = [] + remaining_commands = _unwrap_outer_parentheses(commands) + while len(remaining_commands) > 0: + remaining_commands = remaining_commands.strip() + + # if block + matched_if = IF_BLOCK_PATTERN.match(remaining_commands) + if matched_if: + condition, then_statement = matched_if.groups() + single_commands.append(IfBlock(condition.strip(), then_statement.strip())) + full_matched = matched_if.group(0) + remaining_commands = remaining_commands.removeprefix(full_matched).lstrip("; \n") + continue + + # command until next separator + separator_position, separator_length = _find_first_top_level_command_separator(remaining_commands) + if separator_position is not None and separator_length is not None: + single_commands.append(remaining_commands[:separator_position].strip()) + remaining_commands = remaining_commands[separator_position + separator_length :].strip() + continue + + # single last command + single_commands.append(remaining_commands) + break + + return single_commands diff --git a/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py b/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py new file mode 100644 index 00000000..03f03ce6 --- /dev/null +++ b/sbom/sbom/cmd_graph/savedcmd_parser/savedcmd_parser.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +from typing import Any +import sbom.sbom_logging as sbom_logging +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.single_command_parsers import SINGLE_COMMAND_PARSERS +from sbom.cmd_graph.savedcmd_parser.tokenizer import CmdParsingError +from sbom.path_utils import PathStr + + +def parse_inputs_from_commands(commands: str, fail_on_unknown_build_command: bool) -> list[PathStr]: + """ + Extract input files referenced in a set of command-line commands. + + Args: + commands (str): Command line expression to parse. + fail_on_unknown_build_command (bool): Whether to fail if an unknown build command is encountered. If False, errors are logged as warnings. + + Returns: + list[PathStr]: List of input file paths required by the commands. + """ + + def log_error_or_warning(message: str, /, **kwargs: Any) -> None: + if fail_on_unknown_build_command: + sbom_logging.error(message, **kwargs) + else: + sbom_logging.warning(message, **kwargs) + + input_files: list[PathStr] = [] + for single_command in split_commands(commands): + if isinstance(single_command, IfBlock): + inputs = parse_inputs_from_commands(single_command.then_statement, fail_on_unknown_build_command) + if inputs: + log_error_or_warning( + "Skipped parsing command {then_statement} because input files in IfBlock 'then' statement are not supported", + then_statement=single_command.then_statement, + ) + continue + + matched_parser = next( + (parser for pattern, parser in SINGLE_COMMAND_PARSERS if pattern.match(single_command)), None + ) + if matched_parser is None: + log_error_or_warning( + "Skipped parsing command {single_command} because no matching parser was found", + single_command=single_command, + ) + continue + try: + inputs = matched_parser(single_command) + input_files.extend(inputs) + except CmdParsingError as e: + log_error_or_warning( + "Skipped parsing command {single_command} because of command parsing error: {error_message}", + single_command=single_command, + error_message=e.message, + ) + + return [input.strip().rstrip("/") for input in input_files] diff --git a/sbom/sbom/cmd_graph/savedcmd_parser.py b/sbom/sbom/cmd_graph/savedcmd_parser/single_command_parsers.py similarity index 60% rename from sbom/sbom/cmd_graph/savedcmd_parser.py rename to sbom/sbom/cmd_graph/savedcmd_parser/single_command_parsers.py index d72f781b..353db3d3 100644 --- a/sbom/sbom/cmd_graph/savedcmd_parser.py +++ b/sbom/sbom/cmd_graph/savedcmd_parser/single_command_parsers.py @@ -3,99 +3,19 @@ import re import shlex -from dataclasses import dataclass -from typing import Any, Callable, Union +from typing import Callable + import sbom.sbom_logging as sbom_logging +from sbom.cmd_graph.savedcmd_parser.command_splitter import IfBlock, split_commands +from sbom.cmd_graph.savedcmd_parser.tokenizer import ( + CmdParsingError, + Positional, + tokenize_single_command, + tokenize_single_command_positionals_only, +) from sbom.path_utils import PathStr -class CmdParsingError(Exception): - def __init__(self, message: str): - super().__init__(message) - self.message = message - - -@dataclass -class Option: - name: str - value: str | None = None - - -@dataclass -class Positional: - value: str - - -_SUBCOMMAND_PATTERN = re.compile(r"\$\$\(([^()]*)\)") -"""Pattern to match $$(...) blocks""" - - -def _tokenize_single_command(command: str, flag_options: list[str] | None = None) -> list[Union[Option, Positional]]: - """ - Parse a shell command into a list of Options and Positionals. - - Positional: the command and any positional arguments. - - Options: handles flags and options with values provided as space-separated, or equals-sign - (e.g., '--opt val', '--opt=val', '--flag'). - - Args: - command: Command line string. - flag_options: Options that are flags without values (e.g., '--verbose'). - - Returns: - List of `Option` and `Positional` objects in command order. - """ - - # Wrap all $$(...) blocks in double quotes to prevent shlex from splitting them. - command_with_protected_subcommands = _SUBCOMMAND_PATTERN.sub(lambda m: f'"$$({m.group(1)})"', command) - tokens = shlex.split(command_with_protected_subcommands) - - parsed: list[Option | Positional] = [] - i = 0 - while i < len(tokens): - token = tokens[i] - - # Positional - if not token.startswith("-"): - parsed.append(Positional(token)) - i += 1 - continue - - # Option without value (--flag) - if (token.startswith("-") and i + 1 < len(tokens) and tokens[i + 1].startswith("-")) or ( - flag_options and token in flag_options - ): - parsed.append(Option(name=token)) - i += 1 - continue - - # Option with equals sign (--opt=val) - if "=" in token: - name, value = token.split("=", 1) - parsed.append(Option(name=name, value=value)) - i += 1 - continue - - # Option with space-separated value (--opt val) - if i + 1 < len(tokens) and not tokens[i + 1].startswith("-"): - parsed.append(Option(name=token, value=tokens[i + 1])) - i += 2 - continue - - raise CmdParsingError(f"Unrecognized token: {token} in command {command}") - - return parsed - - -def _tokenize_single_command_positionals_only(command: str) -> list[str]: - command_parts = _tokenize_single_command(command) - positionals = [p.value for p in command_parts if isinstance(p, Positional)] - if len(positionals) != len(command_parts): - raise CmdParsingError( - f"Invalid command format: expected positional arguments only but got options in command {command}." - ) - return positionals - - def _parse_dd_command(command: str) -> list[PathStr]: match = re.match(r"dd.*?if=(\S+)", command) if match: @@ -104,7 +24,7 @@ def _parse_dd_command(command: str) -> list[PathStr]: def _parse_cat_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["cat", input1, input2, ...] return [p for p in positionals[1:]] @@ -129,7 +49,7 @@ def _parse_compound_command(command: str) -> list[PathStr]: if match is None: raise CmdParsingError("No inner commands found for compound command") input_files: list[PathStr] = [] - inner_commands = _split_commands(match.group(1)) + inner_commands = split_commands(match.group(1)) for inner_command in inner_commands: if isinstance(inner_command, IfBlock): sbom_logging.error( @@ -157,7 +77,7 @@ def _parse_compound_command(command: str) -> list[PathStr]: def _parse_objcopy_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command, flag_options=["-S", "-w"]) + command_parts = tokenize_single_command(command, flag_options=["-S", "-w"]) positionals = [part.value for part in command_parts if isinstance(part, Positional)] # expect positionals to be ['objcopy', input_file] or ['objcopy', input_file, output_file] if not (len(positionals) == 2 or len(positionals) == 3): @@ -184,7 +104,7 @@ def _parse_noop(command: str) -> list[PathStr]: def _parse_ar_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ['ar', flags, output, input1, input2, ...] flags = positionals[1] if "r" not in flags: @@ -196,7 +116,7 @@ def _parse_ar_command(command: str) -> list[PathStr]: def _parse_ar_piped_xargs_command(command: str) -> list[PathStr]: printf_command, _ = command.split("|", 1) - positionals = _tokenize_single_command_positionals_only(printf_command.strip()) + positionals = tokenize_single_command_positionals_only(printf_command.strip()) # expect positionals to be ['printf', '{prefix_path}%s ', input1, input2, ...] prefix_path = positionals[1].rstrip("%s ") return [f"{prefix_path}{filename}" for filename in positionals[2:]] @@ -232,52 +152,52 @@ def _parse_rustdoc_command(command: str) -> list[PathStr]: def _parse_syscallhdr_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command.strip(), flag_options=["--emit-nr"]) + command_parts = tokenize_single_command(command.strip(), flag_options=["--emit-nr"]) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be ["sh", path/to/syscallhdr.sh, input, output] return [positionals[2]] def _parse_syscalltbl_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command.strip()) + command_parts = tokenize_single_command(command.strip()) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be ["sh", path/to/syscalltbl.sh, input, output] return [positionals[2]] def _parse_mkcapflags_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["sh", path/to/mkcapflags.sh, output, input1, input2] return [positionals[3], positionals[4]] def _parse_orc_hash_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["sh", path/to/orc_hash.sh, '<', input, '>', output] return [positionals[3]] def _parse_xen_hypercalls_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["sh", path/to/xen-hypercalls.sh, output, input1, input2, ...] return positionals[3:] def _parse_gen_initramfs_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command) + command_parts = tokenize_single_command(command) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be ["sh", path/to/gen_initramfs.sh, input1, input2, ...] return positionals[2:] def _parse_vdso2c_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ['vdso2c', raw_input, stripped_input, output] return [positionals[1], positionals[2]] def _parse_ld_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command( + command_parts = tokenize_single_command( command=command.strip(), flag_options=[ "-shared", @@ -310,7 +230,7 @@ def _parse_sed_command(command: str) -> list[PathStr]: def _parse_awk(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command) + command_parts = tokenize_single_command(command) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be ["awk", input1, input2, ...] return positionals[1:] @@ -318,7 +238,7 @@ def _parse_awk(command: str) -> list[PathStr]: def _parse_nm_piped_command(command: str) -> list[PathStr]: nm_command, _ = command.split("|", 1) - command_parts = _tokenize_single_command( + command_parts = tokenize_single_command( command=nm_command.strip(), flag_options=["p", "--defined-only"], ) @@ -334,19 +254,19 @@ def _parse_pnm_to_logo_command(command: str) -> list[PathStr]: def _parse_relacheck(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["relachek", input, log_reference] return [positionals[1]] def _parse_perl_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command.strip()) + positionals = tokenize_single_command_positionals_only(command.strip()) # expect positionals to be ["perl", input] return [positionals[1]] def _parse_strip_command(command: str) -> list[PathStr]: - command_parts = _tokenize_single_command(command, flag_options=["--strip-debug"]) + command_parts = tokenize_single_command(command, flag_options=["--strip-debug"]) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be ["strip", input1, input2, ...] return positionals[1:] @@ -354,7 +274,7 @@ def _parse_strip_command(command: str) -> list[PathStr]: def _parse_mkpiggy_command(command: str) -> list[PathStr]: mkpiggy_command, _ = command.split(">", 1) - positionals = _tokenize_single_command_positionals_only(mkpiggy_command) + positionals = tokenize_single_command_positionals_only(mkpiggy_command) # expect positionals to be ["mkpiggy", input] return [positionals[1]] @@ -371,7 +291,7 @@ def _parse_relocs_command(command: str) -> list[PathStr]: def _parse_mk_elfconfig_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["mk_elfconfig", "<", input, ">", output] return [positionals[2]] @@ -395,7 +315,7 @@ def _parse_bison_command(command: str) -> list[PathStr]: def _parse_tools_build_command(command: str) -> list[PathStr]: - positionals = _tokenize_single_command_positionals_only(command) + positionals = tokenize_single_command_positionals_only(command) # expect positionals to be ["tools/build", "input1", "input2", "input3", "output"] return positionals[1:-1] @@ -411,7 +331,7 @@ def _parse_extract_cert_command(command: str) -> list[PathStr]: def _parse_dtc_command(command: str) -> list[PathStr]: wno_flags = [command_part for command_part in shlex.split(command) if command_part.startswith("-Wno-")] - command_parts = _tokenize_single_command(command, flag_options=wno_flags) + command_parts = tokenize_single_command(command, flag_options=wno_flags) positionals = [p.value for p in command_parts if isinstance(p, Positional)] # expect positionals to be [path/to/dtc, input] return [positionals[1]] @@ -492,173 +412,3 @@ def _parse_gen_header(command: str) -> list[PathStr]: (re.compile(r"^(.*/)?gen_header.py"), _parse_gen_header), (re.compile(r"^(.*/)?scripts/rustdoc_test_gen"), _parse_noop), ] - - -# If Block pattern to match a simple, single-level if-then-fi block. Nested If blocks are not supported. -IF_BLOCK_PATTERN = re.compile( - r""" - ^if(.*?);\s* # Match 'if ;' (non-greedy) - then(.*?);\s* # Match 'then ;' (non-greedy) - fi\b # Match 'fi' - """, - re.VERBOSE, -) - - -@dataclass -class IfBlock: - condition: str - then_statement: str - - -def _unwrap_outer_parentheses(s: str) -> str: - s = s.strip() - if not (s.startswith("(") and s.endswith(")")): - return s - - count = 0 - for i, char in enumerate(s): - if char == "(": - count += 1 - elif char == ")": - count -= 1 - # If count is 0 before the end, outer parentheses don't match - if count == 0 and i != len(s) - 1: - return s - - # outer parentheses do match, unwrap once - return _unwrap_outer_parentheses(s[1:-1]) - - -def _find_first_top_level_command_separator( - commands: str, separators: list[str] = [";", "&&"] -) -> tuple[int | None, int | None]: - in_single_quote = False - in_double_quote = False - in_curly_braces = 0 - in_braces = 0 - for i, char in enumerate(commands): - if char == "'" and not in_double_quote: - # Toggle single quote state (unless inside double quotes) - in_single_quote = not in_single_quote - elif char == '"' and not in_single_quote: - # Toggle double quote state (unless inside single quotes) - in_double_quote = not in_double_quote - - if in_single_quote or in_double_quote: - continue - - # Toggle braces state - if char == "{": - in_curly_braces += 1 - if char == "}": - in_curly_braces -= 1 - - if char == "(": - in_braces += 1 - if char == ")": - in_braces -= 1 - - if in_curly_braces > 0 or in_braces > 0: - continue - - # return found separator position and separator length - for separator in separators: - if commands[i : i + len(separator)] == separator: - return i, len(separator) - - return None, None - - -def _split_commands(commands: str) -> list[str | IfBlock]: - """ - Splits a string of command-line commands into individual parts. - - This function handles: - - Top-level command separators (e.g., `;` and `&&`) to split multiple commands. - - Conditional if-blocks, returning them as `IfBlock` instances. - - Preserves the order of commands and trims whitespace. - - Args: - commands (str): The raw command string. - - Returns: - list[str | IfBlock]: A list of single commands or `IfBlock` objects. - """ - single_commands: list[str | IfBlock] = [] - remaining_commands = _unwrap_outer_parentheses(commands) - while len(remaining_commands) > 0: - remaining_commands = remaining_commands.strip() - - # if block - matched_if = IF_BLOCK_PATTERN.match(remaining_commands) - if matched_if: - condition, then_statement = matched_if.groups() - single_commands.append(IfBlock(condition.strip(), then_statement.strip())) - full_matched = matched_if.group(0) - remaining_commands = remaining_commands.removeprefix(full_matched).lstrip("; \n") - continue - - # command until next separator - separator_position, separator_length = _find_first_top_level_command_separator(remaining_commands) - if separator_position is not None and separator_length is not None: - single_commands.append(remaining_commands[:separator_position].strip()) - remaining_commands = remaining_commands[separator_position + separator_length :].strip() - continue - - # single last command - single_commands.append(remaining_commands) - break - - return single_commands - - -def parse_inputs_from_commands(commands: str, fail_on_unknown_build_command: bool) -> list[PathStr]: - """ - Extract input files referenced in a set of command-line commands. - - Args: - commands (str): Command line expression to parse. - fail_on_unknown_build_command (bool): Whether to fail if an unknown build command is encountered. If False, errors are logged as warnings. - - Returns: - list[PathStr]: List of input file paths required by the commands. - """ - - def log_error_or_warning(message: str, /, **kwargs: Any) -> None: - if fail_on_unknown_build_command: - sbom_logging.error(message, **kwargs) - else: - sbom_logging.warning(message, **kwargs) - - input_files: list[PathStr] = [] - for single_command in _split_commands(commands): - if isinstance(single_command, IfBlock): - inputs = parse_inputs_from_commands(single_command.then_statement, fail_on_unknown_build_command) - if inputs: - log_error_or_warning( - "Skipped parsing command {then_statement} because input files in IfBlock 'then' statement are not supported", - then_statement=single_command.then_statement, - ) - continue - - matched_parser = next( - (parser for pattern, parser in SINGLE_COMMAND_PARSERS if pattern.match(single_command)), None - ) - if matched_parser is None: - log_error_or_warning( - "Skipped parsing command {single_command} because no matching parser was found", - single_command=single_command, - ) - continue - try: - inputs = matched_parser(single_command) - input_files.extend(inputs) - except CmdParsingError as e: - log_error_or_warning( - "Skipped parsing command {single_command} because of command parsing error: {error_message}", - single_command=single_command, - error_message=e.message, - ) - - return [input.strip().rstrip("/") for input in input_files] diff --git a/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py b/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py new file mode 100644 index 00000000..09ae2d6e --- /dev/null +++ b/sbom/sbom/cmd_graph/savedcmd_parser/tokenizer.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright (C) 2025 TNG Technology Consulting GmbH + +import re +import shlex +from dataclasses import dataclass +from typing import Union + + +class CmdParsingError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.message = message + + +@dataclass +class Option: + name: str + value: str | None = None + + +@dataclass +class Positional: + value: str + + +_SUBCOMMAND_PATTERN = re.compile(r"\$\$\(([^()]*)\)") +"""Pattern to match $$(...) blocks""" + + +def tokenize_single_command(command: str, flag_options: list[str] | None = None) -> list[Union[Option, Positional]]: + """ + Parse a shell command into a list of Options and Positionals. + - Positional: the command and any positional arguments. + - Options: handles flags and options with values provided as space-separated, or equals-sign + (e.g., '--opt val', '--opt=val', '--flag'). + + Args: + command: Command line string. + flag_options: Options that are flags without values (e.g., '--verbose'). + + Returns: + List of `Option` and `Positional` objects in command order. + """ + + # Wrap all $$(...) blocks in double quotes to prevent shlex from splitting them. + command_with_protected_subcommands = _SUBCOMMAND_PATTERN.sub(lambda m: f'"$$({m.group(1)})"', command) + tokens = shlex.split(command_with_protected_subcommands) + + parsed: list[Option | Positional] = [] + i = 0 + while i < len(tokens): + token = tokens[i] + + # Positional + if not token.startswith("-"): + parsed.append(Positional(token)) + i += 1 + continue + + # Option without value (--flag) + if (token.startswith("-") and i + 1 < len(tokens) and tokens[i + 1].startswith("-")) or ( + flag_options and token in flag_options + ): + parsed.append(Option(name=token)) + i += 1 + continue + + # Option with equals sign (--opt=val) + if "=" in token: + name, value = token.split("=", 1) + parsed.append(Option(name=name, value=value)) + i += 1 + continue + + # Option with space-separated value (--opt val) + if i + 1 < len(tokens) and not tokens[i + 1].startswith("-"): + parsed.append(Option(name=token, value=tokens[i + 1])) + i += 2 + continue + + raise CmdParsingError(f"Unrecognized token: {token} in command {command}") + + return parsed + + +def tokenize_single_command_positionals_only(command: str) -> list[str]: + command_parts = tokenize_single_command(command) + positionals = [p.value for p in command_parts if isinstance(p, Positional)] + if len(positionals) != len(command_parts): + raise CmdParsingError( + f"Invalid command format: expected positional arguments only but got options in command {command}." + ) + return positionals