diff --git a/README.md b/README.md index b42791d..01f80bf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

The basic CLI interface of the Grinder Framework

-## Contents +## Table of Contents 1. [Description](#description) 1. [Grinder Workflow](#grinder-workflow) 1. [Grinder Map](#grinder-map) @@ -89,6 +89,7 @@ For example, the hosts will be automatically checked for availability with ping ### Basic - :heavy_exclamation_mark: [Python 3.6+](https://www.python.org/downloads/) - :heavy_exclamation_mark: [python3-tk](https://docs.python.org/3/library/tkinter.html) library +- :heavy_exclamation_mark: [FreeType](https://www.freetype.org/) library (Python 3.8+ and MacOS required) ### Accounts - :heavy_exclamation_mark: [Shodan](https://account.shodan.io/register) and [Censys](https://censys.io/register) accounts Required to collect hosts, both free and full accounts are suitable. Also, it's possible to use only one account (Censys or Shodan, Shodan is preferable). diff --git a/grinder.py b/grinder.py index 584dd34..9d713c5 100755 --- a/grinder.py +++ b/grinder.py @@ -2,11 +2,28 @@ import sys +from multiprocessing import freeze_support + from grinder.asciiart import AsciiOpener -from grinder.core import GrinderCore from grinder.interface import GrinderInterface + +class GrinderProcessWrap: + """ + Fix Processes "RuntimeError" related + to bootstrapping phase (MacOS case) + """ + + @staticmethod + def import_core(): + freeze_support() + from grinder.core import GrinderCore as _core + + return _core + + if __name__ == "__main__": + GrinderCore = GrinderProcessWrap.import_core() AsciiOpener.print_opener() interface = GrinderInterface() interface.check_python_version() @@ -34,8 +51,7 @@ search_results = ( core.batch_search( - queries_filename=args.queries_file, - not_incremental=args.not_incremental + queries_filename=args.queries_file, not_incremental=args.not_incremental ) if args.run else core.load_results(queries_filename=args.queries_file) diff --git a/grinder/core.py b/grinder/core.py index fca1ea9..ffc60bc 100644 --- a/grinder/core.py +++ b/grinder/core.py @@ -1355,7 +1355,7 @@ def vulners_scan( # Check for top-ports if defined arguments = ( - f"-Pn -sV --script=.{vulners_path} --host-timeout {str(host_timeout)}s" + f"-Pn -sV --open --script=.{vulners_path} --host-timeout {str(host_timeout)}s" ) if top_ports: arguments = f"{arguments} --top-ports {str(top_ports)}" diff --git a/grinder/decorators.py b/grinder/decorators.py index a633f9f..4f861ed 100644 --- a/grinder/decorators.py +++ b/grinder/decorators.py @@ -2,7 +2,7 @@ import sys from functools import wraps -from os import path, makedirs, system +from os import system from time import time, strftime, gmtime diff --git a/grinder/defaultvalues.py b/grinder/defaultvalues.py index 7a685ad..5a91ef9 100644 --- a/grinder/defaultvalues.py +++ b/grinder/defaultvalues.py @@ -132,7 +132,7 @@ class DefaultNmapScanValues: TOP_PORTS = None SUDO = False HOST_TIMEOUT = 30 - ARGUMENTS = "-Pn -T4 -A" + ARGUMENTS = "-Pn -T4 -A --open" WORKERS = 10 @@ -143,7 +143,7 @@ class DefaultProcessManagerValues: PORTS = None SUDO = False - ARGUMENTS = "-Pn -A" + ARGUMENTS = "-Pn -A --open" WORKERS = 10 diff --git a/grinder/nmapprocessmanager.py b/grinder/nmapprocessmanager.py index 0d9e975..5193a0d 100644 --- a/grinder/nmapprocessmanager.py +++ b/grinder/nmapprocessmanager.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -from multiprocessing import Process, JoinableQueue, Manager +from datetime import datetime +from multiprocessing import Process, JoinableQueue, Manager, freeze_support from os import system from time import sleep -from datetime import datetime from grinder.decorators import exception_handler +from grinder.defaultvalues import DefaultProcessManagerValues from grinder.errors import ( NmapProcessingRunError, NmapProcessingManagerOrganizeProcessesError, ) from grinder.nmapconnector import NmapConnector -from grinder.defaultvalues import DefaultProcessManagerValues class NmapProcessingDefaultManagerValues: @@ -23,15 +23,6 @@ class NmapProcessingDefaultManagerValues: EMPTY_QUEUE_POLLING_RATE = 1.0 -class NmapProcessingResults: - """ - This is results collector to gain - results directly from a process - """ - - RESULTS = Manager().dict({}) - - class NmapProcessing(Process): """ Create custom Nmap process. The reason to create @@ -49,6 +40,7 @@ def __init__( ports: str, sudo: bool, hosts_quantity: int, + results_pool: dict, ): Process.__init__(self) self.queue = queue @@ -56,6 +48,7 @@ def __init__( self.ports = ports self.sudo = sudo self.quantity = hosts_quantity + self.results_pool = results_pool @exception_handler(expected_exception=NmapProcessingRunError) def run(self) -> None: @@ -74,9 +67,15 @@ def run(self) -> None: sleep(NmapProcessingDefaultManagerValues.EMPTY_QUEUE_POLLING_RATE) continue try: - index, host = self.queue.get() + # Poll with POLLING_RATE interval sleep(NmapProcessingDefaultManagerValues.POLLING_RATE) + # Get host info from queue + index, host = self.queue.get() + if (index, host) == (None, None): + self.queue.task_done() + return + host_ip = host.get("ip", "") host_port = host.get("port", "") port_postfix = "Default" @@ -103,13 +102,12 @@ def run(self) -> None: results = nm.get_results() if results.get(host_ip).values(): - NmapProcessingResults.RESULTS.update( - {host_ip: results.get(host_ip)} - ) + self.results_pool.update({host_ip: results.get(host_ip)}) except: - self.queue.task_done() - else: - self.queue.task_done() + pass + self.queue.task_done() + if self.queue.empty(): + return class NmapProcessingManager: @@ -127,6 +125,9 @@ def __init__( arguments=DefaultProcessManagerValues.ARGUMENTS, workers=DefaultProcessManagerValues.WORKERS, ): + freeze_support() + self.manager = Manager() + self.results_pool = self.manager.dict({}) self.hosts = hosts self.workers = workers self.arguments = arguments @@ -140,18 +141,32 @@ def organize_processes(self) -> None: :return: None """ queue = JoinableQueue() - for index, host in enumerate(self.hosts): - queue.put((index, host)) processes = [] for _ in range(self.workers): + freeze_support() process = NmapProcessing( - queue, self.arguments, self.ports, self.sudo, len(self.hosts) + queue, + self.arguments, + self.ports, + self.sudo, + len(self.hosts), + self.results_pool, ) process.daemon = True processes.append(process) for process in processes: - process.start() + try: + process.start() + except OSError: + pass + for index, host in enumerate(self.hosts): + queue.put((index, host)) + for _ in range(self.workers): + queue.put((None, None)) queue.join() + for process in processes: + if process.is_alive(): + process.terminate() def start(self) -> None: """ @@ -160,21 +175,19 @@ def start(self) -> None: """ self.organize_processes() - @staticmethod - def get_results() -> dict: + def get_results(self) -> dict: """ Return dictionary with Nmap results :return: Nmap results """ - return NmapProcessingResults.RESULTS + return self.results_pool - @staticmethod - def get_results_count() -> int: + def get_results_count(self) -> int: """ Return quantity of Nmap results :return: quantity of results """ - return len(NmapProcessingResults.RESULTS) + return len(self.results_pool) def __del__(self): """ diff --git a/grinder/pyscriptexecutor.py b/grinder/pyscriptexecutor.py index 00aeacd..86102cf 100644 --- a/grinder/pyscriptexecutor.py +++ b/grinder/pyscriptexecutor.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -from multiprocessing import Process, JoinableQueue, Manager -from os import system +from contextlib import redirect_stdout, redirect_stderr from importlib.machinery import SourceFileLoader -from types import ModuleType +from multiprocessing import Process, JoinableQueue, Manager, freeze_support +from os import system from pathlib import Path -from contextlib import redirect_stdout, redirect_stderr from time import sleep +from types import ModuleType from grinder.decorators import exception_handler from grinder.defaultvalues import DefaultValues @@ -26,30 +26,24 @@ class PyProcessingValues: EMPTY_QUEUE_POLLING_RATE = 1.0 -class PyProcessingResults: - """ - Collect python scripts results - """ - - RESULTS = Manager().dict({}) - - class PyProcessing(Process): """ Create a custom process to run python scripts as an independent worker """ - def __init__(self, queue: JoinableQueue, mute: bool = False): + def __init__(self, results_pool: dict, queue: JoinableQueue, mute: bool = False): """ Initialize the process worker :param queue: general joinable task queue :param mute: bool flag for running scripts in silent mode (w/o output at all) + :param results_pool: pool of results """ Process.__init__(self) self.queue = queue self.mute = mute self.base_path = self._initialize_base_path() + self.results_pool = results_pool @staticmethod def _initialize_base_path() -> Path: @@ -108,13 +102,18 @@ def run(self) -> None: "Unknown script", ) try: + # Poll with POLLING_RATE interval + sleep(PyProcessingValues.POLLING_RATE) + + # Get host info from queue current_progress, host_info, py_script = self.queue.get() + if (current_progress, host_info, py_script) == (None, None, None): + self.queue.task_done() + return + ip = host_info.get("ip") port = host_info.get("port") - # Poll with POLLING_RATE interval - sleep(PyProcessingValues.POLLING_RATE) - # Setup logging log_progress = f"[{current_progress[0]}/{current_progress[1]}] ({current_progress[2]})" log_host = f"{ip}:{port}" @@ -128,12 +127,12 @@ def run(self) -> None: self.queue.task_done() continue try: - if ip not in PyProcessingResults.RESULTS.keys(): - PyProcessingResults.RESULTS.update({ip: result}) + if ip not in self.results_pool.keys(): + self.results_pool.update({ip: result}) else: - old_result = PyProcessingResults.RESULTS.get(ip) + old_result = self.results_pool.get(ip) old_result.update(result) - PyProcessingResults.RESULTS[ip] = old_result + self.results_pool[ip] = old_result except (AttributeError, ConnectionRefusedError): print( f"{log_progress} -> Caught manager error on host {log_host}: simultaneous shared-dict call" @@ -144,10 +143,11 @@ def run(self) -> None: print( f'{log_progress} -> script "{py_script}" crash for {log_host}: {str(script_err)}' ) - self.queue.task_done() else: print(f'{log_progress} -> script "{py_script}" done for {log_host}') - self.queue.task_done() + self.queue.task_done() + if self.queue.empty(): + return class PyProcessingManager: @@ -170,6 +170,9 @@ def __init__( :param workers: number of running processes :param mute: bool flag for running scripts in silent mode (w/o output at all) """ + freeze_support() + self.manager = Manager() + self.results_pool = self.manager.dict({}) self.ip_script_mapping = ip_script_mapping self.hosts_info = hosts_info self.workers = workers @@ -183,10 +186,19 @@ def organize_processes(self) -> None: :return: None """ queue = JoinableQueue() + processes = [] for _ in range(self.workers): - process = PyProcessing(queue, mute=self.mute) + freeze_support() + process = PyProcessing( + results_pool=self.results_pool, queue=queue, mute=self.mute + ) process.daemon = True - process.start() + processes.append(process) + for process in processes: + try: + process.start() + except OSError: + pass hosts_length = len(self.hosts_info) for index, (ip, host_info) in enumerate(self.hosts_info.items()): py_script = self.ip_script_mapping.get(ip) @@ -221,7 +233,12 @@ def organize_processes(self) -> None: script_file, ) ) + for _ in range(self.workers): + queue.put((None, None, None)) queue.join() + for process in processes: + if process.is_alive(): + process.terminate() def start(self) -> None: """ @@ -231,23 +248,21 @@ def start(self) -> None: """ self.organize_processes() - @staticmethod - def get_results() -> dict: + def get_results(self) -> dict: """ Return process manager results :return: dictionary with {ip: results} format """ - return PyProcessingResults.RESULTS + return self.results_pool - @staticmethod - def get_results_count() -> int: + def get_results_count(self) -> int: """ Return overall quantity of results :return: None """ - return len(PyProcessingResults.RESULTS) + return len(self.results_pool) def __del__(self) -> None: """