Changing magics creation so that they don't get created if not in ipy…

…thon (#332)

Adding import of magics to nbinit and removing from __init__ and Pivot class.
Updating docs (including some auto-gen'd)

docs/source/api/msticpy.data.drivers.rst

@@ -4,6 +4,14 @@ msticpy.data.drivers package
 Submodules
 ----------
 
+msticpy.data.drivers.cybereason\_driver module
+----------------------------------------------
+
+.. automodule:: msticpy.data.drivers.cybereason_driver
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 msticpy.data.drivers.driver\_base module
 ----------------------------------------
 

docs/source/api/msticpy.datamodel.rst

@@ -85,14 +85,6 @@ msticpy.datamodel.pivot\_ti\_provider module
    :undoc-members:
    :show-inheritance:
 
-msticpy.datamodel.txt\_df\_magic module
----------------------------------------
-
-.. automodule:: msticpy.datamodel.txt_df_magic
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 Module contents
 ---------------
 

docs/source/api/msticpy.datamodel.soc.rst

@@ -12,6 +12,14 @@ msticpy.datamodel.soc.incident module
    :undoc-members:
    :show-inheritance:
 
+msticpy.datamodel.soc.sentinel\_alert module
+--------------------------------------------
+
+.. automodule:: msticpy.datamodel.soc.sentinel_alert
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/source/api/msticpy.nbtools.rst

@@ -68,6 +68,14 @@ msticpy.nbtools.nbinit module
    :undoc-members:
    :show-inheritance:
 
+msticpy.nbtools.nbmagics module
+-------------------------------
+
+.. automodule:: msticpy.nbtools.nbmagics
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 msticpy.nbtools.observationlist module
 --------------------------------------
 

docs/source/api/msticpy.sectools.rst

@@ -117,14 +117,6 @@ msticpy.sectools.process\_tree\_utils module
    :undoc-members:
    :show-inheritance:
 
-msticpy.sectools.sectools\_magics module
-----------------------------------------
-
-.. automodule:: msticpy.sectools.sectools_magics
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 msticpy.sectools.syslog\_utils module
 -------------------------------------
 

msticpy/common/utility.py

@@ -338,18 +338,26 @@ def md_error(string: str, disp_id: Optional[DisplayHandle] = None):
 
 
 @export
-def is_ipython() -> bool:
+def is_ipython(notebook: bool = False) -> bool:
     """
     Return True if running in IPython environment.
 
+    Parameters
+    ----------
+    notebook : bool, optional
+        If notebook is true this will only return true if running
+        in a Jupyter notebook.
+
     Returns
     -------
     bool
         True if running in IPython environment,
         otherwise False
 
     """
-    return bool(get_ipython())
+    if not notebook:
+        return bool(get_ipython())
+    return get_ipython() and type(get_ipython()).__name__.startswith("ZMQ")
 
 
 def check_kwarg(arg_name: str, legal_args: List[str]):

msticpy/datamodel/entities/entity.py

@@ -86,7 +86,7 @@ def __init__(self, src_entity: Mapping[str, Any] = None, **kwargs):
         self.TimeGenerated = datetime.utcnow()
         self.Type = self._get_entity_type_name(type(self))
         # If we have an unknown entity see if we a type passed in
-        if self.Type == "unknownentity" and "Type" in kwargs:
+        if self.Type == "unknown" and "Type" in kwargs:
             self.Type = kwargs["Type"]
         # Make sure Type is in the class schema dictionary
         self._entity_schema["Type"] = None
@@ -521,8 +521,8 @@ def _get_entity_type_name(cls, entity_type: Type) -> str:
                 )
             )
         except StopIteration:
-            name = None
-        return name or "unknown"
+            name = "unknown"
+        return name
 
     @property
     def node_properties(self) -> Dict[str, Any]:

msticpy/datamodel/pivot.py

@@ -8,7 +8,6 @@
 from typing import Any, Callable, Dict, Iterable, Optional, Type
 
 import pkg_resources
-from IPython import get_ipython
 
 from .._version import VERSION
 from ..common.timespan import TimeSpan
@@ -31,11 +30,6 @@
 _DEF_PIVOT_REG_FILE = "resources/mp_pivot_reg.yaml"
 
 
-# Import IPython magic if in an IPython environment
-if get_ipython():
-    from . import txt_df_magic  # noqa: F401
-
-
 class Pivot:
     """Pivot environment loader."""
 

msticpy/nbtools/nbinit.py

@@ -154,6 +154,7 @@ def _verbose(verbosity: Optional[int] = None) -> int:
     dict(pkg="msticpy.common.wsconfig", tgt="WorkspaceConfig"),
     dict(pkg="msticpy.datamodel.pivot", tgt="Pivot"),
     dict(pkg="msticpy.datamodel", tgt="entities"),
+    dict(pkg="msticpy.nbtools", tgt="nbmagics"),
     dict(pkg="msticpy.vis", tgt="mp_pandas_plot"),
 ]
 _MP_IMPORT_ALL = [

msticpy/nbtools/nbmagics.py

@@ -0,0 +1,167 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""msticpy IPython magics."""
+import re
+from typing import Tuple, List
+
+# pylint: disable=unused-import
+# flake8: noqa: F403
+import pandas as pd
+
+# pylint: enable=unused-import
+from IPython import get_ipython
+from IPython.core import magic_arguments
+from IPython.core.magic import line_cell_magic, Magics, magics_class
+from IPython.core.magic import needs_local_scope, register_cell_magic
+from ..datamodel.pivot_magic_core import run_txt2df
+
+try:
+    from bs4 import BeautifulSoup
+
+    _BS_AVAILABLE = True
+except ImportError:
+    _BS_AVAILABLE = False
+
+from ..sectools import base64unpack as base64
+from ..sectools.iocextract import IoCExtract
+from ..common.utility import is_ipython
+
+from .._version import VERSION
+
+__version__ = VERSION
+__author__ = "Ian Hellen"
+
+
+if is_ipython():
+
+    @magics_class
+    class Base64Magic(Magics):
+        """Base64 IPython magic extension."""
+
+        _STRIP_TAGS = r"</?decoded[^>]*>"
+
+        @line_cell_magic
+        @magic_arguments.magic_arguments()
+        @magic_arguments.argument(
+            "--out", "-o", help="The variable to return the results in"
+        )
+        @magic_arguments.argument(
+            "--pretty",
+            "-p",
+            help="Print formatted version of output",
+            action="store_true",
+        )
+        @magic_arguments.argument(
+            "--clean",
+            "-c",
+            help="Print decoded string with no formatting",
+            action="store_true",
+        )
+        def b64(self, line: str = "", cell: str = None) -> str:
+            """
+            Base64 IPython magic extension.
+
+            Parameters
+            ----------
+            line : str, optional
+                Line contents, by default ""
+            cell : str, optional
+                Cell contents, by default None
+
+            Returns
+            -------
+            str
+                Decoded text
+
+            """
+            if cell is None:
+                results, df_results = base64.unpack(line)
+
+            else:
+                results, df_results = base64.unpack(cell)
+            args = magic_arguments.parse_argstring(self.b64, line)
+
+            if args.clean:
+                results = re.sub(self._STRIP_TAGS, "", results)
+            elif args.pretty:
+                if _BS_AVAILABLE:
+                    xml_str = f"<decoded_string>{results}</decoded_string>"
+                    b_soup = BeautifulSoup(xml_str, "xml")
+                    results = b_soup.prettify()
+            if args.out is not None:
+                self.shell.user_ns[args.out] = (results, df_results)
+            return results
+
+    @magics_class
+    class IoCExtractMagic(Magics):
+        """Ioc Extract IPython magic extension."""
+
+        def __init__(self, shell):
+            """
+            Instantiate magic class.
+
+            Parameters
+            ----------
+            shell : IPython shell
+                IPython shell
+
+            """
+            # You must call the parent constructor
+            super().__init__(shell)
+            self._ioc_extract = IoCExtract()
+
+        @line_cell_magic
+        @magic_arguments.magic_arguments()
+        @magic_arguments.argument(
+            "--out", "-o", help="The variable to return the results in"
+        )
+        @magic_arguments.argument(
+            "--ioc_types",
+            "-i",
+            help="The types of IoC to search for (comma-separated string)",
+        )
+        def ioc(self, line="", cell=None) -> List[Tuple[str, List[str]]]:
+            """
+            Ioc Extract IPython magic extension.
+
+            Parameters
+            ----------
+            line : str, optional
+                Line contents, by default ""
+            cell : str, optional
+                Cell contents, by default None
+
+            Returns
+            -------
+            List[Tuple[str, List[str]]]
+                List of tuples of IoCs found grouped by type.
+
+            """
+            args = magic_arguments.parse_argstring(self.ioc, line)
+            ioc_types = None
+            if args.ioc_types:
+                ioc_types = [ioc_type.strip() for ioc_type in args.ioc_types.split(",")]
+
+            if cell is None:
+                results = self._ioc_extract.extract(src=line, ioc_types=ioc_types)
+            else:
+                results = self._ioc_extract.extract(src=cell, ioc_types=ioc_types)
+            iocs = [(ioc_type, list(ioc_res)) for ioc_type, ioc_res in results.items()]
+
+            if args.out is not None:
+                self.shell.user_ns[args.out] = results
+            return iocs
+
+    IPYTHON = get_ipython()
+    if IPYTHON:
+        IPYTHON.register_magics(Base64Magic)
+        IPYTHON.register_magics(IoCExtractMagic)
+
+    @register_cell_magic
+    @needs_local_scope
+    def txt2df(line, cell, local_ns):
+        """Convert cell text to pandas DataFrame."""
+        return run_txt2df(line, cell, local_ns)

msticpy/sectools/__init__.py

@@ -15,10 +15,4 @@
 from . import process_tree_utils as ptree
 from .._version import VERSION
 
-try:
-    from IPython import get_ipython
-    from . import sectools_magics
-except ImportError as err:
-    pass
-
 __version__ = VERSION