Merge branch 'main' into ianhelle/ws-config-fix-2023-09-01

docs/source/api/msticpy.auth.cloud_mappings_offline.rst

@@ -0,0 +1,7 @@
+msticpy.auth.cloud\_mappings\_offline module
+============================================
+
+.. automodule:: msticpy.auth.cloud_mappings_offline
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/msticpy.auth.rst

@@ -15,6 +15,7 @@ Submodules
    msticpy.auth.azure_auth
    msticpy.auth.azure_auth_core
    msticpy.auth.cloud_mappings
+   msticpy.auth.cloud_mappings_offline
    msticpy.auth.cred_wrapper
    msticpy.auth.keyring_client
    msticpy.auth.keyvault_client

docs/source/data_acquisition/SplunkProvider.rst

@@ -38,7 +38,7 @@ The settings in the file should look like the following:
       Splunk:
           Args:
             host: splunk_host
-            port: 8089
+            port: '8089'
             username: splunk_user
             password: [PLACEHOLDER]
 
@@ -54,7 +54,7 @@ to a Key Vault secret using the MSTICPy configuration editor.
       Splunk:
           Args:
             host: splunk_host
-            port: 8089
+            port: '8089'
             username: splunk_user
             password:
               KeyVault:
@@ -67,8 +67,13 @@ Parameter    Description
 host         (string) The host name (the default is 'localhost').
 username     (string) The Splunk account username, which is used to authenticate the Splunk instance.
 password     (string) The password for the Splunk account.
+splunkToken  (string) The Authorization Bearer Token <JWT> created in the Splunk.
 ===========  ===========================================================================================================================
 
+The username and password are needed for user account authentication.
+On the other hand, splunkToken is needed for Token authentication.
+The user auth method has a priority to token auth method if both username and splunkToken are set.
+
 
 Optional configuration parameters:
 
@@ -106,11 +111,11 @@ in msticpy config file.
 For more information on how to create new user with appropriate roles
 and permissions, follow the Splunk documents:
 
-`Securing the Spunk platform <https://docs.splunk.com/Documentation/Splunk/8.0.5/Security/Addandeditusers>`__
+`Securing the Spunk platform <https://docs.splunk.com/Documentation/Splunk/9.1.1/Security/Addandeditusers>`__
 
 and
 
-`About users and roles <https://docs.splunk.com/Documentation/Splunk/8.0.5/Security/Aboutusersandroles>`__.
+`About users and roles <https://docs.splunk.com/Documentation/Splunk/9.1.1/Security/Aboutusersandroles>`__
 
 The user should have permission to at least run its own searches or more
 depending upon the actions to be performed by user.
@@ -120,10 +125,20 @@ require the following details to specify while connecting:
 
 - host = "localhost" (Splunk server FQDN hostname to connect, for locally
   installed splunk, you can specify localhost)
-- port = 8089 (Splunk REST API )
+- port = "8089" (Splunk REST API)
 - username = "admin" (username to connect to Splunk instance)
 - password = "yourpassword" (password of the user specified in username)
 
+On the other hand, you can use the authentification token to connect.
+
+`Create authentication token <https://docs.splunk.com/Documentation/Splunk/9.1.1/Security/CreateAuthTokens>`__
+
+- host = "localhost" (Splunk server FQDN hostname to connect, for locally
+  installed splunk, you can specify localhost)
+- port = "8089" (Splunk REST API)
+- splunkToken = "<Authorization Bearer Token>" (token can be used instead of username/password)
+
+
 Once you have details, you can specify it in ``msticpyconfig.yaml`` as
 described earlier.
 
@@ -146,6 +161,11 @@ as parameters to connect.
 
     qry_prov.connect(host=<hostname>, username=<username>, password=<password>)
 
+OR
+
+.. code:: ipython3
+
+    qry_prov.connect(host=<hostname>, splunkToken=<token_string>)
 
 
 Listing available queries
@@ -217,7 +237,7 @@ For more information, see
         (default value is: | head 100)
     end: datetime (optional)
         Query end time
-        (default value is: 08/26/2017:00:00:00)
+        (default value is: current time + 1 day)
     index: str (optional)
         Splunk index name
         (default value is: \*)
@@ -229,7 +249,7 @@ For more information, see
         (default value is: \*)
     start: datetime (optional)
         Query start time
-        (default value is: 08/25/2017:00:00:00)
+        (default value is: current time - 1 day)
     timeformat: str (optional)
         Datetime format to use in Splunk query
         (default value is: "%Y-%m-%d %H:%M:%S.%6N")

docs/source/getting_started/Installing.rst

@@ -51,7 +51,7 @@ created and activated in the prompt.
 Installation
 ------------
 
-Run the following command to install the base configuation of *MSTICPy*.
+Run the following command to install the base configuration of *MSTICPy*.
 
 
 ``pip install msticpy``
@@ -71,7 +71,7 @@ known as extras. The syntax for this is:
 
 As of version 0.9.0 *MSTICPy* has its dependencies split into
 extras. This allows you to install only the packages that you
-need and avoid the overhead of time and diskspace of dependencies
+need and avoid the overhead of time and disk space of dependencies
 that you do not need.
 
 .. note:: extras do not affect the which code from *MSTICPy* is
@@ -191,7 +191,7 @@ exception message:
 
 .. code:: bash
 
-    pip install msticpy[ml]
+    python -m pip install msticpy[ml]
 
 .. note:: In some cases you many not get an informative error. We've
    tried to trap all of the cases but if
@@ -237,3 +237,148 @@ se, and choose the conda file saved earlier with the Spark session configuration
     - numpy
     - pip:
         - msticpy[azure]>=2.3.1
+
+Installing for isolated or semi-isolated environments
+-----------------------------------------------------
+
+There are instances in which you may want to use msticpy in an isolated
+or semi-isolated environment (e.g. an environment that does not have internet
+PyPI access to install packages and dependencies).
+
+To do this you need to build a transportable archive of MSTICPy and its
+dependencies and use that as the source to install from in your target environment.
+
+We have included a set of scripts to simplify some of this process. These
+are available in the `tools folder <https://github.com/microsoft/msticpy/tree/main/tools>`__
+of the MSTICPy repo.
+
+- ``build_wheel_from_targz.py`` - builds wheel files from source tar.gz files
+- ``install_all_whl_files.py`` - installs all .whl files in a directory to the target environment
+- ``download_python_package.py`` - downloads a python package and its dependencies to a directory.
+  This script uses docker to perform the download and allows you to build an install
+  package for a Linux environment from a Windows environment.
+
+In the instructions below we give both the manual steps and the script-based steps.
+The latter are recommended to avoid repetitious typing and to avoid missing files.
+
+For either manual or script-based installation it is **essential** that you
+use the same Python version for both source and target environments, since
+pip will download the correct version of the package for the Python version
+that it is executing in. We recommend creating a virtual Python or Conda
+environment for this purpose (this isn't required for the docker-based
+script).
+
+In order to find the correct python version, you can run the following:
+
+.. code-block:: powershell
+
+    python --version
+
+
+Windows Source to Isolated Windows Environment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. On your primary Windows machine with internet access create a virtual environment
+   for the python version you want to use in the target environment.
+2. Download msticpy by running the following:
+
+.. code-block:: powershell
+
+    python -m pip download msticpy --dest \path\to\destination
+
+Within ``\path\to\destination`` you should see a .whl file for msticpy and the other dependencies.
+Some dependencies may not be .whl files, but tar.gz files.
+These files will need to be built into .whl files. To do this, run the following
+for each tar.gz file:
+
+.. code-block:: powershell
+
+    python -m pip wheel {file.tar.gz} -w \path\to\destination
+
+or use the script from MSTICPy repo "tools" folder to process all files
+`build_wheel_from_targz.py <https://github.com/microsoft/msticpy/blob/main/tools/build_wheel_from_targz.py>`__
+to build all the tar.gz files in a directory.
+
+3. Zip and copy the directory folder to your target environment.
+
+4. From the Isolated environment, unzip if needed and then run the following for each .whl file:
+
+.. code-block:: powershell
+
+    python -m pip install "\path\to\destination\{whl_file.whl}"
+
+.. note:: If you have an issue installing any of the packages you can use the script from
+    the MSTICPy repo "tools" folder `install_all_whl_files.py <https://github.com/microsoft/msticpy/blob/main/tools/install_all_whl_files.py>`__
+    to help.
+
+5. Test the installation by running msticpy that suits your needs.
+
+
+Linux Source to Isolated Linux Environment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Follow the *Windows Source to Isolated Windows Environment* instructions above.
+
+
+Windows Source to Isolated Linux Environment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. On your source Windows machine with internet access, download
+   `Docker for Windows <https://docs.docker.com/desktop/install/windows-install/>`__.
+   We are using docker to ensure that the wheels that we are downloading are meant for the Linux architecture.
+
+2. Run the `download_python_package.py
+   <https://github.com/microsoft/msticpy/blob/main/tools/download_python_package.py>`__ script.
+
+Example:
+.. code-block:: powershell
+
+    python \path\to\python\file --python-version "3.8.5" --module-name "msticpy[sentinel]" --module-version "2.7.0" --directory \path\to\destination
+
+3. Zip and copy the directory folder to the isolated environment.
+
+4. From the isolated environment, unzip if needed and then you will need to run the following for each .whl file:
+
+.. code-block:: powershell
+
+    python -m pip install "\path\to\destination\{whl_file.whl}"
+
+.. note:: If you have an issue installing any of the packages you can use the script
+    from MSTICPy repo "tools" folder
+    `install_all_whl_files.py <https://github.com/microsoft/msticpy/blob/main/tools/install_all_whl_files.py>`__
+    to help.
+
+5. Test the installation by running some MSTICPy operations in a Jupyter notebook.
+
+If you are installing within a Jupyter Notebooks, you will need to upload your zip file/directory
+containing all of the whl files.
+
+If you zipped your transfer archive and need to unzip source files, run the following:
+
+.. code-block:: python
+
+    import zipfile
+    import os
+    import shutil
+    file_path =  "./{zip_file_name}"
+    file_name = os.path.split(file_path)[-1]
+    file_name_without_ext = os.path.splitext(file_name)[0]
+    with zipfile.ZipFile(file_path, 'r') as zip_ref:
+        zip_ref.extractall(os.path.join(os.getcwd(), file_name_without_ext))
+
+
+- To install the whl files, run the following in a cell:
+
+.. code-block:: python
+
+    import os
+    directory = "/path/to/whl/files/directory" # edit this to match your directory
+    files = [
+        os.path.join(directory, filename)
+        for filename in os.listdir(directory)
+        if filename.endswith(".whl")
+    ]
+    for file in files:
+        filename = os.path.split(file)[-1]
+        print(f"\nAttempting to install {filename}")
+        %pip install --quiet --no-index --no-deps --find-links . {file}

msticpy/context/azure/sentinel_watchlists.py

@@ -328,5 +328,7 @@ def _check_watchlist_exists(
 
         """
         # Check requested watchlist actually exists
-        existing_watchlists = self.list_watchlists()["name"].values
-        return watchlist_name in existing_watchlists
+        existing_watchlists = self.list_watchlists()
+        if existing_watchlists.empty:
+            return False
+        return watchlist_name in existing_watchlists["name"].values

msticpy/context/tiproviders/intsights.py

@@ -46,42 +46,42 @@ class IntSights(HttpTIProvider):
 
     _QUERIES = {
         "ipv4": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "ipv6": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "dns": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "url": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "md5_hash": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "sha1_hash": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "sha256_hash": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
         "email": _IntSightsParams(
-            path="/public/v2/iocs/ioc-by-value",
+            path="/public/v3/iocs/ioc-by-value",
             params={"iocValue": "{observable}"},
             headers=_DEF_HEADERS,
         ),
@@ -111,27 +111,28 @@ def parse_results(self, response: Dict) -> Tuple[bool, ResultSeverity, Any]:
         ):
             return False, ResultSeverity.information, "Not found."
 
-        if response["RawResult"]["Whitelist"] == "True":
+        if response["RawResult"].get("whitelisted", False):
             return False, ResultSeverity.information, "Whitelisted."
 
-        sev = response["RawResult"]["Severity"]
+        sev = response["RawResult"].get("severity", "Low")
         result_dict = {
-            "threat_actors": response["RawResult"]["RelatedThreatActors"],
-            "geolocation": response["RawResult"].get("Geolocation", ""),
+            "threat_actors": response["RawResult"].get("relatedThreatActors", ""),
+            "geolocation": response["RawResult"].get("geolocation", None),
             "response_code": response["Status"],
-            "tags": response["RawResult"]["Tags"] + response["RawResult"]["SystemTags"],
-            "malware": response["RawResult"]["RelatedMalware"],
-            "campaigns": response["RawResult"]["RelatedCampaigns"],
-            "sources": response["RawResult"]["Sources"],
-            "score": response["RawResult"]["Score"],
+            "tags": response["RawResult"].get("tags", [])
+            + response["RawResult"].get("SystemTags", []),
+            "malware": response["RawResult"].get("relatedMalware", []),
+            "campaigns": response["RawResult"].get("relatedCampaigns", []),
+            "score": response["RawResult"].get("score", 0),
             "first_seen": dt.datetime.strptime(
-                response["RawResult"]["FirstSeen"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                response["RawResult"].get("firstSeen", None), "%Y-%m-%dT%H:%M:%S.%fZ"
             ),
             "last_seen": dt.datetime.strptime(
-                response["RawResult"]["LastSeen"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                response["RawResult"].get("lastSeen", None), "%Y-%m-%dT%H:%M:%S.%fZ"
             ),
             "last_update": dt.datetime.strptime(
-                response["RawResult"]["LastUpdate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                response["RawResult"].get("lastUpdateDate", None),
+                "%Y-%m-%dT%H:%M:%S.%fZ",
             ),
         }