Added ability to start Splunk queries as a job/async

.ci_config/UserExclusion.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!-- IMPORTANT: This file must be saved under %appdata%\PoliCheck_Azure\Config folder -->
+<!--    When adding new values, please use UPPER CASE - all values will be compared only to the UPPER CASE strings -->
+<PoliCheckExclusions>
+
+  <!-- Each of these exclusions is a folder name - if \[name]\ exists in the file path, it will be skipped -->
+  <Exclusion Type="FolderPathFull">.MYPY_CACHE|TESTDATA|HTMLCOV</Exclusion>
+
+  <!-- Each of these exclusions is a folder name - if any folder or file starts with "\[name]", it will be skipped -->
+  <!--<Exclusion Type="FolderPathStart">ABC|XYZ</Exclusion>-->
+
+  <!-- Each of these file types will be completely skipped for the entire scan -->
+  <Exclusion Type="FileType">.CSV</Exclusion>
+
+  <!-- The specified file names will be skipped during the scan regardless which folder they are in -->
+  <Exclusion Type="FileName">TLD_SEED.TXT|QUERY_DATA.CSV|SIGNIN_CHARTS.YAML|GEOPIP.PY</Exclusion>
+
+  </PoliCheckExclusions>

.ci_config/coverage.ini

@@ -0,0 +1,8 @@
+[run]
+omit =
+    */hostedtoolcache.windows.Python/*
+    *.site-packages.msticpy*
+
+[report]
+exclude_lines =
+    @deprecated

.ci_config/credscan.json

@@ -0,0 +1,42 @@
+{
+    "tool": "Credential Scanner",
+    "suppressions": [
+        {
+            "placeholder": ", secret=secret)",
+            "_justification": "This is a code usage example and does not contain a secret."
+        },
+        {
+            "file": "AzureData.rst.txt",
+            "_justification": "This is a code usage example and does not contain a secret."
+        },
+        {
+            "file": "UploadData.rst.txt",
+            "_justification": "This is a code usage example and does not contain a secret."
+        },
+        {
+            "file": "msticpyconfig.rst.txt",
+            "_justification": "This is a code usage example and does not contain a secret."
+        },
+        {
+            "file": "test_splunk_driver.py",
+            "_justification": "This is a test case and does not contain a secret."
+        },
+        {
+            "file": "test_splunk_uploader.py",
+            "_justification": "This is a test case and does not contain a secret."
+        },
+        {
+            "file": "msticpyconfig.yaml",
+            "_justification": "Test data that does not contain a secret"
+        },
+        {
+            "file": "UploadData.rst.txt",
+            "_justification": "This is a code usage example and does not contain a secret."
+        },
+        {
+            "file": "test_pkg_config.py",
+            "_justification": "This is a test case and does not contain a secret."
+        }
+
+    ]
+}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  ignore:
+  - dependency-name: dnspython
+    versions:
+    - 2.1.0
+  - dependency-name: idna
+    versions:
+    - "3.1"
+  - dependency-name: moz-sql-parser
+    versions:
+    - 4.18.21031
+    - 4.21.21059

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '40 13 * * 4'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/python-package.yml

@@ -0,0 +1,126 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MSTICPy CI build and check
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8']
+    env:
+      PROSPECTOR_VER: 1.3.1
+    steps:
+    # Print out details about the run
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJSON(github) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Dump job context
+      env:
+        JOB_CONTEXT: ${{ toJSON(job) }}
+      run: echo "$JOB_CONTEXT"
+    # end print details
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+          ${{ runner.os }}-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip wheel setuptools
+        if [ -f requirements-all.txt ]; then
+          python -m pip install -r requirements-all.txt
+        elif [ -f requirements.txt ]; then
+          python -m pip install -r requirements.txt;
+        fi
+        python -m pip install -e .
+    - name: Install test dependencies
+      run: |
+        python -m pip install flake8 black bandit mypy lxml pylint types-attrs
+        python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
+        python -m pip install Pygments responses pytest-xdist markdown beautifulsoup4 Pillow
+        python -m pip install "pandas>=1.3.0"
+    - name: Prepare test dummy data
+      run: |
+        mkdir ~/.msticpy
+        mkdir ~/.msticpy/mordor
+        cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
+        touch ~/.msticpy/GeoLite2-City.mmdb
+        cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor
+        touch ~/.msticpy/mordor/mitre_tact_cache.pkl
+        touch ~/.msticpy/mordor/mitre_tech_cache.pkl
+        touch ~/.msticpy/mordor/mordor_cache.pkl
+    - name: Pytest
+      env:
+        MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }}
+        IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }}
+        MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
+        MSTICPY_BUILD_SOURCE: fork
+      run: |
+        pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml
+      if: ${{ always() }}
+    - name: black
+      run: |
+        black -t py36 --diff --check --exclude venv .
+      if: ${{ always() }}
+    - name: flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 msticpy --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+      if: ${{ always() }}
+    - name: pylint
+      run: |
+        pylint msticpy --disable=bad-continuation,duplicate-code --disable=E1135,E1101,E1133
+      if: ${{ always() }}
+    - name: mypy
+      run: |
+        mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy
+      if: ${{ always() }}
+    - name: bandit
+      run: |
+        bandit -x tests -r -s B303,B404,B603,B607,B608 msticpy
+      if: ${{ always() }}
+    - name: flake8
+      run: |
+        flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+      if: ${{ always() }}
+    - name: prospector
+      run: |
+        # install this separately since it uses older versions of pylint/isort
+        # chain running to installation since this sometimes fails
+        python -m pip install prospector[with_pyroma]==${{ env.PROSPECTOR_VER }} && prospector --ignore-paths tests --without-tool pylint
+      if: ${{ always() }}
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: pytest-results-${{ matrix.python-version }}
+        path: junit/test-${{ matrix.python-version }}-results.xml
+      # Use always() to always run this step to publish test results when there are test failures
+      if: ${{ always() }}
+    - name: Upload mypy test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: Mypy results ${{ matrix.python-version }}
+        path: junit/mypy-test-${{ matrix.python-version }}-results.xml
+      # Use always() to always run this step to publish test results when there are test failures
+      if: ${{ always() }}

.gitignore

@@ -102,3 +102,21 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+/msticpy.code-workspace
+/docs/source/_build/**
+**/.vscode*
+**/Kqlmagic_temp_files/**
+
+# pycharm project settings
+*\.idea*
+
+#MorphChart test output
+morphchart_package/
+
+# Merge conflict files
+**.orig
+
+#kql magic temp files
+/docs/notebooks/kqlmagic/*
+/kqlmagic/**
+/GitExtensions.settings