Initial commit

Author: kevgliss

.bowerrc

@@ -0,0 +1,3 @@
+{
+    "directory": "app/bower_components"
+}

.editorconfig

@@ -0,0 +1,21 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+
+root = true
+
+
+[*]
+
+# Change these settings to your own preference
+indent_style = space
+indent_size = 2
+
+# We recommend you to keep these unchanged
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.gitignore

@@ -0,0 +1,5 @@
+node_modules
+dist
+.tmp
+.sass-cache
+app/bower_components

.jshintrc

@@ -0,0 +1,24 @@
+{
+  "node": true,
+  "browser": true,
+  "esnext": true,
+  "bitwise": true,
+  "camelcase": true,
+  "curly": true,
+  "eqeqeq": true,
+  "immed": true,
+  "indent": 2,
+  "latedef": true,
+  "newcap": true,
+  "noarg": true,
+  "quotmark": "single",
+  "regexp": true,
+  "undef": true,
+  "unused": true,
+  "strict": true,
+  "trailing": true,
+  "smarttabs": true,
+  "globals": {
+    "angular": false
+  }
+}

.travis.yml

@@ -0,0 +1,7 @@
+language: node_js
+node_js:
+  - '0.8'
+  - '0.10'
+before_script:
+  - 'npm install -g bower grunt-cli'
+  - 'bower install'

LICENSE

@@ -0,0 +1 @@
+

README.rst

@@ -0,0 +1,13 @@
+Ideas:
+
+Allow user to create 'parsers' regexes that crawl over any result set, 
+optionally can be limited to type (exportable?). These 'parsers' should
+be able to promote user defined IOCs that can then be automatically searched
+by other plugins (splunk, sumo, etc)
+
+Allow user to create 'whitelist' of uninterested or known sigs that can be 
+avoided in order to not waste processing time
+
+Allow user to create 'workflows'. These workflows should allow the user to 
+specify the whole data collection chain. Including the extraction of IOCs
+and retrieval of impact results.

TODO.md

@@ -0,0 +1,56 @@
+[app:main]
+use = egg:analys
+
+pyramid.reload_templates = true
+pyramid.debug.all = true
+pyramid.default_locale_name = en
+
+datastore.host = 127.0.0.1
+datastore.port = 27017
+message_queue.host = 127.0.0.1
+message_queue.port = 6379
+
+
+[server:main]
+use = egg:waitress#main
+host = 127.0.0.1
+port = 6543
+
+[server:datastore]
+host = 127.0.0.1
+port = 27017
+
+[server:message_queue]
+host = 127.0.0.1
+port = 6379
+
+# Begin logging configuration
+
+[loggers]
+keys = root, analys
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_analys]
+level = DEBUG
+handlers =
+qualname = analys
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+
+# End logging configuration

analys-dev.ini

@@ -0,0 +1,48 @@
+[app:main]
+use = egg:analys
+
+pyramid.reload_templates = true
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.debug_templates = true
+pyramid.default_locale_name = en
+
+datastore.host = 192.168.221.132
+datastore.port = 27017
+
+[server:main]
+use = egg:waitress#main
+host = 192.168.221.132
+port = 6543
+
+# Begin logging configuration
+
+[loggers]
+keys = root, analys
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_analys]
+level = DEBUG
+handlers =
+qualname = analys
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+
+# End logging configuration

analys.ini

@@ -0,0 +1,38 @@
+"""Main entry point
+"""
+from pyramid.config import Configurator
+from pyramid.events import NewRequest
+from pyramid.renderers import JSON
+
+from analys.datastore import Datastore
+from bson import json_util
+from redis import Redis
+
+def main(global_config, **settings):
+    config = Configurator(settings=settings)
+
+    config.include("cornice")
+    config.scan("analys.views")
+    config.add_static_view(name='static', path='./static')
+
+    ds = Datastore(settings['datastore.host'], settings['datastore.port'])
+    config.registry.settings['datastore'] = ds
+    redis = Redis(settings['message_queue.host'], int(settings['message_queue.port']))
+    config.registry.settings['message_queue'] = redis
+
+    config.add_renderer('mongo_json', json_util)
+
+    def add_datastore(event):
+        settings = event.request.registry.settings
+        datastore = settings['datastore']
+        event.request.datastore = datastore
+
+    def add_message_queue(event):
+        settings = event.request.registry.settings
+        message_queue = settings['message_queue']
+        event.request.message_queue = message_queue
+
+    config.add_subscriber(add_message_queue, NewRequest)
+    config.add_subscriber(add_datastore, NewRequest)
+
+    return config.make_wsgi_app()

analys/__init__.py

@@ -0,0 +1,101 @@
+"""
+.. module:: extractor
+    :platform: Unix
+    :synopsis: A helper module that understands that some zip files submitted
+    to Analys will be zips that may be password protected.
+
+    This module is a helper module that understands that
+    some zip files submitted to Analys will be zips that may be password
+    protected. If this is the case Analys will attempt to bruteforce the password.
+    All files contained inside of the zip will submitted individually to Analys.
+
+.. version:: 1.0
+.. moduleauthor:: Kevin Glisson <[email protected]>
+"""
+import os
+import zipfile
+import rarfile
+import logging
+from analys.plugins.interfaces import File
+log = logging.getLogger(__file__)
+
+class Extractor(object):
+    """ Extract attempts to identify the password used to
+        encrypt a zip file. The most common passwords.
+
+        :param data: raw file bytes
+        :type data: byte string
+
+    """
+    def __init__(self, file, passwords):
+        self.file = file
+        self.samples = []
+        self.max_depth = 5 #only allow five levels of recursion for compressed files
+        self.depth = 0
+        self.passwords = passwords
+
+    def extract(self):
+        if self.depth <= self.max_depth:
+            if 'zip' in self.file.extension():
+                with zipfile.ZipFile(self.file.create_temp_file(), 'r') as myzip:
+                    #sanity checks on the file
+                    for member in myzip.namelist():
+                        filename = os.path.basename(member)
+                        if not filename:
+                            continue
+
+                        try:
+                            sample = myzip.open(member, 'r',).read()
+                            f = File(member, sample)
+                            if f.extension() in ['zip', 'rar']:
+                                self.depth = self.depth + 1
+                                self.extract(f)
+                            else:
+                                self.samples.append(f)
+
+                        except (zipfile.BadZipfile, RuntimeError) as err:
+                            for password in self.passwords:
+
+                                try:
+                                    sample = myzip.open(member, 'r', password).read()
+                                    f = File(member, sample)
+                                    if f.extension() in ['zip', 'rar']:
+                                        self.depth = self.depth + 1
+                                        self.extract(f)
+                                    else:
+                                        self.samples.append(f)
+                                    break
+
+                                except (zipfile.BadZipfile, RuntimeError) as err:
+                                    continue
+
+            if 'rar' in self.file.extension():
+                with rarfile.RarFile(self.file.create_temp_file(), 'r') as myrar:
+                    #sanity checks on the file
+                    for member in myrar.infolist():
+                        if not member.filename:
+                            continue
+                        try:
+                            sample = myrar.open(member, 'r').read()
+                            f = File(member, sample)
+                            if f.extension() in ['zip', 'rar']:
+                                self.depth = self.depth + 1
+                                self.extract(f)
+                            else:
+                                self.samples.append(f)
+                        except:
+                            for password in self.passwords:
+                                try:
+                                    sample = myrar.open(member, 'r',
+                                            password).read()
+                                    f = File(member, sample)
+                                    if f.extension() in ['zip', 'rar']:
+                                        self.depth = self.depth + 1
+                                        self.extract(f)
+                                    else:
+                                        self.samples.append(f)
+                                    break
+                                except:
+                                    continue
+        return self.samples
+

analys/common/__init__.py

@@ -0,0 +1,39 @@
+"""
+.. module: ip
+    :platform: Unix
+    :synopsis: This simple module uses the socket module
+    to attempt to verify a real ip address.
+
+.. version:: 1.0
+.. moduleauthor:: Kevin Glisson <[email protected]>
+"""
+import socket
+
+def is_ipv4(address):
+    """ Returns True if address is a valid ipv4 address,
+        returns false otherwise.
+
+    """
+    try:
+        addr= socket.inet_pton(socket.AF_INET, address)
+    except AttributeError: # no inet_pton here, sorry
+        try:
+            addr = socket.inet_aton(address)
+        except socket.error:
+            return False
+        return address.count('.') == 3
+    except socket.error: # not a valid address
+        return False
+
+    return True
+
+def is_ipv6(address):
+    """ Returns True if address is a valid ipv6 address,
+        returns false otherwise.
+
+    """
+    try:
+        addr= socket.inet_pton(socket.AF_INET6, address)
+    except socket.error: # not a valid address
+        return False
+    return True