Merge pull request #102 from intelowlproject/develop

version 1.0.0

Author: mlodic

.dockerignore

@@ -1,4 +1,8 @@
 .gitignore
+.vscode
 __pycache__
+.env
 env_file_app
-env_file_postgres
+env_file_postgres
+env_file_integrations
+venv/

.env_template

@@ -5,16 +5,20 @@
 ### the COMPOSE_FILE variable each seperated with ':'. If you are on windows, replace all ':' with ';'.
 ### Reference to Docker's official Docs: https://docs.docker.com/compose/reference/envvars/#compose_file#compose_file
 
-## Default
+###### Default (Production) ######
+
 COMPOSE_FILE=docker-compose.yml
 
-## To run all additional integrations
+## To run all additional integrations in production
 #COMPOSE_FILE=docker-compose.yml:./integrations/docker-compose.peframe.yml
 
-## To run tests or for local development
+###### For Tests or local development ######
+
 #COMPOSE_FILE=docker-compose-for-tests.yml
+
+## To run all additional integrations in development
 #COMPOSE_FILE=docker-compose-for-tests.yml:./integrations/docker-compose-for-tests.peframe.yml
 
-## For travis
-#COMPOSE_FILE=docker-compose-for-travis.yml
+###### For travis ######
+
 #COMPOSE_FILE=docker-compose-for-travis.yml./integrations/docker-compose.peframe.yml

.flake8

@@ -0,0 +1,11 @@
+[flake8]
+max-line-length = 88
+ignore =
+    W503, # line break before binary operator
+    E231, # missing whitespace after ','
+exclude =
+    Dockerfile,
+    docker-compose*,
+    venv,
+    migrations,
+    virtualenv

.travis.yml

@@ -16,5 +16,6 @@ install:
 - sudo docker-compose -f docker-compose-for-travis.yml build
 - sudo docker-compose -f docker-compose-for-travis.yml up -d
 script:
-- sudo docker exec -ti intel_owl_uwsgi python manage.py test tests
-
+- sudo docker exec -ti intel_owl_uwsgi black . --check --exclude "migrations|venv"
+- sudo docker exec -ti intel_owl_uwsgi flake8 . --count
+- sudo docker exec -ti intel_owl_uwsgi python manage.py test tests

Dockerfile

@@ -6,8 +6,11 @@ ENV PYTHONPATH /opt/deploy/intel_owl
 
 RUN mkdir -p /var/log/intel_owl /var/log/intel_owl/django /var/log/intel_owl/uwsgi /opt/deploy/files_required /opt/deploy/yara /opt/deploy/configuration
 
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev vim libfuzzy-dev net-tools python-psycopg2 git osslsigncode exiftool
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev \
+        vim libfuzzy-dev net-tools python-psycopg2 git osslsigncode exiftool \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 RUN pip3 install --upgrade pip
 
 COPY requirements.txt $PYTHONPATH/requirements.txt

Dockerfile_nginx

@@ -1,2 +1,8 @@
+# Stage 1: Get build artifacts from intelowl-ng
+FROM intelowlproject/intelowl_ng:latest AS angular-prod-build
+
+# Stage 2: Inject the build artifacts into nginx container
 FROM library/nginx:1.16.1-alpine
+
+COPY --from=angular-prod-build /usr/src/app/dist /var/www/angular_build
 VOLUME /var/log/nginx

Dockerfile_nginx_no_angular

@@ -0,0 +1,2 @@
+FROM library/nginx:1.16.1-alpine
+VOLUME /var/log/nginx

README.md

@@ -4,6 +4,8 @@
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/intelowlproject/IntelOwl.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/intelowlproject/IntelOwl/context:python)
 [![CodeFactor](https://www.codefactor.io/repository/github/intelowlproject/intelowl/badge)](https://www.codefactor.io/repository/github/intelowlproject/intelowl)
 [![Build Status](https://travis-ci.com/intelowlproject/IntelOwl.svg?branch=master)](https://travis-ci.org/intelowlproject/IntelOwl)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+
 # Intel Owl
 
 Do you want to get **threat intelligence data** about a file, an IP or a domain?
@@ -24,6 +26,7 @@ Main features:
 - full django-python application
 - easily and completely customizable, both the APIs and the analyzers
 - clone the project, set up the configuration and you are ready to run
+- Official frontend client: **[IntelOwl-ng](https://github.com/intelowlproject/IntelOwl-ng)** provides features such as dashboard, visualizations of analysis data, easy to use forms for requesting new analysis, etc.
 
 ### Free internal modules available
 * Static Doc Analysis
@@ -37,7 +40,7 @@ Main features:
 ### Free modules that require additional configuration
 * Cuckoo (requires at least one working Cuckoo instance)
 * MISP (requires at least one working MISP instance)
-* Yara (Community, Neo23x0 and Intezer rules are already available. There's the chance to add your own rules)
+* Yara (Community, Neo23x0, Intezer and McAfee rules are already available. There's the chance to add your own rules)
 
 ### External services available
 #### required paid or trial api key
@@ -70,8 +73,11 @@ Main features:
 * Threatminer
 * Abuse.ch MalwareBazaar
 * Abuse.ch URLhaus
+* Team Cymru Malware Hash Registry
+* Tranco Rank
 * Google DoH
-* CloudFlare DoH
+* CloudFlare DoH Classic
+* CloudFlare DoH Malware
 * Classic DNS resolution
 
 ### Documentation
@@ -99,7 +105,8 @@ license terms.
 [GitPython](https://github.com/gitpython-developers/GitPython),
 [Yara community rules](https://github.com/Yara-Rules),
 [Neo23x0 Yara sigs](https://github.com/Neo23x0/signature-base),
-[Intezer Yara sigs](https://github.com/intezer/yara-rules)
+[Intezer Yara sigs](https://github.com/intezer/yara-rules),
+[McAfee Yara sigs](https://github.com/advanced-threat-research/Yara-Rules)
 
 ### Acknowledgments
 This project was created and will be upgraded thanks to the following organizations:
@@ -111,9 +118,9 @@ This project was created and will be upgraded thanks to the following organizati
 #### Google Summer Of Code
 The project was accepted to the GSoC 2020 under the Honeynet Project!! 
 
-Stay tuned for upcoming [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) developed by Eshann Bansal [Twitter](https://twitter.com/mask0fmydisguis)
+Stay tuned for upcoming [new features](https://www.honeynet.org/gsoc/gsoc-2020/google-summer-of-code-2020-project-ideas/#intel-owl-improvements) developed by Eshaan Bansal ([Twitter](https://twitter.com/mask0fmydisguis)).
 
 
 ### About the author 
 Feel free to contact the author at any time:
-Matteo Lodi [Twitter](https://twitter.com/matte_lodi)
+Matteo Lodi ([Twitter](https://twitter.com/matte_lodi))

api_app/admin.py

@@ -1,5 +1,12 @@
 from django.contrib import admin
+
+from rest_framework_simplejwt.token_blacklist.admin import OutstandingTokenAdmin
+from rest_framework_simplejwt.token_blacklist.models import OutstandingToken
+from rest_framework_simplejwt.tokens import RefreshToken
+from rest_framework_simplejwt.utils import datetime_from_epoch
+
 from .models import Job, Tag
+from intel_owl.settings import CLIENT_TOKEN_LIFETIME_DAYS, SIMPLE_JWT as jwt_settings
 
 
 class JobAdminView(admin.ModelAdmin):
@@ -21,5 +28,89 @@ class TagAdminView(admin.ModelAdmin):
     search_fields = ("label", "color")
 
 
+# SimpleJWT stuff
+class CustomOutstandingTokenAdmin(OutstandingTokenAdmin):
+    """
+    Custom admin view for OutstandingToken model of simplejwt package\n
+    allows bulk deletion and refresh token creation
+    """
+
+    # default actions
+    actions = []
+
+    # searchable fields
+    search_fields = (
+        "user__username",
+        "user__id",
+        "jti",
+    )
+
+    __fieldsets_custom = [
+        (
+            "Create API Token For",
+            {
+                "fields": ("user", "token",),
+                "description": f"""
+                    <h3>Token will be auto-generated on save.</h3>
+                    <h5>Please note that this token,</h5>
+                    <ol>
+                      <li>can only be used with the PyIntelOwl client.</li>
+                      <li>is rotated on every authenticated request
+                      and saves itself via pyintelowl</li>
+                      <li>
+                        automatically expires if goes
+                        unused for {CLIENT_TOKEN_LIFETIME_DAYS} days.
+                    </li>
+                    </ol>
+                """,
+            },
+        ),
+    ]
+
+    def add_view(self, request, extra_content=None):
+        self.fieldsets = self.__fieldsets_custom
+        return super(CustomOutstandingTokenAdmin, self).add_view(request)
+
+    def get_readonly_fields(self, *args, **kwargs):
+        fields = [f.name for f in self.model._meta.fields]
+        # only user field is writeable
+        fields.remove("user")
+        return fields
+
+    def has_delete_permission(self, *args, **kwargs):
+        return True
+
+    def has_add_permission(self, *args, **kwargs):
+        return True
+
+    def has_change_permission(self, *args, **kwargs):
+        return False
+
+    def save_model(self, request, obj, form, change):
+        if obj.user:
+            refresh = RefreshToken()
+            # custom claims
+            refresh["client"] = "pyintelowl"
+            refresh["user_id"] = obj.user.id
+            # overwrite lifetime/expiry
+            refresh.set_exp(
+                lifetime=jwt_settings.get("PYINTELOWL_TOKEN_LIFETIME", None)
+            )
+            token = OutstandingToken.objects.create(
+                user=obj.user,
+                jti=refresh.payload["jti"],
+                token=str(refresh),
+                created_at=refresh.current_time,
+                expires_at=datetime_from_epoch(refresh["exp"]),
+            )
+            return token
+
+        return None
+
+
 admin.site.register(Job, JobAdminView)
 admin.site.register(Tag, TagAdminView)
+# Unregister the default admin view for OutstandingToken
+admin.site.unregister(OutstandingToken)
+# Register our custom admin view for OutstandingToken
+admin.site.register(OutstandingToken, CustomOutstandingTokenAdmin)