Merge pull request #148 from intelowlproject/develop

Elastic Search, LDAP, groups/permissions, various other improvements.

Author: eshaan7

.dockerignore

@@ -5,4 +5,6 @@ __pycache__
 env_file_app
 env_file_postgres
 env_file_integrations
-venv/
+venv/
+settings/ldap_config.py
+docker-compose-override.yml

.flake8

@@ -8,4 +8,5 @@ exclude =
     docker-compose*,
     venv,
     migrations,
-    virtualenv
+    virtualenv,
+    ldap_config.py

.gitignore

@@ -7,3 +7,5 @@ env_file_postgres
 env_file_integrations
 .env
 venv/
+settings/ldap_config.py
+docker-compose-override.yml

.travis.yml

@@ -4,6 +4,7 @@ branches:
   - develop
 sudo: required
 language: python
+cache: pip
 dist: bionic
 python:
 - '3.6'
@@ -18,4 +19,6 @@ install:
 script:
 - sudo docker exec -ti intel_owl_uwsgi black . --check --exclude "migrations|venv"
 - sudo docker exec -ti intel_owl_uwsgi flake8 . --count
-- sudo docker exec -ti intel_owl_uwsgi python manage.py test tests
+- sudo docker exec -ti intel_owl_uwsgi python manage.py test tests
+after_success:
+- bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -4,6 +4,7 @@ ENV PYTHONUNBUFFERED 1
 ENV DJANGO_SETTINGS_MODULE intel_owl.settings
 ENV PYTHONPATH /opt/deploy/intel_owl
 ENV LOG_PATH /var/log/intel_owl
+ENV ELASTICSEARCH_DSL_VERSION 7.1.4
 
 RUN mkdir -p ${LOG_PATH} \
     ${LOG_PATH}/django ${LOG_PATH}/uwsgi \
@@ -13,20 +14,23 @@ RUN mkdir -p ${LOG_PATH} \
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends apt-utils libsasl2-dev libssl-dev \
-        vim libfuzzy-dev net-tools python-psycopg2 git osslsigncode exiftool \
+        vim libldap2-dev python-dev libfuzzy-dev net-tools python-psycopg2 git osslsigncode exiftool \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 RUN pip3 install --upgrade pip
 
 COPY requirements.txt $PYTHONPATH/requirements.txt
 WORKDIR $PYTHONPATH
 
-RUN pip3 install --compile -r requirements.txt
+RUN pip3 install --no-cache-dir --compile -r requirements.txt
+# install elasticsearch-dsl's appropriate version as specified by user
+RUN pip3 install --no-cache-dir django-elasticsearch-dsl==${ELASTICSEARCH_DSL_VERSION}
 
 COPY . $PYTHONPATH
 
 RUN touch ${LOG_PATH}/django/api_app.log ${LOG_PATH}/django/api_app_errors.log \
     && touch ${LOG_PATH}/django/celery.log ${LOG_PATH}/django/celery_errors.log \
+    && touch ${LOG_PATH}/django/django_auth_ldap.log \
     && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ \
 # this is cause stringstifer creates this directory during the build and cause celery to crash
     && rm -rf /root/.local

README.md

@@ -46,9 +46,12 @@ Documentation about IntelOwl installation, usage, contribution can be found at h
 - Static RTF Analysis
 - Static PDF Analysis
 - Static PE Analysis
+- Static APK Analysis
 - Static Generic File Analysis
 - Strings analysis
 - PE Signature verification
+- PE Capabilities Extraction
+- Emulated Javascript Analysis
 
 **Free modules that require additional configuration**:
 
@@ -125,7 +128,10 @@ license terms.
 [Yara community rules](https://github.com/Yara-Rules),
 [Neo23x0 Yara sigs](https://github.com/Neo23x0/signature-base),
 [Intezer Yara sigs](https://github.com/intezer/yara-rules),
-[McAfee Yara sigs](https://github.com/advanced-threat-research/Yara-Rules)
+[McAfee Yara sigs](https://github.com/advanced-threat-research/Yara-Rules),
+[APKiD](https://github.com/rednaga/APKiD/blob/master/LICENSE.COMMERCIAL),
+[Box-JS](https://github.com/CapacitorSet/box-js/blob/master/LICENSE),
+[Capa](https://github.com/fireeye/capa/blob/master/LICENSE.txt)
 
 ### Acknowledgments
 
@@ -147,4 +153,4 @@ Feel free to contact the author at any time:
 Matteo Lodi ([Twitter](https://twitter.com/matte_lodi))
 
 
-We also have a dedicated twitter account for the project: [@intel_owl](https://twitter.com/intel_owl).
+We also have a dedicated twitter account for the project: [@intel_owl](https://twitter.com/intel_owl).

api_app/admin.py

@@ -4,26 +4,28 @@
 from rest_framework_simplejwt.token_blacklist.models import OutstandingToken
 from rest_framework_simplejwt.tokens import RefreshToken
 from rest_framework_simplejwt.utils import datetime_from_epoch
+from guardian.admin import GuardedModelAdmin
 
 from .models import Job, Tag
 from intel_owl.settings import CLIENT_TOKEN_LIFETIME_DAYS, SIMPLE_JWT as jwt_settings
 
 
-class JobAdminView(admin.ModelAdmin):
+class JobAdminView(GuardedModelAdmin):
     list_display = (
         "id",
+        "status",
         "source",
         "observable_name",
-        "status",
         "observable_classification",
+        "file_name",
         "file_mimetype",
         "received_request_time",
     )
     list_display_link = ("id", "status")
     search_fields = ("source", "md5", "observable_name")
 
 
-class TagAdminView(admin.ModelAdmin):
+class TagAdminView(GuardedModelAdmin):
     list_display = ("id", "label", "color")
     search_fields = ("label", "color")
 

api_app/api.py

@@ -1,6 +1,7 @@
 import logging
 
 from api_app import models, serializers, helpers
+from api_app.permissions import ExtendedObjectPermissions
 from .script_analyzers import general
 
 from wsgiref.util import FileWrapper
@@ -10,6 +11,9 @@
 from rest_framework.response import Response
 from rest_framework import status, viewsets
 from rest_framework.decorators import api_view
+from rest_framework.permissions import DjangoObjectPermissions
+from guardian.decorators import permission_required_or_403
+from rest_framework_guardian.filters import ObjectPermissionsFilter
 
 
 logger = logging.getLogger(__name__)
@@ -19,6 +23,7 @@
 
 
 @api_view(["GET"])
+@permission_required_or_403("api_app.view_job")
 def ask_analysis_availability(request):
     """
     This is useful to avoid repeating the same analysis multiple times.
@@ -115,12 +120,13 @@ def ask_analysis_availability(request):
     except Exception as e:
         logger.exception(f"ask_analysis_availability requester:{source} error:{e}.")
         return Response(
-            {"error": "error in ask_analysis_availability. Check logs."},
+            {"detail": "error in ask_analysis_availability. Check logs."},
             status=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
 
 
 @api_view(["POST"])
+@permission_required_or_403("api_app.add_job")
 def send_analysis_request(request):
     """
     This endpoint allows to start a Job related to a file or an observable
@@ -145,6 +151,9 @@ def send_analysis_request(request):
         list of id's of tags to apply to job
     :param [run_all_available_analyzers]: bool
         default False
+    :param [private]: bool
+        default False,
+        enable it to allow view permissions to only requesting user's groups.
     :param [force_privacy]: bool
         default False,
         enable it if you want to avoid to run analyzers with privacy issues
@@ -172,7 +181,9 @@ def send_analysis_request(request):
 
         params = {"source": source}
 
-        serializer = serializers.JobSerializer(data=data_received)
+        serializer = serializers.JobSerializer(
+            data=data_received, context={"request": request}
+        )
         if serializer.is_valid():
             serialized_data = serializer.validated_data
             logger.info(f"serialized_data: {serialized_data}")
@@ -234,20 +245,20 @@ def send_analysis_request(request):
 
             # save the arrived data plus new params into a new job object
             serializer.save(**params)
-            job_id = serializer.data.get("id", "")
+            job_id = serializer.data.get("id", None)
             md5 = serializer.data.get("md5", "")
-            logger.info(f"new job_id {job_id} for md5 {md5}")
+            logger.info(f"New Job added with ID: #{job_id} and md5: {md5}.")
             if not job_id:
                 return Response({"error": "815"}, status=status.HTTP_400_BAD_REQUEST)
 
         else:
             error_message = f"serializer validation failed: {serializer.errors}"
-            logger.info(error_message)
+            logger.error(error_message)
             return Response(
                 {"error": error_message}, status=status.HTTP_400_BAD_REQUEST
             )
 
-        is_sample = serializer.data.get("is_sample", "")
+        is_sample = serializer.data.get("is_sample", False)
         if not test:
             general.start_analyzers(
                 params["analyzers_to_execute"], analyzers_config, job_id, md5, is_sample
@@ -267,12 +278,13 @@ def send_analysis_request(request):
     except Exception as e:
         logger.exception(f"receive_analysis_request requester:{source} error:{e}.")
         return Response(
-            {"error": "error in send_analysis_request. Check logs"},
+            {"detail": "error in send_analysis_request. Check logs"},
             status=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
 
 
 @api_view(["GET"])
+@permission_required_or_403("api_app.view_job")
 def ask_analysis_result(request):
     """
     Endpoint to retrieve the status and results of a specific Job based on its ID
@@ -299,6 +311,12 @@ def ask_analysis_result(request):
         job_id = data_received["job_id"]
         try:
             job = models.Job.objects.get(id=job_id)
+            # check permission
+            if not request.user.has_perm("api_app.view_job", job):
+                return Response(
+                    {"detail": "You don't have permission to perform this operation."},
+                    status=status.HTTP_403_FORBIDDEN,
+                )
         except models.Job.DoesNotExist:
             response_dict = {"status": "not_available"}
         else:
@@ -308,7 +326,7 @@ def ask_analysis_result(request):
                 "job_id": str(job.id),
             }
             # adding elapsed time
-            finished_analysis_time = getattr(job, "finished_analysis_time", "")
+            finished_analysis_time = getattr(job, "finished_analysis_time", None)
             if not finished_analysis_time:
                 finished_analysis_time = helpers.get_now()
             elapsed_time = finished_analysis_time - job.received_request_time
@@ -360,32 +378,37 @@ def download_sample(request):
     """
     this method is used to download a sample from a Job ID
     :param request: job_id
-    :return 200 found, 404 not found
+    :returns: 200 if found, 404 not found, 403 forbidden
     """
     try:
         data_received = request.query_params
         logger.info(f"Get binary by Job ID. Data received {data_received}")
         if "job_id" not in data_received:
             return Response({"error": "821"}, status=status.HTTP_400_BAD_REQUEST)
+        # get job object
         try:
             job = models.Job.objects.get(id=data_received["job_id"])
         except models.Job.DoesNotExist:
-            return Response({"answer": "not found"}, status=status.HTTP_200_OK)
+            return Response({"detail": "not found"}, status=status.HTTP_404_NOT_FOUND)
+        # check permission
+        if not request.user.has_perm("api_app.view_job", job):
+            return Response(
+                {"detail": "You don't have permission to perform this operation."},
+                status=status.HTTP_403_FORBIDDEN,
+            )
+        # make sure it is a sample
         if not job.is_sample:
             return Response(
-                {"answer": "job without sample"}, status=status.HTTP_400_BAD_REQUEST
+                {"detail": "job without sample"}, status=status.HTTP_400_BAD_REQUEST
             )
-        file_mimetype = job.file_mimetype
-        response = HttpResponse(FileWrapper(job.file), content_type=file_mimetype)
-        response["Content-Disposition"] = "attachment; filename={}".format(
-            job.file_name
-        )
+        response = HttpResponse(FileWrapper(job.file), content_type=job.file_mimetype)
+        response["Content-Disposition"] = f"attachment; filename={job.file_name}"
         return response
 
     except Exception as e:
         logger.exception(f"download_sample requester:{str(request.user)} error:{e}.")
         return Response(
-            {"error": "error in download_sample. Check logs."},
+            {"detail": "error in download_sample. Check logs."},
             status=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
 
@@ -406,23 +429,31 @@ class JobViewSet(viewsets.ReadOnlyModelViewSet):
         if wrong HTTP method
     """
 
-    queryset = models.Job.objects.all()
+    queryset = models.Job.objects.order_by("-received_request_time").all()
     serializer_class = serializers.JobSerializer
-
-    def list(self, request):
-        queryset = (
-            models.Job.objects.order_by("-received_request_time")
-            .defer("analysis_reports", "errors")
-            .all()
-        )
-        serializer = serializers.JobListSerializer(queryset, many=True)
-        return Response(serializer.data)
+    serializer_action_classes = {
+        "list": serializers.JobListSerializer,
+    }
+    permission_classes = (ExtendedObjectPermissions,)
+    filter_backends = (ObjectPermissionsFilter,)
+
+    def get_serializer_class(self, *args, **kwargs):
+        """
+        Instantiate the list of serializers per action from class attribute
+        (must be defined).
+        """
+        kwargs["partial"] = True
+        try:
+            return self.serializer_action_classes[self.action]
+        except (KeyError, AttributeError):
+            return super(JobViewSet, self).get_serializer_class()
 
 
 class TagViewSet(viewsets.ModelViewSet):
     """
     REST endpoint to pefrom CRUD operations on Job tags.
     Requires authentication.
+    POST/PUT/DELETE requires model/object level permission.
 
     :methods_allowed:
         GET, POST, PUT, DELETE, OPTIONS
@@ -437,3 +468,4 @@ class TagViewSet(viewsets.ModelViewSet):
 
     queryset = models.Tag.objects.all()
     serializer_class = serializers.TagSerializer
+    permission_classes = (DjangoObjectPermissions,)

api_app/apps.py

@@ -3,3 +3,7 @@
 
 class ApiAppConfig(AppConfig):
     name = "api_app"
+
+    def ready(self):
+        # flake8: noqa
+        import api_app.signal_handlers as signals