Isort linter addition + fix to s3 storage, intezer and fortiguard analyzer, dockerfile versions, CI python version + added "minutes_ago" to "ask_analysis_availability" API (#760)

* added isort

* added isort to requirements

* isorted files

* updated isort

* little adjusts

* fix to s3 storage

* aligned integrations dockerfiles to other dockerfiles version

* fixed intezer analyzers to correctly manage HashDoesNotExistError error

* fix to fortiguard + updated to python 3.9 in CI

* adding "minutes_ago" option to "ask_analysis_availability" + shut down quarkengine test for now

* fixup api

* adjust in test_api

* cherry-picking 0ssigeno work

Co-authored-by: Eshaan Bansal <[email protected]>
Co-authored-by: Simone Berni <[email protected]>

Author: 3 people

.github/workflows/pull_request_automation.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Before Install
       run: |
         sudo apt-get update
@@ -40,6 +40,9 @@ jobs:
     - name: "Lint with flake8 (PEP8 enforcer + linter)"
       run: |
         sudo docker exec intelowl_uwsgi flake8 . --count
+    - name: "Isort check"
+      run: |
+        sudo docker exec intelowl_uwsgi isort . --profile black --filter-files --check-only --diff
     - name: "Test: API"
       run: |
         sudo docker/scripts/coverage_test.sh tests.test_api tests.test_auth

.pre-commit-config.yaml

@@ -1,9 +1,14 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 21.9b0
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 4.0.1
     hooks:
     - id: flake8
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files"]

api_app/admin.py

@@ -2,7 +2,6 @@
 # See the file 'LICENSE' for copying permission.
 
 from django.contrib import admin
-
 from durin.admin import AuthTokenAdmin
 from durin.models import AuthToken, Client
 from guardian.admin import GuardedModelAdmin

api_app/analyzers_manager/classes.py

@@ -1,28 +1,25 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-import time
-import logging
-import requests
 import json
+import logging
+import time
 from abc import ABCMeta
 
-from api_app.exceptions import (
-    AnalyzerRunException,
-    AnalyzerConfigurationException,
-)
-from api_app.core.classes import Plugin
-
-from .models import AnalyzerReport
-from .constants import HashChoices, ObservableTypes, TypeChoices
+import requests
+from django.conf import settings
 
+from api_app.core.classes import Plugin
+from api_app.exceptions import AnalyzerConfigurationException, AnalyzerRunException
 from tests.mock_utils import (
     if_mock_connections,
-    patch,
     mocked_docker_analyzer_get,
     mocked_docker_analyzer_post,
+    patch,
 )
-from django.conf import settings
+
+from .constants import HashChoices, ObservableTypes, TypeChoices
+from .models import AnalyzerReport
 
 logger = logging.getLogger(__name__)
 

api_app/analyzers_manager/controller.py

@@ -3,21 +3,21 @@
 
 import logging
 from typing import Dict, List
-from celery import uuid, chord
-from django.utils.module_loading import import_string
+
+from celery import chord, uuid
 from django.conf import settings
+from django.utils.module_loading import import_string
 from rest_framework.exceptions import ValidationError
 
 from intel_owl.celery import app as celery_app
 from intel_owl.consts import DEFAULT_QUEUE
 
+from ..exceptions import AlreadyFailedJobException, NotRunnableAnalyzer
+from ..helpers import get_now
+from ..models import TLP, Job
 from .classes import BaseAnalyzerMixin, DockerBasedAnalyzer
-from .models import AnalyzerReport
 from .dataclasses import AnalyzerConfig
-from ..models import Job, TLP
-from ..helpers import get_now
-from ..exceptions import AlreadyFailedJobException, NotRunnableAnalyzer
-
+from .models import AnalyzerReport
 
 logger = logging.getLogger(__name__)
 

api_app/analyzers_manager/dataclasses.py

@@ -1,10 +1,11 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
-import typing
 import dataclasses
+import typing
 
 from api_app.core.dataclasses import AbstractConfig
-from .constants import TypeChoices, HashChoices
+
+from .constants import HashChoices, TypeChoices
 from .serializers import AnalyzerConfigSerializer
 
 __all__ = [

api_app/analyzers_manager/file_analyzers/apkid.py

@@ -1,7 +1,7 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-from api_app.analyzers_manager.classes import FileAnalyzer, DockerBasedAnalyzer
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
 
 
 class APKiD(FileAnalyzer, DockerBasedAnalyzer):

api_app/analyzers_manager/file_analyzers/boxjs_scan.py

@@ -1,7 +1,7 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-from api_app.analyzers_manager.classes import FileAnalyzer, DockerBasedAnalyzer
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
 
 
 class BoxJS(FileAnalyzer, DockerBasedAnalyzer):

api_app/analyzers_manager/file_analyzers/capa_info.py

@@ -1,7 +1,7 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-from api_app.analyzers_manager.classes import FileAnalyzer, DockerBasedAnalyzer
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
 
 
 class CapaInfo(FileAnalyzer, DockerBasedAnalyzer):

api_app/analyzers_manager/file_analyzers/clamav.py

@@ -1,7 +1,7 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
 
-from api_app.analyzers_manager.classes import FileAnalyzer, DockerBasedAnalyzer
+from api_app.analyzers_manager.classes import DockerBasedAnalyzer, FileAnalyzer
 
 
 class ClamAV(FileAnalyzer, DockerBasedAnalyzer):