diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7baeb64..e2a4f31 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -45,6 +45,11 @@ jobs: run: poetry run pyupgrade --py311-plus **/*.py test: runs-on: ubuntu-latest + services: + spamassassin: + image: instantlinux/spamassassin:4.0.0-6 + ports: + - 783:783 strategy: matrix: python-version: [3.11] @@ -63,6 +68,8 @@ jobs: run: poetry install - name: Make dummy frontend directory run: mkdir -p frontend/dist/ + - name: Wait until SpamAssasin ready + run: poetry run python scripts/ping.py - name: Run tests run: poetry run pytest -v --cov=app --cov-report=term-missing - name: Coveralls diff --git a/backend/__init__.py b/backend/__init__.py index e69de29..ffff975 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -0,0 +1,3 @@ +from .clients.emailrep import EmailRep # noqa: F401 +from .clients.inquest import InQuest # noqa: F401 +from .clients.spamassasin import SpamAssassin # noqa: F401 diff --git a/backend/api/endpoints/analyze.py b/backend/api/endpoints/analyze.py index 34457b9..8fbb92a 100644 --- a/backend/api/endpoints/analyze.py +++ b/backend/api/endpoints/analyze.py @@ -3,14 +3,21 @@ from pydantic import ValidationError from redis import Redis -from backend import deps, schemas -from backend.core import settings +from backend import clients, deps, schemas, settings from backend.factories.response import ResponseFactory router = APIRouter() -async def _analyze(file: bytes) -> schemas.Response: +async def _analyze( + file: bytes, + *, + spam_assassin: clients.SpamAssassin, + email_rep: clients.EmailRep, + optional_inquest: clients.InQuest | None = None, + optional_vt: clients.VirusTotal | None = None, + optional_urlscan: clients.UrlScan | None = None, +) -> schemas.Response: try: payload = schemas.FilePayload(file=file) except ValidationError as exc: @@ -19,7 +26,14 @@ async def _analyze(file: bytes) -> schemas.Response: detail=jsonable_encoder(exc.errors()), ) from exc - return await ResponseFactory.from_bytes(payload.file) + return await ResponseFactory.call( + payload.file, + email_rep=email_rep, + spam_assassin=spam_assassin, + optional_inquest=optional_inquest, + optional_urlscan=optional_urlscan, + optional_vt=optional_vt, + ) def cache_response( @@ -45,8 +59,20 @@ async def analyze( *, background_tasks: BackgroundTasks, optional_redis: deps.OptionalRedis, + spam_assassin: deps.SpamAssassin, + email_rep: deps.EmailRep, + optional_inquest: deps.OptionalInQuest, + optional_vt: deps.OptionalVirusTotal, + optional_urlscan: deps.OptionalUrlScan, ) -> schemas.Response: - response = await _analyze(payload.file.encode()) + response = await _analyze( + payload.file.encode(), + email_rep=email_rep, + spam_assassin=spam_assassin, + optional_inquest=optional_inquest, + optional_urlscan=optional_urlscan, + optional_vt=optional_vt, + ) if optional_redis is not None: background_tasks.add_task( @@ -67,8 +93,20 @@ async def analyze_file( *, background_tasks: BackgroundTasks, optional_redis: deps.OptionalRedis, + spam_assassin: deps.SpamAssassin, + email_rep: deps.EmailRep, + optional_inquest: deps.OptionalInQuest, + optional_vt: deps.OptionalVirusTotal, + optional_urlscan: deps.OptionalUrlScan, ) -> schemas.Response: - response = await _analyze(file) + response = await _analyze( + file, + email_rep=email_rep, + spam_assassin=spam_assassin, + optional_inquest=optional_inquest, + optional_urlscan=optional_urlscan, + optional_vt=optional_vt, + ) if optional_redis is not None: background_tasks.add_task( diff --git a/backend/api/endpoints/lookup.py b/backend/api/endpoints/lookup.py index 4f3c4b9..d1a4bb6 100644 --- a/backend/api/endpoints/lookup.py +++ b/backend/api/endpoints/lookup.py @@ -1,7 +1,6 @@ from fastapi import APIRouter, HTTPException, status -from backend import deps, schemas -from backend.core import settings +from backend import deps, schemas, settings router = APIRouter() diff --git a/backend/api/endpoints/submit.py b/backend/api/endpoints/submit.py index 564fd9c..82b27ff 100644 --- a/backend/api/endpoints/submit.py +++ b/backend/api/endpoints/submit.py @@ -1,11 +1,9 @@ +import httpx from fastapi import APIRouter, HTTPException, status -from httpx._exceptions import HTTPError -from backend.core.utils import has_inquest_api_key, has_virustotal_api_key +from backend import deps, schemas from backend.schemas.eml import Attachment -from backend.schemas.submission import SubmissionResult -from backend.submitters.inquest import InQuestSubmitter -from backend.submitters.virustotal import VirusTotalSubmitter +from backend.utils import attachment_to_file router = APIRouter() @@ -17,12 +15,9 @@ description="Submit an attachment to InQuest", status_code=200, ) -async def submit_to_inquest(attachment: Attachment) -> SubmissionResult: - if not has_inquest_api_key(): - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="You don't have the InQuest API key", - ) +async def submit_to_inquest( + attachment: Attachment, *, optional_inquest: deps.OptionalInQuest +) -> schemas.SubmissionResult: # check ext type valid_types = ["doc", "docx", "ppt", "pptx", "xls", "xlsx"] if attachment.extension not in valid_types: @@ -31,36 +26,45 @@ async def submit_to_inquest(attachment: Attachment) -> SubmissionResult: detail=f"{attachment.extension} is not supported.", ) - submitter = InQuestSubmitter(attachment) + if optional_inquest is None: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="You don't have the InQuest API key", + ) + try: - return await submitter.submit() - except HTTPError as e: + return await optional_inquest.submit(attachment_to_file(attachment)) + except httpx.HTTPStatusError as e: raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Something went wrong with InQuest submission: {e!s}", + status_code=e.response.status_code, + detail=f"Something went wrong with InQuest submission: {e}", ) from e @router.post( "/virustotal", - response_model=SubmissionResult, response_description="Return a submission result", summary="Submit an attachment to VirusTotal", description="Submit an attachment to VirusTotal", status_code=200, ) -async def submit_to_virustotal(attachment: Attachment) -> SubmissionResult: - if not has_virustotal_api_key(): +async def submit_to_virustotal( + attachment: Attachment, *, optional_vt: deps.OptionalVirusTotal +) -> schemas.SubmissionResult: + if optional_vt is None: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="You don't have the VirusTotal API key", ) - submitter = VirusTotalSubmitter(attachment) try: - return await submitter.submit() - except HTTPError as e: + await optional_vt.scan_file_async(attachment_to_file(attachment)) + sha256 = attachment.hash.sha256 + return schemas.SubmissionResult( + reference_url=f"https://www.virustotal.com/gui/file/{sha256}/detection" + ) + except httpx.HTTPStatusError as e: raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Something went wrong with VirusTotal submission: {e!s}", + status_code=e.response.status_code, + detail=f"Something went wrong with VirusTotal submission: {e}", ) from e diff --git a/backend/clients/__init__.py b/backend/clients/__init__.py new file mode 100644 index 0000000..5e97916 --- /dev/null +++ b/backend/clients/__init__.py @@ -0,0 +1,8 @@ +import vt + +from .emailrep import EmailRep # noqa: F401 +from .inquest import InQuest # noqa: F401 +from .spamassasin import SpamAssassin # noqa: F401 +from .urlscan import UrlScan # noqa: F401 + +VirusTotal = vt.Client diff --git a/backend/clients/emailrep.py b/backend/clients/emailrep.py new file mode 100644 index 0000000..6a568f3 --- /dev/null +++ b/backend/clients/emailrep.py @@ -0,0 +1,13 @@ +import httpx + +from backend import schemas + + +class EmailRep(httpx.AsyncClient): + def __init__(self) -> None: + super().__init__(base_url="https://emailrep.io") + + async def lookup(self, email: str) -> schemas.EmailRepLookup: + r = await self.get(f"/{email}") + r.raise_for_status() + return schemas.EmailRepLookup.model_validate(r.json()) diff --git a/backend/clients/inquest.py b/backend/clients/inquest.py new file mode 100644 index 0000000..890538a --- /dev/null +++ b/backend/clients/inquest.py @@ -0,0 +1,27 @@ +import io + +import httpx +from starlette.datastructures import Secret + +from backend import schemas + + +class InQuest(httpx.AsyncClient): + def __init__(self, api_key: Secret) -> None: + super().__init__( + base_url="https://labs.inquest.net", + headers={"Authorization": f"Basic: {api_key}"}, + ) + + async def lookup(self, sha256: str) -> schemas.InQuestLookup: + r = await self.get("/api/dfi/details", params={"sha256": sha256}) + r.raise_for_status() + return schemas.InQuestLookup.model_validate(r.json()) + + async def submit(self, f: io.BytesIO) -> schemas.SubmissionResult: + r = await self.post("/api/dfi/upload", files={"file": f}) + r.raise_for_status() + data = r.json()["data"] + return schemas.SubmissionResult( + reference_url=f"https://labs.inquest.net/dfi/sha256/{data}" + ) diff --git a/backend/services/spamassassin.py b/backend/clients/spamassasin.py similarity index 54% rename from backend/services/spamassassin.py rename to backend/clients/spamassasin.py index 48dd2d7..f5ef5d8 100644 --- a/backend/services/spamassassin.py +++ b/backend/clients/spamassasin.py @@ -1,46 +1,32 @@ -from dataclasses import dataclass -from typing import Any - import aiospamc +from aiospamc.header_values import Headers from async_timeout import timeout - -@dataclass -class Detail: - name: str - score: float - description: str - - -@dataclass -class Report: - score: float - details: list[Detail] - level = 5.0 - - def is_spam(self, level: float = 5.0) -> bool: - return self.score is None or self.score > level +from backend import schemas, settings class Parser: - def __init__(self, headers: dict[str, Any], body: str): + def __init__(self, headers: Headers, body: str): self.headers = headers self.body = body self.score = 0.0 - self.details: list[Detail] = [] + self.details: list[schemas.SpamAssassinDetail] = [] def _parse_headers(self): - spam_value = self.headers["Spam"] - self.score = spam_value.score + spam_value = self.headers.get("Spam") + if spam_value is not None: + self.score = spam_value.score - def _parse_detail(self, line: str) -> Detail: + def _parse_detail(self, line: str) -> schemas.SpamAssassinDetail: parts = line.split() score = float(parts[0]) name = parts[1] description = " ".join(parts[2:]) - return Detail(name=name, score=score, description=description) + return schemas.SpamAssassinDetail( + name=name, score=score, description=description + ) - def _parse_details(self, details: str) -> list[Detail]: + def _parse_details(self, details: str) -> list[schemas.SpamAssassinDetail]: lines = details.splitlines() normalized_line: list[str] = [] @@ -56,33 +42,35 @@ def _parse_details(self, details: str) -> list[Detail]: def _parse_body(self): lines = [line for line in self.body.splitlines() if line != ""] - demiliter_index = 0 + delimiter_index = 0 for index, line in enumerate(lines): if "---" in line: - demiliter_index = index + 1 + delimiter_index = index + 1 break - details = "\n".join(lines[demiliter_index:]) + details = "\n".join(lines[delimiter_index:]) self.details = self._parse_details(details) - def parse(self): + def parse(self) -> schemas.SpamAssassinReport: self._parse_headers() self._parse_body() - - def to_report(self) -> Report: - return Report(score=self.score, details=self.details) + return schemas.SpamAssassinReport(score=self.score, details=self.details) class SpamAssassin: - def __init__(self, host: str = "127.0.0.1", port: int = 783, timeout: int = 10): + def __init__( + self, + host: str = settings.SPAMASSASSIN_HOST, + port: int = settings.SPAMASSASSIN_PORT, + timeout: int = settings.SPAMASSASSIN_TIMEOUT, + ): self.host = host self.port = port self.timeout = timeout - async def report(self, message: bytes) -> Report: + async def report(self, message: bytes) -> schemas.SpamAssassinReport: async with timeout(self.timeout): response = await aiospamc.report(message, host=self.host, port=self.port) parser = Parser(headers=response.headers, body=response.body.decode()) - parser.parse() - return parser.to_report() + return parser.parse() diff --git a/backend/clients/urlscan.py b/backend/clients/urlscan.py new file mode 100644 index 0000000..0ad52d4 --- /dev/null +++ b/backend/clients/urlscan.py @@ -0,0 +1,26 @@ +from urllib.parse import urlparse + +import httpx +from starlette.datastructures import Secret + +from backend import schemas + + +class UrlScan(httpx.AsyncClient): + def __init__(self, api_key: Secret) -> None: + super().__init__( + base_url="https://urlscan.io", headers={"api-key": str(api_key)} + ) + + async def lookup( + self, + url: str, + ) -> schemas.UrlScanLookup: + parsed = urlparse(url) + params = { + "q": f'task.url:"{url}" AND task.domain:"{parsed.hostname}" AND verdicts.malicious:true', + "size": 1, + } + r = await self.get("/api/v1/search/", params=params) + r.raise_for_status() + return schemas.UrlScanLookup.model_validate(r.json()) diff --git a/backend/core/__init__.py b/backend/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/core/resources.py b/backend/core/resources.py deleted file mode 100644 index 0085dde..0000000 --- a/backend/core/resources.py +++ /dev/null @@ -1,3 +0,0 @@ -import httpx - -httpx_client: httpx.AsyncClient = httpx.AsyncClient() diff --git a/backend/core/utils.py b/backend/core/utils.py deleted file mode 100644 index de84e4d..0000000 --- a/backend/core/utils.py +++ /dev/null @@ -1,28 +0,0 @@ -from starlette.datastructures import Secret - -import backend.core.settings - - -def is_secret(value) -> bool: - return isinstance(value, Secret) - - -def has_urlscan_api_key() -> bool: - return ( - is_secret(backend.core.settings.URLSCAN_API_KEY) - and str(backend.core.settings.URLSCAN_API_KEY) != "" - ) - - -def has_virustotal_api_key() -> bool: - return ( - is_secret(backend.core.settings.VIRUSTOTAL_API_KEY) - and str(backend.core.settings.VIRUSTOTAL_API_KEY) != "" - ) - - -def has_inquest_api_key() -> bool: - return ( - is_secret(backend.core.settings.INQUEST_API_KEY) - and str(backend.core.settings.INQUEST_API_KEY) != "" - ) diff --git a/backend/core/datastructures.py b/backend/datastructures.py similarity index 100% rename from backend/core/datastructures.py rename to backend/datastructures.py diff --git a/backend/deps.py b/backend/deps.py index 85eca8f..4d10068 100644 --- a/backend/deps.py +++ b/backend/deps.py @@ -1,22 +1,17 @@ import typing -from contextlib import contextmanager +from contextlib import asynccontextmanager, contextmanager from fastapi import Depends from redis import Redis +from starlette.datastructures import Secret -from backend.core import settings - - -def cast_optional_str(v: typing.Any | None) -> str | None: - if v is None: - return None - - return str(v) +from backend import clients, settings +from backend.datastructures import DatabaseURL @contextmanager def _get_optional_redis( - redis_url: str | None = cast_optional_str(settings.REDIS_URL), + redis_url: DatabaseURL | None = settings.REDIS_URL, ) -> typing.Generator[Redis | None, None, None]: if redis_url is None: yield None @@ -28,9 +23,83 @@ def _get_optional_redis( redis.close() -def get_optional_redis(redis_url: str | None = cast_optional_str(settings.REDIS_URL)): - with _get_optional_redis(redis_url) as optional_redis: +def get_optional_redis(): + with _get_optional_redis(settings.REDIS_URL) as optional_redis: yield optional_redis +@asynccontextmanager +async def _get_optional_vt(api_key: Secret | None = settings.VIRUSTOTAL_API_KEY): + if api_key is None: + yield None + else: + async with clients.VirusTotal(apikey=str(api_key)) as client: + yield client + + +async def get_optional_vt(): + async with _get_optional_vt(settings.VIRUSTOTAL_API_KEY) as client: + yield client + + +@asynccontextmanager +async def _get_optional_inquest(api_key: Secret | None = settings.INQUEST_API_KEY): + if api_key is None: + yield None + else: + async with clients.InQuest(api_key=api_key) as client: + yield client + + +async def get_optional_inquest(): + async with _get_optional_inquest(settings.INQUEST_API_KEY) as client: + yield client + + +@asynccontextmanager +async def _get_optional_urlscan(api_key: Secret | None = settings.URLSCAN_API_KEY): + if api_key is None: + yield None + else: + async with clients.UrlScan(api_key=api_key) as client: + yield client + + +async def get_optional_urlscan(): + async with _get_optional_urlscan(settings.URLSCAN_API_KEY) as client: + yield client + + +@asynccontextmanager +async def _get_email_rep(): + async with clients.EmailRep() as client: + yield client + + +async def get_email_rep(): + async with _get_email_rep() as client: + yield client + + +def get_spam_assassin() -> clients.SpamAssassin: + return clients.SpamAssassin( + host=settings.SPAMASSASSIN_HOST, + port=settings.SPAMASSASSIN_PORT, + timeout=settings.SPAMASSASSIN_TIMEOUT, + ) + + OptionalRedis = typing.Annotated[Redis | None, Depends(get_optional_redis)] + +OptionalInQuest = typing.Annotated[ + clients.InQuest | None, Depends(get_optional_inquest) +] +OptionalVirusTotal = typing.Annotated[ + clients.VirusTotal | None, Depends(get_optional_vt) +] +OptionalUrlScan = typing.Annotated[ + clients.UrlScan | None, Depends(get_optional_urlscan) +] + +EmailRep = typing.Annotated[clients.EmailRep, Depends(get_email_rep)] +SpamAssassin = typing.Annotated[clients.SpamAssassin, Depends(get_spam_assassin)] diff --git a/backend/factories/__init__.py b/backend/factories/__init__.py index e69de29..1893d3d 100644 --- a/backend/factories/__init__.py +++ b/backend/factories/__init__.py @@ -0,0 +1,8 @@ +from .emailrep import EmailRepVerdictFactory # noqa: F401 +from .eml import EmlFactory # noqa: F401 +from .inquest import InQuestVerdictFactory # noqa: F401 +from .oldid import OleIDVerdictFactory # noqa: F401 +from .response import ResponseFactory # noqa: F401 +from .spamassassin import SpamAssassinVerdictFactory # noqa: F401 +from .urlscan import UrlScanVerdictFactory # noqa: F401 +from .virustotal import VirusTotalVerdictFactory # noqa: F401 diff --git a/backend/factories/abstract.py b/backend/factories/abstract.py new file mode 100644 index 0000000..26bc18c --- /dev/null +++ b/backend/factories/abstract.py @@ -0,0 +1,16 @@ +import typing +from abc import ABC, abstractmethod + + +class AbstractFactory(ABC): + @classmethod + @abstractmethod + def call(cls, *args: typing.Any, **kwargs: typing.Any): + raise NotImplementedError() + + +class AbstractAsyncFactory(ABC): + @classmethod + @abstractmethod + async def call(cls, *args: typing.Any, **kwargs: typing.Any): + raise NotImplementedError() diff --git a/backend/factories/emailrep.py b/backend/factories/emailrep.py index 7cdebcb..3025460 100644 --- a/backend/factories/emailrep.py +++ b/backend/factories/emailrep.py @@ -1,34 +1,45 @@ -from loguru import logger +from functools import partial -from backend.schemas.verdict import Detail, Verdict -from backend.services.emailrep import EmailRep +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io +from backend import clients, schemas -class EmailRepVerdictFactory: - def __init__(self, email: str): - self.email = email - self.name = "EmailRep" +from .abstract import AbstractAsyncFactory - async def to_model(self) -> Verdict: - details: list[Detail] = [] - malicious = False +NAME_OR_KEY = "EmailRep" - email_rep = EmailRep() - try: - res = await email_rep.get(self.email) - if res.suspicious is True: - malicious = True - description = f"{self.email} is suspicious. See https://emailrep.io/{self.email} for details." - details.append(Detail(key="EmailRep", description=description)) - else: - description = f"{self.email} is not suspicious. See https://emailrep.io/{self.email} for details." - details.append(Detail(key="EmailRep", description=description)) - except Exception as error: - logger.error(error) - return Verdict(name=self.name, malicious=malicious, details=details) +@future_safe +async def lookup(email: str, *, client: clients.EmailRep) -> schemas.EmailRepLookup: + return await client.lookup(email) + +@future_safe +async def transform(lookup: schemas.EmailRepLookup, *, name_or_key: str = NAME_OR_KEY): + details: list[schemas.VerdictDetail] = [] + malicious = False + + description = f"{lookup.email} is not suspicious. See https://emailrep.io/{lookup.email} for details." + if lookup.suspicious: + malicious = True + description = f"{lookup.email} is suspicious. See https://emailrep.io/{lookup.email} for details." + + details.append(schemas.VerdictDetail(key=name_or_key, description=description)) + return schemas.Verdict(name=name_or_key, malicious=malicious, details=details) + + +class EmailRepVerdictFactory(AbstractAsyncFactory): @classmethod - async def from_email(cls, email) -> Verdict: - obj = cls(email) - return await obj.to_model() + async def call( + cls, email: str, *, client: clients.EmailRep, name_or_key: str = NAME_OR_KEY + ) -> schemas.Verdict: + f_result: FutureResultE[schemas.Verdict] = flow( + lookup(email, client=client), + bind(partial(transform, name_or_key=name_or_key)), + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/factories/eml.py b/backend/factories/eml.py index ac5b29e..80f645f 100644 --- a/backend/factories/eml.py +++ b/backend/factories/eml.py @@ -1,4 +1,3 @@ -from hashlib import sha256 from io import BytesIO from typing import Any @@ -6,11 +5,17 @@ import dateparser from eml_parser import EmlParser from ioc_finder import parse_domain_names, parse_email_addresses, parse_ipv4_addresses +from returns.functions import raise_exception +from returns.pipeline import flow +from returns.pointfree import bind +from returns.result import ResultE, safe -from backend.schemas.eml import Eml -from backend.services.extractor import parse_urls_from_body -from backend.services.outlookmsgfile import Message -from backend.services.validator import is_eml_file +from backend import schemas +from backend.outlookmsgfile import Message +from backend.utils import parse_urls_from_body +from backend.validator import is_eml_file + +from .abstract import AbstractFactory def is_inline_forward_attachment(attachment: dict) -> bool: @@ -33,102 +38,119 @@ def is_inline_forward_attachment(attachment: dict) -> bool: return is_rfc822 and is_inline -class EmlFactory: - def __init__(self, eml_file: bytes): - self.eml_file = eml_file - parser = EmlParser(include_raw_body=True, include_attachment_data=True) - self.parsed = parser.decode_email_bytes(eml_file) - self.parsed["identifier"] = sha256(eml_file).hexdigest() +@safe +def to_eml(data: bytes) -> bytes: + if is_eml_file(data): + return data + + # assume data is a msg file + file = BytesIO(data) + message = Message(file) + email = message.to_email() + return email.as_bytes() + - def _normalize_received_date(self, received: dict): - date = received.get("date", "") - if date != "": - return received +@safe +def parse(data: bytes) -> dict: + parser = EmlParser(include_raw_body=True, include_attachment_data=True) + return parser.decode_email_bytes(data) - src = received.get("src", "") - parts = src.split(";") - date_ = parts[-1].strip() - received["date"] = dateparser.parse(date_) + +def _normalize_received_date(received: dict): + date = received.get("date", "") + if date != "": return received - def _normalize_received(self, received: list[dict]) -> list[dict]: - if len(received) == 0: - return [] + src = received.get("src", "") + parts = src.split(";") + date_ = parts[-1].strip() + received["date"] = dateparser.parse(date_) + return received - received = [self._normalize_received_date(r) for r in received] - received.reverse() - first = received[0] - base_date = arrow.get(first.get("date", "")) - for r in received: - date = arrow.get(r.get("date", "")) - delay = (date - base_date).seconds - r["delay"] = delay - base_date = date +def _normalize_received(received: list[dict]) -> list[dict]: + if len(received) == 0: + return [] - return received + received = [_normalize_received_date(r) for r in received] + received.reverse() + + first = received[0] + base_date = arrow.get(first.get("date", "")) + for r in received: + date = arrow.get(r.get("date", "")) + delay = (date - base_date).seconds + r["delay"] = delay + base_date = date + + return received + + +@safe +def normalize_header(parsed: dict) -> dict: + header = parsed.get("header", {}) + # set message-id as a top-level attribute + message_id = header.get("header", {}).get("message-id", []) + if len(message_id) > 0: + header["message_id"] = message_id[0] + + received = header.get("received", []) + header["received"] = _normalize_received(received) + parsed["header"] = header + return parsed + + +def _normalize_body(body: dict[str, Any]) -> dict[str, Any]: + content = body.get("content", "") + content_type = body.get("content_type", "") + body["urls"] = parse_urls_from_body(content, content_type) + body["emails"] = parse_email_addresses(content) + body["domains"] = parse_domain_names(content) + body["ip_addresses"] = parse_ipv4_addresses(content) + + for key in ["uri", "email", "domain", "ip"]: + body.pop(key, None) + + return body + + +@safe +def normalize_bodies(parsed: dict) -> dict: + bodies = parsed.get("body", []) + parsed["bodies"] = [_normalize_body(body) for body in bodies] + parsed.pop("body", None) + return parsed + + +@safe +def normalize_attachments(parsed: dict) -> dict: + # change "attachment" to "attachments" + attachments = parsed.get("attachment", []) + + non_inline_forward_attachments = [] + for attachment in attachments: + if not is_inline_forward_attachment(attachment): + non_inline_forward_attachments.append(attachment) + + parsed["attachments"] = non_inline_forward_attachments + parsed.pop("attachment", None) + return parsed + + +@safe +def transform(parsed: dict) -> schemas.Eml: + return schemas.Eml.model_validate(parsed) - def _normalize_header(self): - header = self.parsed.get("header", {}) - # set message-id as a top-level attribute - message_id = header.get("header", {}).get("message-id", []) - if len(message_id) > 0: - header["message_id"] = message_id[0] - - received = header.get("received", []) - header["received"] = self._normalize_received(received) - self.parsed["header"] = header - - def _normalize_body(self, body: dict[str, Any]) -> dict[str, Any]: - content = body.get("content", "") - content_type = body.get("content_type", "") - body["urls"] = parse_urls_from_body(content, content_type) - body["emails"] = parse_email_addresses(content) - body["domains"] = parse_domain_names(content) - body["ip_addresses"] = parse_ipv4_addresses(content) - - for key in ["uri", "email", "domain", "ip"]: - if key in body: - del body[key] - - return body - - def _normalize_bodies(self): - bodies = self.parsed.get("body", []) - self.parsed["bodies"] = [self._normalize_body(body) for body in bodies] - del self.parsed["body"] - - def _normalize_attachments(self): - # change "attachment" to "attachments" - attachments = self.parsed.get("attachment", []) - - non_inline_forward_attachments = [] - for attachment in attachments: - if not is_inline_forward_attachment(attachment): - non_inline_forward_attachments.append(attachment) - - self.parsed["attachments"] = non_inline_forward_attachments - if "attachment" in self.parsed: - del self.parsed["attachment"] - - def normalize(self): - self._normalize_header() - self._normalize_attachments() - self._normalize_bodies() - - def to_model(self) -> Eml: - self.normalize() - return Eml.model_validate(self.parsed) +class EmlFactory(AbstractFactory): @classmethod - def from_bytes(cls, data: bytes) -> Eml: - if is_eml_file(data): - obj = cls(data) - return obj.to_model() - - # assume data is a msg file - file = BytesIO(data) - message = Message(file) - email = message.to_email() - obj = cls(email.as_bytes()) - return obj.to_model() + def call(cls, data: bytes) -> schemas.Eml: + result: ResultE[schemas.Eml] = flow( + to_eml(data), + bind(parse), + bind(normalize_attachments), + bind(normalize_bodies), + bind(normalize_header), + bind(transform), + ) + return result.alt(raise_exception).unwrap() diff --git a/backend/factories/inquest.py b/backend/factories/inquest.py index 6582406..276b27d 100644 --- a/backend/factories/inquest.py +++ b/backend/factories/inquest.py @@ -1,139 +1,90 @@ -from dataclasses import dataclass, field from functools import partial import aiometer -from loguru import logger +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io -from backend.core.settings import INQUEST_API_KEY -from backend.schemas.verdict import Detail, Verdict -from backend.services.inquest import InQuest +from backend import clients, schemas, settings, types +NAME = "InQuest" -@dataclass -class InQuestAlert: - category: str - description: str - title: str - reference: str | None - @classmethod - def build(cls, dicts: list[dict]) -> list["InQuestAlert"]: - return [ - cls( - category=d.get("category", ""), - description=d.get("description", ""), - title=d.get("title", ""), - reference=d.get("reference"), - ) - for d in dicts - ] - - -@dataclass -class InQuestVerdict: - sha256: str - classification: str - alerts: list[InQuestAlert] = field(default_factory=list) +@future_safe +async def lookup(sha256: str, *, client: clients.InQuest) -> schemas.InQuestLookup: + return await client.lookup(sha256) - @property - def malicious(self) -> bool: - return self.classification == "MALICIOUS" - - @property - def reference_link(self) -> str: - return f"https://labs.inquest.net/dfi/sha256/{self.sha256}" - - @property - def description(self) -> str: - malicious_alerts = [ - alert for alert in self.alerts if alert.category == "malicious" - ] - descriptions = [alert.description for alert in malicious_alerts] - return " / ".join(descriptions) - - @classmethod - def build(cls, dict_: dict) -> "InQuestVerdict": - data = dict_.get("data", {}) - sha256 = data.get("sha256", "") - classification = data.get("classification", "") - alerts = data.get("inquest_alerts", []) - return cls( - sha256=sha256, - classification=classification, - alerts=InQuestAlert.build(alerts), - ) - - -async def get_result(client: InQuest, sha256: str) -> dict | None: - try: - return await client.dfi_details(sha256) - except Exception as e: - logger.exception(e) - return None - - -async def bulk_get_results(sha256s: list[str]) -> list[dict]: - if len(sha256s) == 0: - return [] - - client = InQuest() +@future_safe +async def bulk_lookup( + sha256s: types.ListSet[str], + *, + client: clients.InQuest, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, +) -> list[schemas.InQuestLookup]: + f_results = [lookup(sha256, client=client) for sha256 in set(sha256s)] results = await aiometer.run_all( - [partial(get_result, client, sha256) for sha256 in sha256s] + [f_result.awaitable for f_result in f_results], + max_at_once=max_at_once, + max_per_second=max_per_second, ) - return [result for result in results if result is not None] - - -async def get_inquest_verdicts(sha256s: list[str]) -> list[InQuestVerdict]: - if str(INQUEST_API_KEY) == "": - return [] - - results = await bulk_get_results(sha256s) - - verdicts: list[InQuestVerdict] = [] - for result in results: - verdicts.append(InQuestVerdict.build(result)) - - return verdicts - - -class InQuestVerdictFactory: - def __init__(self, sha256s: list[str]): - self.sha256s = sha256s - self.name = "InQuest" - - async def to_model(self) -> Verdict: - malicious_verdicts: list[InQuestVerdict] = [] - - verdicts = await get_inquest_verdicts(self.sha256s) - for verdict in verdicts: - if verdict.malicious: - malicious_verdicts.append(verdict) + values = [unsafe_perform_io(result.value_or(None)) for result in results] + return [value for value in values if value is not None] + + +@future_safe +async def transform(lookups: list[schemas.InQuestLookup], *, name: str = NAME): + malicious_lookups = [lookup for lookup in lookups if lookup.malicious] + + if len(malicious_lookups) == 0: + return schemas.Verdict( + name=name, + malicious=False, + details=[ + schemas.VerdictDetail( + key="benign", + description="There is no malicious attachment or InQuest doesn't have information about the attachments.", + ) + ], + ) - if len(malicious_verdicts) == 0: - return Verdict( - name=self.name, - malicious=False, - details=[ - Detail( - key="benign", - description="There is no malicious attachment or InQuest doesn't have information about the attachments.", - ) - ], + return schemas.Verdict( + name=name, + malicious=True, + score=100, + details=[ + schemas.VerdictDetail( + key=lookup.data.sha256, + description=lookup.description, + reference_link=lookup.reference_link, ) + for lookup in malicious_lookups + ], + ) - details: list[Detail] = [] - details = [ - Detail( - key=verdict.sha256, - description=verdict.description, - reference_link=verdict.reference_link, - ) - for verdict in malicious_verdicts - ] - return Verdict(name=self.name, malicious=True, score=100, details=details) +class InQuestVerdictFactory: @classmethod - async def from_sha256s(cls, sha256s: list[str]) -> Verdict: - obj = cls(sha256s) - return await obj.to_model() + async def call( + cls, + sha256s: types.ListSet[str], + *, + client: clients.InQuest, + name: str = NAME, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, + ) -> schemas.Verdict: + f_result: FutureResultE[schemas.Verdict] = flow( + bulk_lookup( + sha256s, + client=client, + max_at_once=max_at_once, + max_per_second=max_per_second, + ), + bind(partial(transform, name=name)), + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/factories/oldid.py b/backend/factories/oldid.py index 5221d6a..df17254 100644 --- a/backend/factories/oldid.py +++ b/backend/factories/oldid.py @@ -1,75 +1,98 @@ import base64 +import itertools -from loguru import logger +from returns.result import safe -from backend.schemas.eml import Attachment -from backend.schemas.verdict import Detail, Verdict -from backend.services.oleid import OleID +from backend import schemas +from backend.oleid import OleID +from .abstract import AbstractFactory -class OleIDVerdictFactory: - def __init__(self, attachments: list[Attachment]): - self.attachments = attachments - self.name = "oleid" +NAME = "oleid" - def _parse_as_ole_file(self, attachment: Attachment) -> list[Detail]: - details: list[Detail] = [] - data: bytes = base64.b64decode(attachment.raw) - oleid = OleID(data) - file_info = f"{attachment.filename}({attachment.hash.sha256})" - if oleid.has_vba_macros(): - key = "vba" - description = f"{file_info} contains VBA macros." - details.append(Detail(key=key, description=description)) +@safe +def parse(attachment: schemas.Attachment) -> OleID: + data: bytes = base64.b64decode(attachment.raw) + return OleID(data) - if oleid.has_xlm_macros(): - key = "xlm" - description = f"{file_info} contains XLM macros." - details.append(Detail(key=key, description=description)) - if oleid.has_flash_objects(): - key = "flash" - description = f"{file_info} contains Flash objects." - details.append(Detail(key=key, description=description)) +@safe +def attachment_to_details( + attachment: schemas.Attachment, +) -> list[schemas.VerdictDetail]: + details: list[schemas.VerdictDetail] = [] + file_info = f"{attachment.filename}({attachment.hash.sha256})" - if oleid.is_encrypted(): - key = "encrypted" - description = f"{file_info} is encrypted." - details.append(Detail(key=key, description=description)) + def inner(oleid: OleID): + if oleid.has_vba_macros: + details.append( + schemas.VerdictDetail( + key="vba", description=f"{file_info} contains VBA macros." + ) + ) - if oleid.has_external_relationships(): - key = "ext_rels" - description = f"{file_info} contains external relationships." - details.append(Detail(key=key, description=description)) + if oleid.has_xlm_macros: + details.append( + schemas.VerdictDetail( + key="xlm", description=f"{file_info} contains XLM macros." + ) + ) - if oleid.has_object_pool(): - key = "ObjectPool" - description = f"{file_info} contains an ObjectPool stream." - details.append(Detail(key=key, description=description)) + if oleid.has_flash_objects: + details.append( + schemas.VerdictDetail( + key="flash", description=f"{file_info} contains Flash objects." + ) + ) - return details + if oleid.has_encrypted: + details.append( + schemas.VerdictDetail( + key="encrypted", description=f"{file_info} is encrypted." + ) + ) - def to_model(self) -> Verdict: - details: list[Detail] = [] + if oleid.has_external_relationships: + details.append( + schemas.VerdictDetail( + key="ext_rels", + description=f"{file_info} contains external relationships.", + ) + ) + + if oleid.has_object_pool: + details.append( + schemas.VerdictDetail( + key="ObjectPool", + description=f"{file_info} contains an ObjectPool stream.", + ) + ) - for attachment in self.attachments: - try: - details.extend(self._parse_as_ole_file(attachment)) - except Exception as error: - logger.exception(error) + parse(attachment).map(inner) + return details + + +class OleIDVerdictFactory(AbstractFactory): + @classmethod + def call( + cls, attachments: list[schemas.Attachment], *, name: str = NAME + ) -> schemas.Verdict: + details = list( + itertools.chain.from_iterable( + [ + attachment_to_details(attachment).value_or([]) + for attachment in attachments + ] + ) + ) malicious = len(details) > 0 if not malicious: details.append( - Detail( + schemas.VerdictDetail( key="benign", description="There is no suspicious OLE file in attachments.", ) ) - return Verdict(name=self.name, malicious=malicious, details=details) - - @classmethod - def from_attachments(cls, attachments: list[Attachment]) -> Verdict: - obj = cls(attachments) - return obj.to_model() + return schemas.Verdict(name=name, malicious=malicious, details=details) diff --git a/backend/factories/response.py b/backend/factories/response.py index d504e34..d58fe84 100644 --- a/backend/factories/response.py +++ b/backend/factories/response.py @@ -2,68 +2,140 @@ from functools import partial import aiometer +from loguru import logger +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io -from backend.core.utils import ( - has_inquest_api_key, - has_urlscan_api_key, - has_virustotal_api_key, -) -from backend.factories.eml import EmlFactory -from backend.factories.inquest import InQuestVerdictFactory -from backend.factories.oldid import OleIDVerdictFactory -from backend.factories.spamassassin import SpamAssassinVerdictFactory -from backend.factories.urlscan import UrlscanVerdictFactory -from backend.factories.virustotal import VirusTotalVerdictFactory -from backend.schemas.eml import Attachment, Body -from backend.schemas.response import Response -from backend.schemas.verdict import Verdict - - -def aggregate_urls_from_bodies(bodies: list[Body]) -> list[str]: - urls: list[str] = [] - for body in bodies: - urls.extend(body.urls) - return list(set(urls)) - - -def aggregate_sha256s_from_attachments(attachments: list[Attachment]) -> list[str]: - sha256s: list[str] = [] - for attachment in attachments: - sha256s.append(attachment.hash.sha256) - return list(set(sha256s)) - - -class ResponseFactory: - def __init__(self, eml_file: bytes): - self.eml_file = eml_file - - async def to_model(self) -> Response: - eml = EmlFactory.from_bytes(self.eml_file) - id_ = hashlib.sha256(self.eml_file).hexdigest() - - urls = aggregate_urls_from_bodies(eml.bodies) - sha256s = aggregate_sha256s_from_attachments(eml.attachments) - - verdicts: list[Verdict] = [] - - async_tasks = [ - partial(SpamAssassinVerdictFactory.from_bytes, self.eml_file), - ] - if has_urlscan_api_key(): - async_tasks.append(partial(UrlscanVerdictFactory.from_urls, urls)) - if has_virustotal_api_key(): - async_tasks.append(partial(VirusTotalVerdictFactory.from_sha256s, sha256s)) - if has_inquest_api_key(): - async_tasks.append(partial(InQuestVerdictFactory.from_sha256s, sha256s)) - - # Add SpamAsassin, urlscan, virustotal verdicts - verdicts = await aiometer.run_all(async_tasks) - # Add OleID verdict - verdicts.append(OleIDVerdictFactory.from_attachments(eml.attachments)) - - return Response(eml=eml, verdicts=verdicts, id=id_) +from backend import clients, schemas, types +from .abstract import AbstractAsyncFactory +from .emailrep import EmailRepVerdictFactory +from .eml import EmlFactory +from .inquest import InQuestVerdictFactory +from .oldid import OleIDVerdictFactory +from .spamassassin import SpamAssassinVerdictFactory +from .urlscan import UrlScanVerdictFactory +from .virustotal import VirusTotalVerdictFactory + + +def log_exception(exception: Exception): + logger.exception(exception) + + +@future_safe +async def parse(eml_file: bytes) -> schemas.Response: + return schemas.Response( + eml=EmlFactory.call(eml_file), id=hashlib.sha256(eml_file).hexdigest() + ) + + +@future_safe +async def get_spam_assassin_verdict( + eml_file: bytes, *, client: clients.SpamAssassin +) -> schemas.Verdict: + return await SpamAssassinVerdictFactory.call(eml_file, client=client) + + +@future_safe +async def get_oleid_verdict(attachments: list[schemas.Attachment]) -> schemas.Verdict: + return OleIDVerdictFactory.call(attachments) + + +@future_safe +async def get_email_rep_verdicts(from_, *, client: clients.EmailRep) -> schemas.Verdict: + return await EmailRepVerdictFactory.call(from_, client=client) + + +@future_safe +async def get_urlscan_verdict( + urls: types.ListSet[str], *, client: clients.UrlScan +) -> schemas.Verdict: + return await UrlScanVerdictFactory.call(urls, client=client) + + +@future_safe +async def get_inquest_verdict( + sha256s: types.ListSet[str], *, client: clients.InQuest +) -> schemas.Verdict: + return await InQuestVerdictFactory.call(sha256s, client=client) + + +@future_safe +async def get_vt_verdict( + sha256s: types.ListSet[str], *, client: clients.VirusTotal +) -> schemas.Verdict: + return await VirusTotalVerdictFactory.call(sha256s, client=client) + + +@future_safe +async def set_verdicts( + response: schemas.Response, + *, + eml_file: bytes, + email_rep: clients.EmailRep, + spam_assassin: clients.SpamAssassin, + optional_vt: clients.VirusTotal | None = None, + optional_urlscan: clients.UrlScan | None = None, + optional_inquest: clients.InQuest | None = None, +) -> schemas.Response: + f_results: list[FutureResultE[schemas.Verdict]] = [ + get_spam_assassin_verdict(eml_file, client=spam_assassin), + get_oleid_verdict(response.eml.attachments), + ] + + if response.eml.header.from_ is not None: + f_results.append( + get_email_rep_verdicts(response.eml.header.from_, client=email_rep) + ) + + if optional_vt is not None: + f_results.append(get_vt_verdict(response.sha256s, client=optional_vt)) + + if optional_inquest is not None: + f_results.append(get_inquest_verdict(response.sha256s, client=optional_inquest)) + + if optional_urlscan is not None: + f_results.append(get_urlscan_verdict(response.urls, client=optional_urlscan)) + + results = await aiometer.run_all([f_result.awaitable for f_result in f_results]) + values = [ + unsafe_perform_io(result.alt(log_exception).value_or(None)) + for result in results + ] + response.verdicts = [value for value in values if value is not None] + return response + + +class ResponseFactory( + AbstractAsyncFactory, +): @classmethod - async def from_bytes(cls, eml_file: bytes) -> Response: - obj = cls(eml_file) - return await obj.to_model() + async def call( + cls, + eml_file: bytes, + *, + email_rep: clients.EmailRep, + spam_assassin: clients.SpamAssassin, + optional_vt: clients.VirusTotal | None = None, + optional_urlscan: clients.UrlScan | None = None, + optional_inquest: clients.InQuest | None = None, + ) -> schemas.Response: + f_result: FutureResultE[schemas.Response] = flow( + parse(eml_file), + bind( + partial( + set_verdicts, + eml_file=eml_file, + email_rep=email_rep, + spam_assassin=spam_assassin, + optional_vt=optional_vt, + optional_urlscan=optional_urlscan, + optional_inquest=optional_inquest, + ) + ), + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/factories/spamassassin.py b/backend/factories/spamassassin.py index 33214e2..4a1f070 100644 --- a/backend/factories/spamassassin.py +++ b/backend/factories/spamassassin.py @@ -1,54 +1,50 @@ -from dataclasses import dataclass - -from loguru import logger - -from backend.core import settings -from backend.schemas.verdict import Detail, Verdict -from backend.services.spamassassin import SpamAssassin - -HOST = settings.SPAMASSASSIN_HOST -PORT = settings.SPAMASSASSIN_PORT -TIMEOUT = settings.SPAMASSASSIN_TIMEOUT - - -@dataclass -class Result: - details: list[Detail] - score: float - malicious: bool - - -class SpamAssassinVerdictFactory: - def __init__(self, eml_file: bytes): - self.eml_file = eml_file - self.name = "SpamAssassin" - - async def _get_spam_assassin_report(self): - assassin = SpamAssassin(host=HOST, port=PORT, timeout=TIMEOUT) - return await assassin.report(self.eml_file) - - async def to_model(self) -> Verdict: - try: - report = await self._get_spam_assassin_report() - except Exception as error: - logger.exception(error) - return Verdict(name=self.name, malicious=False, details=[]) - - details: list[Detail] = [] - details = [ - Detail(key=detail.name, score=detail.score, description=detail.description) - for detail in report.details - ] - score = report.score - malicious = report.is_spam() - return Verdict( - name=self.name, - malicious=malicious, - score=score, - details=details, - ) +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io + +from backend import clients, schemas + +from .abstract import AbstractAsyncFactory + +NAME = "SpamAssassin" + +@future_safe +async def report( + eml_file: bytes, *, client: clients.SpamAssassin +) -> schemas.SpamAssassinReport: + return await client.report(eml_file) + + +@future_safe +async def transform( + report: schemas.SpamAssassinReport, *, name: str = NAME +) -> schemas.Verdict: + details = [ + schemas.VerdictDetail( + key=detail.name, score=detail.score, description=detail.description + ) + for detail in report.details + ] + score = report.score + malicious = report.is_spam() + return schemas.Verdict( + name=name, + malicious=malicious, + score=score, + details=details, + ) + + +class SpamAssassinVerdictFactory(AbstractAsyncFactory): @classmethod - async def from_bytes(cls, eml_file: bytes) -> Verdict: - obj = cls(eml_file) - return await obj.to_model() + async def call( + cls, eml_file: bytes, *, client: clients.SpamAssassin + ) -> schemas.Verdict: + f_result: FutureResultE[schemas.Verdict] = flow( + report(eml_file, client=client), bind(transform) + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/factories/urlscan.py b/backend/factories/urlscan.py index 5ef3299..74fa038 100644 --- a/backend/factories/urlscan.py +++ b/backend/factories/urlscan.py @@ -1,110 +1,94 @@ -from dataclasses import dataclass +import itertools from functools import partial import aiometer -from loguru import logger - -from backend.schemas.verdict import Detail, Verdict -from backend.services.urlscan import Urlscan - - -@dataclass -class UrlscanVerdict: - score: int - malicious: bool - uuid: str - url: str - - @property - def reference_link(self) -> str: - return f"https://urlscan.io/result/{self.uuid}/" - - @property - def description(self) -> str: - return f"{self.url} is malicious." - - -async def bulk_get_results(uuids: list[str]) -> list[dict]: - if len(uuids) == 0: - return [] - - api = Urlscan() - results = await aiometer.run_all([partial(api.result, uuid) for uuid in uuids]) - return [result for result in results if result is not None] - - -async def get_urlscan_verdicts(url: str) -> list[UrlscanVerdict]: - api = Urlscan() - - res = await api.search(url) - if res is None: - return [] - - results = res.get("results", []) - uuids = [result.get("_id", "") for result in results] - results = await bulk_get_results(uuids) - - verdicts: list[UrlscanVerdict] = [] - for result in results: - score = result.get("verdicts", {}).get("overall", {}).get("score") - malicous = result.get("verdicts", {}).get("overall", {}).get("malicious") - uuid = result.get("task", {}).get("uuid", "") - verdicts.append( - UrlscanVerdict(score=score, malicious=malicous, uuid=uuid, url=url) +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io + +from backend import clients, schemas, settings, types + +from .abstract import AbstractAsyncFactory + +NAME = "urlscan.io" + + +@future_safe +async def lookup(url: str, *, client: clients.UrlScan) -> schemas.UrlScanLookup: + return await client.lookup(url) + + +@future_safe +async def bulk_lookup( + urls: types.ListSet[str], + *, + client: clients.UrlScan, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, +) -> list[schemas.UrlScanLookup]: + f_results = [lookup(url, client=client) for url in set(urls)] + results = await aiometer.run_all( + [f_result.awaitable for f_result in f_results], + max_at_once=max_at_once, + max_per_second=max_per_second, + ) + values = [unsafe_perform_io(result.value_or(None)) for result in results] + return [value for value in values if value is not None] + + +@future_safe +async def transform(lookups: list[schemas.UrlScanLookup], *, name: str = NAME): + results = itertools.chain.from_iterable([lookup.results for lookup in lookups]) + malicious_results = [result for result in results if result.verdicts.malicious] + + if len(malicious_results) == 0: + return schemas.Verdict( + name=name, + malicious=False, + details=[ + schemas.VerdictDetail( + key="benign", + description="There is no malicious URL in bodies.", + ) + ], ) - return verdicts - - -def find_malicous_verdict(verdicts: list[UrlscanVerdict]) -> UrlscanVerdict | None: - for verdict in verdicts: - if verdict.malicious: - return verdict - return None - -class UrlscanVerdictFactory: - def __init__(self, urls: list[str]): - self.urls = urls - self.name = "urlscan.io" - - async def to_model(self) -> Verdict: - malicious_verdicts: list[UrlscanVerdict] = [] - - for url in self.urls: - try: - verdicts = await get_urlscan_verdicts(url) - malicious_verdict = find_malicous_verdict(verdicts) - if malicious_verdict is not None: - malicious_verdicts.append(malicious_verdict) - except Exception as e: - logger.exception(e) - continue - - if len(malicious_verdicts) == 0: - return Verdict( - name=self.name, - malicious=False, - details=[ - Detail( - key="benign", - description="There is no malicious URL in bodies.", - ) - ], + return schemas.Verdict( + name=name, + malicious=True, + score=100, + details=[ + schemas.VerdictDetail( + key=result.task.url, + description=f"{result.task.url} is malicious.", + reference_link=result.link, ) + for result in malicious_results + ], + ) - details: list[Detail] = [] - details = [ - Detail( - key=verdict.url, - score=verdict.score, - description=verdict.description, - reference_link=verdict.reference_link, - ) - for verdict in malicious_verdicts - ] - return Verdict(name=self.name, malicious=True, score=100, details=details) +class UrlScanVerdictFactory(AbstractAsyncFactory): @classmethod - async def from_urls(cls, urls: list[str]) -> Verdict: - obj = cls(urls) - return await obj.to_model() + async def call( + cls, + urls: types.ListSet[str], + *, + client: clients.UrlScan, + name: str = NAME, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, + ): + f_result: FutureResultE[schemas.Verdict] = flow( + bulk_lookup( + urls, + client=client, + max_at_once=max_at_once, + max_per_second=max_per_second, + ), + bind(partial(transform, name=name)), + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/factories/virustotal.py b/backend/factories/virustotal.py index f9fbca8..bc126eb 100644 --- a/backend/factories/virustotal.py +++ b/backend/factories/virustotal.py @@ -1,103 +1,96 @@ -from dataclasses import dataclass from functools import partial import aiometer import vt -from loguru import logger - -from backend.core.settings import VIRUSTOTAL_API_KEY -from backend.schemas.verdict import Detail, Verdict - - -@dataclass -class VirusTotalVerdict: - malicious: int - sha256: str - - @property - def reference_link(self) -> str: - return f"https://www.virustotal.com/gui/file/{self.sha256}/detection" - - @property - def description(self) -> str: - return f"{self.malicious} reports say {self.sha256} is malicious." - - -async def get_file(client: vt.Client, sha256: str) -> vt.Object | None: - try: - return await client.get_object_async(f"/files/{sha256}") - except Exception as e: - logger.exception(e) - return None - - -async def bulk_get_files(sha256s: list[str]) -> list[vt.Object]: - if str(VIRUSTOTAL_API_KEY) == "": - return [] - - if len(sha256s) == 0: - return [] - - async with vt.Client(str(VIRUSTOTAL_API_KEY)) as client: - files = await aiometer.run_all( - [partial(get_file, client, sha256) for sha256 in sha256s] +from returns.functions import raise_exception +from returns.future import FutureResultE, future_safe +from returns.pipeline import flow +from returns.pointfree import bind +from returns.unsafe import unsafe_perform_io + +from backend import clients, schemas, settings, types + +from .abstract import AbstractAsyncFactory + +NAME = "VirusTotal" + + +@future_safe +async def get_file_object(sha256: str, *, client: clients.VirusTotal) -> vt.Object: + return await client.get_object_async(f"/files/{sha256}") + + +@future_safe +async def bulk_get_file_objects( + sha256s: types.ListSet[str], + *, + client: clients.VirusTotal, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, +) -> list[vt.Object]: + f_results = [get_file_object(sha256, client=client) for sha256 in set(sha256s)] + results = await aiometer.run_all( + [f_result.awaitable for f_result in f_results], + max_at_once=max_at_once, + max_per_second=max_per_second, + ) + values = [unsafe_perform_io(result.value_or(None)) for result in results] + return [value for value in values if value is not None] + + +@future_safe +async def transform(objects: list[vt.Object], *, name: str = NAME) -> schemas.Verdict: + details: list[schemas.VerdictDetail] = [] + + for obj in objects: + malicious = int(obj.last_analysis_stats.get("malicious", 0)) + sha256 = str(obj.sha256) + if malicious == 0: + continue + + details.append( + schemas.VerdictDetail( + key=sha256, + score=malicious, + description=f"{malicious} reports say {sha256} is malicious.", + reference_link=f"https://www.virustotal.com/gui/file/{sha256}/detection", + ) ) - return [file_ for file_ in files if file_ is not None] + if len(details) == 0: + return schemas.Verdict( + name=name, + malicious=False, + details=[ + schemas.VerdictDetail( + key="benign", + description="There is no malicious attachment or VirusTotal doesn't have information about the attachments.", + ) + ], + ) -async def get_virustotal_verdicts(sha256s: list[str]) -> list[VirusTotalVerdict]: - if str(VIRUSTOTAL_API_KEY) == "": - return [] - - files = await bulk_get_files(sha256s) - - verdicts: list[VirusTotalVerdict] = [] - for file_ in files: - malicious = int(file_.last_analysis_stats.get("malicious", 0)) - sha256 = str(file_.sha256) - verdicts.append(VirusTotalVerdict(malicious=malicious, sha256=sha256)) - - return verdicts - - -class VirusTotalVerdictFactory: - def __init__(self, sha256s: list[str]): - self.sha256s = sha256s - self.name = "VirusTotal" - - async def to_model(self) -> Verdict: - malicious_verdicts: list[VirusTotalVerdict] = [] - - verdicts = await get_virustotal_verdicts(self.sha256s) - for verdict in verdicts: - if verdict.malicious > 0: - malicious_verdicts.append(verdict) - - if len(malicious_verdicts) == 0: - return Verdict( - name=self.name, - malicious=False, - details=[ - Detail( - key="benign", - description="There is no malicious attachment or VirusTotal doesn't have information about the attachments.", - ) - ], - ) + return schemas.Verdict(name=name, malicious=True, score=100, details=details) - details: list[Detail] = [] - details = [ - Detail( - key=verdict.sha256, - score=verdict.malicious, - description=verdict.description, - reference_link=verdict.reference_link, - ) - for verdict in malicious_verdicts - ] - return Verdict(name=self.name, malicious=True, score=100, details=details) +class VirusTotalVerdictFactory(AbstractAsyncFactory): @classmethod - async def from_sha256s(cls, sha256s: list[str]) -> Verdict: - obj = cls(sha256s) - return await obj.to_model() + async def call( + cls, + sha256s: types.ListSet[str], + *, + client: clients.VirusTotal, + name: str = NAME, + max_per_second: float | None = settings.ASYNC_MAX_PER_SECOND, + max_at_once: int | None = settings.ASYNC_MAX_AT_ONCE, + ) -> schemas.Verdict: + f_result: FutureResultE[schemas.Verdict] = flow( + bulk_get_file_objects( + sha256s, + client=client, + max_at_once=max_at_once, + max_per_second=max_per_second, + ), + bind(partial(transform, name=name)), + ) + result = await f_result.awaitable() + return unsafe_perform_io(result.alt(raise_exception).unwrap()) diff --git a/backend/main.py b/backend/main.py index 768901f..c5fa7dd 100644 --- a/backend/main.py +++ b/backend/main.py @@ -3,8 +3,8 @@ from fastapi.staticfiles import StaticFiles from loguru import logger +from backend import settings from backend.api.api import api_router -from backend.core import settings def create_app(): diff --git a/backend/oleid.py b/backend/oleid.py new file mode 100644 index 0000000..6e0465e --- /dev/null +++ b/backend/oleid.py @@ -0,0 +1,49 @@ +import oletools.oleid +from olefile import isOleFile +from returns.maybe import Maybe + +from backend.utils import is_truthy + + +class OleID: + def __init__(self, data: bytes): + self.oid: oletools.oleid.OleID | None = None + + if isOleFile(data): + self.oid = oletools.oleid.OleID(data=data) + self.oid.check() + + @property + def maybe_oid(self) -> Maybe[oletools.oleid.OleID]: + return Maybe.from_optional(self.oid) + + def _is_truthy_by_indicator_id(self, indicator_id: str) -> bool: + return ( + self.maybe_oid.bind_optional(lambda oid: oid.get_indicator(indicator_id)) + .bind_optional(lambda i: is_truthy(i.value)) + .value_or(False) + ) + + @property + def has_encrypted(self) -> bool: + return self._is_truthy_by_indicator_id("encrypted") + + @property + def has_vba_macros(self) -> bool: + return self._is_truthy_by_indicator_id("vba") + + @property + def has_xlm_macros(self) -> bool: + return self._is_truthy_by_indicator_id("xlm") + + @property + def has_flash_objects(self) -> bool: + return self._is_truthy_by_indicator_id("flash") + + @property + def has_external_relationships(self) -> bool: + return self._is_truthy_by_indicator_id("ext_rels") + + @property + def has_object_pool(self) -> bool: + return self._is_truthy_by_indicator_id("ObjectPool") diff --git a/backend/services/outlookmsgfile.py b/backend/outlookmsgfile.py similarity index 100% rename from backend/services/outlookmsgfile.py rename to backend/outlookmsgfile.py diff --git a/backend/schemas/__init__.py b/backend/schemas/__init__.py index 4d7eac3..5fc32a7 100644 --- a/backend/schemas/__init__.py +++ b/backend/schemas/__init__.py @@ -1,6 +1,9 @@ -from .emailrep import EmailRepResponse # noqa: F401 -from .eml import Eml # noqa: F401 +from .emailrep import EmailRepLookup # noqa: F401 +from .eml import Attachment, Body, Eml # noqa: F401 +from .inquest import InQuestLookup # noqa: F401 from .payload import FilePayload, Payload # noqa: F401 from .response import Response # noqa: F401 +from .spamassasin import SpamAssassinDetail, SpamAssassinReport # noqa: F401 from .submission import SubmissionResult # noqa: F401 -from .verdict import Verdict # noqa: F401 +from .urlscan import UrlScanLookup # noqa: F401 +from .verdict import Verdict, VerdictDetail # noqa: F401 diff --git a/backend/schemas/emailrep.py b/backend/schemas/emailrep.py index 8cd4327..ba52007 100644 --- a/backend/schemas/emailrep.py +++ b/backend/schemas/emailrep.py @@ -3,7 +3,7 @@ from .api_model import APIModel -class EmailRepResponse(APIModel): +class EmailRepLookup(APIModel): email: str reputation: str suspicious: bool diff --git a/backend/schemas/inquest.py b/backend/schemas/inquest.py new file mode 100644 index 0000000..b18aaef --- /dev/null +++ b/backend/schemas/inquest.py @@ -0,0 +1,34 @@ +from pydantic import BaseModel, Field + + +class InquestAlert(BaseModel): + category: str + description: str + reference: None + title: str + + +class Data(BaseModel): + sha256: str + classification: str + inquest_alerts: list[InquestAlert] = Field(default_factory=list) + + +class InQuestLookup(BaseModel): + data: Data + + @property + def malicious(self) -> bool: + return self.data.classification == "MALICIOUS" + + @property + def reference_link(self) -> str: + return f"https://labs.inquest.net/dfi/sha256/{self.data.sha256}" + + @property + def description(self) -> str: + malicious_alerts = [ + alert for alert in self.data.inquest_alerts if alert.category == "malicious" + ] + descriptions = [alert.description for alert in malicious_alerts] + return " / ".join(descriptions) diff --git a/backend/schemas/payload.py b/backend/schemas/payload.py index 12c2f55..95dbe3f 100644 --- a/backend/schemas/payload.py +++ b/backend/schemas/payload.py @@ -1,6 +1,6 @@ from pydantic import field_validator -from backend.services.validator import is_eml_or_msg_file +from backend.validator import is_eml_or_msg_file from .api_model import APIModel diff --git a/backend/schemas/response.py b/backend/schemas/response.py index 778bada..5d0cc78 100644 --- a/backend/schemas/response.py +++ b/backend/schemas/response.py @@ -1,10 +1,24 @@ -from backend.schemas.eml import Eml -from backend.schemas.verdict import Verdict +import itertools +from functools import cached_property + +from pydantic import Field from .api_model import APIModel +from .eml import Eml +from .verdict import Verdict class Response(APIModel): eml: Eml - verdicts: list[Verdict] + verdicts: list[Verdict] = Field(default_factory=list) id: str + + @cached_property + def urls(self) -> set[str]: + return set( + itertools.chain.from_iterable([body.urls for body in self.eml.bodies]) + ) + + @cached_property + def sha256s(self) -> set[str]: + return {attachment.hash.sha256 for attachment in self.eml.attachments} diff --git a/backend/schemas/spamassasin.py b/backend/schemas/spamassasin.py new file mode 100644 index 0000000..5decfe1 --- /dev/null +++ b/backend/schemas/spamassasin.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, Field + + +class SpamAssassinDetail(BaseModel): + name: str + score: float + description: str + + +class SpamAssassinReport(BaseModel): + score: float + details: list[SpamAssassinDetail] = Field(default_factory=list) + + def is_spam(self, level: float = 5.0) -> bool: + return self.score is None or self.score > level diff --git a/backend/schemas/urlscan.py b/backend/schemas/urlscan.py new file mode 100644 index 0000000..b51d51d --- /dev/null +++ b/backend/schemas/urlscan.py @@ -0,0 +1,28 @@ +from pydantic import BaseModel, Field + + +class Verdicts(BaseModel): + score: int + malicious: bool + + +class Page(BaseModel): + url: str + + +Task = Page + + +class Result(BaseModel): + page: Page + task: Task + verdicts: Verdicts + result: str + + @property + def link(self): + return self.result.replace("/api/v1/", "") + + +class UrlScanLookup(BaseModel): + results: list[Result] = Field(default_factory=list) diff --git a/backend/schemas/verdict.py b/backend/schemas/verdict.py index b02e99b..b07850f 100644 --- a/backend/schemas/verdict.py +++ b/backend/schemas/verdict.py @@ -1,15 +1,17 @@ +from pydantic import Field + from .api_model import APIModel -class Detail(APIModel): +class VerdictDetail(APIModel): key: str - score: float | int | None = None + score: float | int | None = Field(default=None) description: str - reference_link: str | None = None + reference_link: str | None = Field(default=None) class Verdict(APIModel): name: str malicious: bool - score: float | int | None = None - details: list[Detail] + score: float | int | None = Field(default=None) + details: list[VerdictDetail] = Field(default_factory=list) diff --git a/backend/services/__init__.py b/backend/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/services/emailrep.py b/backend/services/emailrep.py deleted file mode 100644 index f13e91e..0000000 --- a/backend/services/emailrep.py +++ /dev/null @@ -1,16 +0,0 @@ -from backend.core.resources import httpx_client -from backend.schemas.emailrep import EmailRepResponse - - -class EmailRep: - HOST = "emailrep.io" - BASE_URL = f"https://{HOST}" - - def __init__(self, client=httpx_client): - self.client = httpx_client - - async def get(self, email: str) -> EmailRepResponse: - url = f"{self.BASE_URL}/{email}" - r = await self.client.get(url) - r.raise_for_status() - return EmailRepResponse.model_validate(r.json()) diff --git a/backend/services/extractor.py b/backend/services/extractor.py deleted file mode 100644 index f96f836..0000000 --- a/backend/services/extractor.py +++ /dev/null @@ -1,62 +0,0 @@ -import urllib.parse - -import html2text -from bs4 import BeautifulSoup -from ioc_finder import parse_urls - - -def is_html(content_type: str) -> bool: - return "text/html" in content_type - - -def unpack_safelink_url(url: str) -> str: - # convert a Microsoft safelink back to a normal URL - parsed = urllib.parse.urlparse(url) - if parsed.netloc.endswith(".safelinks.protection.outlook.com"): - parsed_query = urllib.parse.parse_qs(parsed.query) - safelink_urls = parsed_query.get("url") - if safelink_urls is not None: - return urllib.parse.unquote(safelink_urls[0]) - - return url - - -def unpack_safelink_urls(urls: list[str]) -> list[str]: - return [unpack_safelink_url(url) for url in urls] - - -def normalize_url(url: str): - # remove ] and > from the end of the URL - url = url.rstrip(">") - return url.rstrip("]") - - -def normalize_urls(urls: list[str]) -> list[str]: - unique_urls = list(set(urls)) - return [normalize_url(url) for url in unique_urls] - - -def get_href_links(html: str) -> list[str]: - soup = BeautifulSoup(html, "html.parser") - links: list[str] = [str(link.get("href")) for link in soup.findAll("a")] - return [ - link - for link in links - if link.startswith("http://") or link.startswith("https://") - ] - - -def parse_urls_from_body(content: str, content_type: str) -> list[str]: - urls: list[str] = [] - - if is_html(content_type): - # extract href links - urls.extend(get_href_links(content)) - - # convert HTML to text - h = html2text.HTML2Text() - h.ignore_links = True - content = h.handle(content) - - urls.extend(parse_urls(content, parse_urls_without_scheme=False)) - return normalize_urls(unpack_safelink_urls(urls)) diff --git a/backend/services/inquest.py b/backend/services/inquest.py deleted file mode 100644 index 166e3d2..0000000 --- a/backend/services/inquest.py +++ /dev/null @@ -1,48 +0,0 @@ -from io import BytesIO -from typing import cast - -from httpx._client import AsyncClient -from httpx._exceptions import HTTPError - -from backend.core.resources import httpx_client -from backend.core.settings import INQUEST_API_KEY - - -class InQuest: - HOST = "labs.inquest.net" - BASE_URL = f"https://{HOST}/api" - - def __init__( - self, client: AsyncClient = httpx_client, api_key: str = str(INQUEST_API_KEY) - ): - self.client = client - self.api_key = api_key - - def _url_for(self, path: str) -> str: - return f"{self.BASE_URL}{path}" - - def _headers(self) -> dict: - return {"Authorization": f"Basic: {self.api_key}"} - - async def dfi_details(self, sha256: str) -> dict | None: - try: - r = await self.client.get( - self._url_for("/dfi/details"), - params={"sha256": sha256}, - headers=self._headers(), - ) - r.raise_for_status() - return cast(dict, r.json()) - except HTTPError: - return None - - async def dfi_upload(self, file_: BytesIO) -> dict: - files = {"file": file_} - - r = await self.client.post( - self._url_for("/dfi/upload"), - files=files, - headers=self._headers(), - ) - r.raise_for_status() - return cast(dict, r.json()) diff --git a/backend/services/oleid.py b/backend/services/oleid.py deleted file mode 100644 index d8e4f1a..0000000 --- a/backend/services/oleid.py +++ /dev/null @@ -1,71 +0,0 @@ -from typing import Any - -import oletools.oleid -from olefile import isOleFile - - -def is_truthy(v: Any) -> bool: - if v is None: - return False - - if isinstance(v, bool): - return v is True - - if isinstance(v, int): - return v > 0 - - try: - return str(v).upper() == "YES" - except Exception: - return False - - -class OleID: - def __init__(self, data: bytes): - self.oid: oletools.oleid.OleID | None = None - - if isOleFile(data): - self.oid = oletools.oleid.OleID(data=data) - self.oid.check() - - def is_encrypted(self) -> bool: - if self.oid is None: - return False - - encrypted = self.oid.get_indicator("encrypted") - return is_truthy(encrypted.value) - - def has_vba_macros(self) -> bool: - if self.oid is None: - return False - - macros = self.oid.get_indicator("vba") - return is_truthy(macros.value) - - def has_xlm_macros(self) -> bool: - if self.oid is None: - return False - - macros = self.oid.get_indicator("xlm") - return is_truthy(macros.value) - - def has_flash_objects(self) -> bool: - if self.oid is None: - return False - - flash = self.oid.get_indicator("flash") - return is_truthy(flash.value) - - def has_external_relationships(self) -> bool: - if self.oid is None: - return False - - flash = self.oid.get_indicator("ext_rels") - return is_truthy(flash.value) - - def has_object_pool(self) -> bool: - if self.oid is None: - return False - - flash = self.oid.get_indicator("ObjectPool") - return is_truthy(flash.value) diff --git a/backend/services/urlscan.py b/backend/services/urlscan.py deleted file mode 100644 index e94d868..0000000 --- a/backend/services/urlscan.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import cast - -from httpx._exceptions import HTTPError - -from backend.core.resources import httpx_client -from backend.core.settings import URLSCAN_API_KEY - - -class Urlscan: - HOST = "urlscan.io" - VERSION = "v1" - BASE_URL = f"https://{HOST}/api/{VERSION}" - - def __init__(self, client=httpx_client): - self.client = client - - def _url_for(self, path: str) -> str: - return f"{self.BASE_URL}{path}" - - def _headers(self) -> dict: - return {"API-Key": str(URLSCAN_API_KEY)} - - async def result(self, uuid: str) -> dict | None: - try: - r = await self.client.get( - self._url_for(f"/result/{uuid}/"), headers=self._headers() - ) - r.raise_for_status() - return cast(dict, r.json()) - except HTTPError: - return None - - async def search(self, url: str, size: int = 10) -> dict | None: - params = {"q": f'task.url:"{url}"', "size": size} - try: - r = await self.client.get( - self._url_for("/search/"), params=params, headers=self._headers() - ) - r.raise_for_status() - return cast(dict, r.json()) - except HTTPError: - return None diff --git a/backend/core/settings.py b/backend/settings.py similarity index 68% rename from backend/core/settings.py rename to backend/settings.py index 318af74..7f72f7f 100644 --- a/backend/core/settings.py +++ b/backend/settings.py @@ -27,6 +27,14 @@ REDIS_FIELD: str = config("REDIS_FIELD", cast=str, default="analysis") # 3rd party API keys -URLSCAN_API_KEY: Secret = config("URLSCAN_API_KEY", cast=Secret, default="") -VIRUSTOTAL_API_KEY: Secret = config("VIRUSTOTAL_API_KEY", cast=Secret, default="") -INQUEST_API_KEY: Secret = config("INQUEST_API_KEY", cast=Secret, default="") +VIRUSTOTAL_API_KEY: Secret | None = config( + "VIRUSTOTAL_API_KEY", cast=Secret, default=None +) +INQUEST_API_KEY: Secret | None = config("INQUEST_API_KEY", cast=Secret, default=None) +URLSCAN_API_KEY: Secret | None = config("URLSCAN_API_KEY", cast=Secret, default=None) + +# Async/aiometer +ASYNC_MAX_AT_ONCE: int | None = config("ASYNC_MAX_AT_ONCE", cast=int, default=None) +ASYNC_MAX_PER_SECOND: float | None = config( + "ASYNC_MAX_PER_SECOND", cast=float, default=None +) diff --git a/backend/submitters/__init__.py b/backend/submitters/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backend/submitters/abstract.py b/backend/submitters/abstract.py deleted file mode 100644 index ba34e21..0000000 --- a/backend/submitters/abstract.py +++ /dev/null @@ -1,22 +0,0 @@ -import base64 -from abc import ABC, abstractmethod -from io import BytesIO - -from backend.schemas.eml import Attachment -from backend.schemas.submission import SubmissionResult - - -class AbstractSubmitter(ABC): - def __init__(self, attachment: Attachment): - self.attachment = attachment - - def attachment_as_file(self) -> BytesIO: - bytes_ = base64.b64decode(self.attachment.raw) - - file_like = BytesIO(bytes_) - file_like.name = self.attachment.filename - return file_like - - @abstractmethod - async def submit(self) -> SubmissionResult: - raise NotImplementedError() diff --git a/backend/submitters/inquest.py b/backend/submitters/inquest.py deleted file mode 100644 index 5f51674..0000000 --- a/backend/submitters/inquest.py +++ /dev/null @@ -1,15 +0,0 @@ -from backend.schemas.submission import SubmissionResult -from backend.services.inquest import InQuest -from backend.submitters.abstract import AbstractSubmitter - - -class InQuestSubmitter(AbstractSubmitter): - async def submit(self) -> SubmissionResult: - client = InQuest() - file_ = self.attachment_as_file() - res = await client.dfi_upload(file_) - - data = res.get("data", "") - return SubmissionResult( - reference_url=f"https://labs.inquest.net/dfi/sha256/{data}" - ) diff --git a/backend/submitters/virustotal.py b/backend/submitters/virustotal.py deleted file mode 100644 index 2aed170..0000000 --- a/backend/submitters/virustotal.py +++ /dev/null @@ -1,15 +0,0 @@ -import vt - -from backend.core.settings import VIRUSTOTAL_API_KEY -from backend.schemas.submission import SubmissionResult -from backend.submitters.abstract import AbstractSubmitter - - -class VirusTotalSubmitter(AbstractSubmitter): - async def submit(self) -> SubmissionResult: - async with vt.Client(str(VIRUSTOTAL_API_KEY)) as client: - await client.scan_file_async(self.attachment_as_file()) - sha256 = self.attachment.hash.sha256 - return SubmissionResult( - reference_url=f"https://www.virustotal.com/gui/file/{sha256}/detection" - ) diff --git a/backend/types.py b/backend/types.py new file mode 100644 index 0000000..28d2873 --- /dev/null +++ b/backend/types.py @@ -0,0 +1,5 @@ +import typing + +T = typing.TypeVar("T") + +ListSet = typing.Union[list[T], set[T]] diff --git a/backend/utils.py b/backend/utils.py new file mode 100644 index 0000000..9c78e81 --- /dev/null +++ b/backend/utils.py @@ -0,0 +1,91 @@ +import base64 +import typing +import urllib.parse +from io import BytesIO +from typing import Any + +import html2text +from bs4 import BeautifulSoup +from ioc_finder import parse_urls + +from backend.schemas.eml import Attachment + + +def is_html(content_type: str) -> bool: + return "text/html" in content_type + + +def unpack_safelink_url(url: str) -> str: + # convert a Microsoft safelink back to a normal URL + parsed = urllib.parse.urlparse(url) + if parsed.netloc.endswith(".safelinks.protection.outlook.com"): + parsed_query = urllib.parse.parse_qs(parsed.query) + safelink_urls = parsed_query.get("url") + if safelink_urls is not None: + return urllib.parse.unquote(safelink_urls[0]) + + return url + + +def unpack_safelink_urls(urls: typing.Iterable[str]) -> set[str]: + return {unpack_safelink_url(url) for url in urls} + + +def normalize_url(url: str): + # remove ] and > from the end of the URL + url = url.rstrip(">") + return url.rstrip("]") + + +def normalize_urls(urls: typing.Iterable[str]) -> set[str]: + return {normalize_url(url) for url in urls} + + +def get_href_links(html: str) -> set[str]: + soup = BeautifulSoup(html, "html.parser") + links: set[str] = {str(link.get("href")) for link in soup.findAll("a")} + return { + link + for link in links + if link.startswith("http://") or link.startswith("https://") + } + + +def parse_urls_from_body(content: str, content_type: str) -> set[str]: + urls: set[str] = set() + + if is_html(content_type): + # extract href links + urls.update(get_href_links(content)) + + # convert HTML to text + h = html2text.HTML2Text() + h.ignore_links = True + content = h.handle(content) + + urls.update(parse_urls(content, parse_urls_without_scheme=False)) + return normalize_urls(unpack_safelink_urls(urls)) + + +def is_truthy(v: Any) -> bool: + if v is None: + return False + + if isinstance(v, bool): + return v is True + + if isinstance(v, int): + return v > 0 + + try: + return str(v).upper() == "YES" + except Exception: + return False + + +def attachment_to_file(attachment: Attachment) -> BytesIO: + bytes_ = base64.b64decode(attachment.raw) + + file_like = BytesIO(bytes_) + file_like.name = attachment.filename + return file_like diff --git a/backend/services/validator.py b/backend/validator.py similarity index 59% rename from backend/services/validator.py rename to backend/validator.py index 4fb3ba4..0ad5a9d 100644 --- a/backend/services/validator.py +++ b/backend/validator.py @@ -1,5 +1,3 @@ -from typing import cast - import magic EML_MIME_TYPES = ["message/rfc822", "text/html", "text/plain"] @@ -7,18 +5,13 @@ def check_mime_type(data: bytes, valid_types: list[str]) -> bool: - detected = magic.detect_from_content(data) - mime_type = cast(str, detected.mime_type) - - if mime_type in valid_types: - return True + detected = magic.detect_from_content(data) # type: ignore - return False + return str(detected.mime_type) in valid_types def is_eml_or_msg_file(data: bytes): - valid_types = EML_MIME_TYPES + MSG_MIME_TYPES - return check_mime_type(data, valid_types) + return check_mime_type(data, EML_MIME_TYPES + MSG_MIME_TYPES) def is_eml_file(data: bytes) -> bool: diff --git a/frontend/src/components/attachments/AttachmentSubmissionNotification.vue b/frontend/src/components/attachments/AttachmentSubmissionNotification.vue index 9e37c2b..6806799 100644 --- a/frontend/src/components/attachments/AttachmentSubmissionNotification.vue +++ b/frontend/src/components/attachments/AttachmentSubmissionNotification.vue @@ -2,7 +2,7 @@
The submission result will be available at here. - Please wait for a while.` + Please wait for a while.
diff --git a/poetry.lock b/poetry.lock index 40f01bc..55b151b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -120,20 +120,6 @@ files = [ [package.dependencies] anyio = ">=3.2,<5" -[[package]] -name = "aioresponses" -version = "0.7.6" -description = "Mock out requests made by ClientSession from aiohttp package" -optional = false -python-versions = "*" -files = [ - {file = "aioresponses-0.7.6-py2.py3-none-any.whl", hash = "sha256:d2c26defbb9b440ea2685ec132e90700907fd10bcca3e85ec2f157219f0d26f7"}, - {file = "aioresponses-0.7.6.tar.gz", hash = "sha256:f795d9dbda2d61774840e7e32f5366f45752d1adc1b74c9362afd017296c7ee1"}, -] - -[package.dependencies] -aiohttp = ">=3.3.0,<4.0.0" - [[package]] name = "aiosignal" version = "1.3.1" @@ -484,6 +470,17 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "ci-py" +version = "1.0.0" +description = "Python toolkit for working with Continuous Integration services." +optional = false +python-versions = "*" +files = [ + {file = "ci-py-1.0.0.tar.gz", hash = "sha256:47fe9b2ec5ce286c62243654bef3aebcba77bac1217e0ebdf2abef80ec015d89"}, + {file = "ci_py-1.0.0-py2.py3-none-any.whl", hash = "sha256:bc5d13c8dff8f402ac6340699083502115a2c55b96bf5b37204ac77bc81b605e"}, +] + [[package]] name = "circus" version = "0.18.0" @@ -883,13 +880,13 @@ test = ["coverage", "pytest"] [[package]] name = "fastapi" -version = "0.109.0" +version = "0.109.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.109.0-py3-none-any.whl", hash = "sha256:8c77515984cd8e8cfeb58364f8cc7a28f0692088475e2614f7bf03275eba9093"}, - {file = "fastapi-0.109.0.tar.gz", hash = "sha256:b978095b9ee01a5cf49b19f4bc1ac9b8ca83aa076e770ef8fd9af09a2b88d191"}, + {file = "fastapi-0.109.1-py3-none-any.whl", hash = "sha256:510042044906b17b6d9149135d90886ade170bf615efcfb5533f568ae6d88534"}, + {file = "fastapi-0.109.1.tar.gz", hash = "sha256:5402389843a3561918634eb327e86b9ae98645a9e7696bede9074449c48d610a"}, ] [package.dependencies] @@ -898,7 +895,7 @@ starlette = ">=0.35.0,<0.36.0" typing-extensions = ">=4.8.0" [package.extras] -all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] [[package]] name = "file-magic" @@ -1086,13 +1083,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "hypothesis" -version = "6.97.4" +version = "6.97.5" description = "A library for property-based testing" optional = false python-versions = ">=3.8" files = [ - {file = "hypothesis-6.97.4-py3-none-any.whl", hash = "sha256:9069fe3fb18d9b7dd218bd69ab50bbc66426819dfac7cc7168ba85034d98a4df"}, - {file = "hypothesis-6.97.4.tar.gz", hash = "sha256:28ff724fa81ccc55f64f0f1eb06e4a75db6a195fe0857e9b3184cf4ff613a103"}, + {file = "hypothesis-6.97.5-py3-none-any.whl", hash = "sha256:35fe2f7bf1e7a62f410d3fa9e67663ba242b48546f5a82a329ca773227a719c2"}, + {file = "hypothesis-6.97.5.tar.gz", hash = "sha256:67b552abce4d4f434c16dc3221d0ce45cdc78a6090ae3332c5fd59c44280c13a"}, ] [package.dependencies] @@ -1385,38 +1382,38 @@ files = [ [[package]] name = "mypy" -version = "1.8.0" +version = "1.5.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, - {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, - {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, - {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, - {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, - {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, - {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, - {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, - {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, - {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, - {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, - {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, - {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, - {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, - {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, - {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, - {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, - {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, - {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, - {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, - {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, - {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, - {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, - {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, - {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, - {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, - {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, + {file = "mypy-1.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f33592ddf9655a4894aef22d134de7393e95fcbdc2d15c1ab65828eee5c66c70"}, + {file = "mypy-1.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:258b22210a4a258ccd077426c7a181d789d1121aca6db73a83f79372f5569ae0"}, + {file = "mypy-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9ec1f695f0c25986e6f7f8778e5ce61659063268836a38c951200c57479cc12"}, + {file = "mypy-1.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:abed92d9c8f08643c7d831300b739562b0a6c9fcb028d211134fc9ab20ccad5d"}, + {file = "mypy-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:a156e6390944c265eb56afa67c74c0636f10283429171018446b732f1a05af25"}, + {file = "mypy-1.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ac9c21bfe7bc9f7f1b6fae441746e6a106e48fc9de530dea29e8cd37a2c0cc4"}, + {file = "mypy-1.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:51cb1323064b1099e177098cb939eab2da42fea5d818d40113957ec954fc85f4"}, + {file = "mypy-1.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:596fae69f2bfcb7305808c75c00f81fe2829b6236eadda536f00610ac5ec2243"}, + {file = "mypy-1.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:32cb59609b0534f0bd67faebb6e022fe534bdb0e2ecab4290d683d248be1b275"}, + {file = "mypy-1.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:159aa9acb16086b79bbb0016145034a1a05360626046a929f84579ce1666b315"}, + {file = "mypy-1.5.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f6b0e77db9ff4fda74de7df13f30016a0a663928d669c9f2c057048ba44f09bb"}, + {file = "mypy-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26f71b535dfc158a71264e6dc805a9f8d2e60b67215ca0bfa26e2e1aa4d4d373"}, + {file = "mypy-1.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc3a600f749b1008cc75e02b6fb3d4db8dbcca2d733030fe7a3b3502902f161"}, + {file = "mypy-1.5.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:26fb32e4d4afa205b24bf645eddfbb36a1e17e995c5c99d6d00edb24b693406a"}, + {file = "mypy-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:82cb6193de9bbb3844bab4c7cf80e6227d5225cc7625b068a06d005d861ad5f1"}, + {file = "mypy-1.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4a465ea2ca12804d5b34bb056be3a29dc47aea5973b892d0417c6a10a40b2d65"}, + {file = "mypy-1.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9fece120dbb041771a63eb95e4896791386fe287fefb2837258925b8326d6160"}, + {file = "mypy-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d28ddc3e3dfeab553e743e532fb95b4e6afad51d4706dd22f28e1e5e664828d2"}, + {file = "mypy-1.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:57b10c56016adce71fba6bc6e9fd45d8083f74361f629390c556738565af8eeb"}, + {file = "mypy-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:ff0cedc84184115202475bbb46dd99f8dcb87fe24d5d0ddfc0fe6b8575c88d2f"}, + {file = "mypy-1.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8f772942d372c8cbac575be99f9cc9d9fb3bd95c8bc2de6c01411e2c84ebca8a"}, + {file = "mypy-1.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5d627124700b92b6bbaa99f27cbe615c8ea7b3402960f6372ea7d65faf376c14"}, + {file = "mypy-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:361da43c4f5a96173220eb53340ace68cda81845cd88218f8862dfb0adc8cddb"}, + {file = "mypy-1.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:330857f9507c24de5c5724235e66858f8364a0693894342485e543f5b07c8693"}, + {file = "mypy-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:c543214ffdd422623e9fedd0869166c2f16affe4ba37463975043ef7d2ea8770"}, + {file = "mypy-1.5.1-py3-none-any.whl", hash = "sha256:f757063a83970d67c444f6e01d9550a7402322af3557ce7630d3c957386fa8f5"}, + {file = "mypy-1.5.1.tar.gz", hash = "sha256:b031b9601f1060bf1281feab89697324726ba0c0bae9d7cd7ab4b690940f0b92"}, ] [package.dependencies] @@ -1426,7 +1423,6 @@ typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] install-types = ["pip"] -mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] @@ -1811,6 +1807,25 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "pytest-docker" +version = "3.1.1" +description = "Simple pytest fixtures for Docker and Docker Compose based tests" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-docker-3.1.1.tar.gz", hash = "sha256:2371524804a752aaa766c79b9eee8e634534afddb82597f3b573da7c5d6ffb5f"}, + {file = "pytest_docker-3.1.1-py3-none-any.whl", hash = "sha256:fd0d48d6feac41f62acbc758319215ec9bb805c2309622afb07c27fa5c5ae362"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +pytest = ">=4.0,<9.0" + +[package.extras] +docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] +tests = ["mypy (>=0.500,<2.000)", "pytest-mypy (>=0.10,<1.0)", "pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)", "types-requests (>=2.31,<3.0)", "types-setuptools (>=69.0,<70.0)"] + [[package]] name = "pytest-env" version = "1.1.3" @@ -1930,17 +1945,17 @@ files = [ [[package]] name = "python-multipart" -version = "0.0.6" +version = "0.0.7" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.7" files = [ - {file = "python_multipart-0.0.6-py3-none-any.whl", hash = "sha256:ee698bab5ef148b0a760751c261902cd096e57e10558e11aca17646b74ee1c18"}, - {file = "python_multipart-0.0.6.tar.gz", hash = "sha256:e9925a80bb668529f1b67c7fdb0a5dacdd7cbfc6fb0bff3ea443fe22bdd62132"}, + {file = "python_multipart-0.0.7-py3-none-any.whl", hash = "sha256:b1fef9a53b74c795e2347daac8c54b252d9e0df9c619712691c1cc8021bd3c49"}, + {file = "python_multipart-0.0.7.tar.gz", hash = "sha256:288a6c39b06596c1b988bb6794c6fbc80e6c369e35e5062637df256bee0c9af9"}, ] [package.extras] -dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==1.7.3)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] +dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatch", "invoke (==2.2.0)", "more-itertools (==4.3.0)", "pbr (==4.3.0)", "pluggy (==1.0.0)", "py (==1.11.0)", "pytest (==7.2.0)", "pytest-cov (==4.0.0)", "pytest-timeout (==2.1.0)", "pyyaml (==5.1)"] [[package]] name = "pytz" @@ -2263,18 +2278,22 @@ socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] -name = "respx" -version = "0.20.2" -description = "A utility for mocking out the Python HTTPX and HTTP Core libraries." +name = "returns" +version = "0.22.0" +description = "Make your functions return something meaningful, typed, and safe!" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8.1,<4.0" files = [ - {file = "respx-0.20.2-py2.py3-none-any.whl", hash = "sha256:ab8e1cf6da28a5b2dd883ea617f8130f77f676736e6e9e4a25817ad116a172c9"}, - {file = "respx-0.20.2.tar.gz", hash = "sha256:07cf4108b1c88b82010f67d3c831dae33a375c7b436e54d87737c7f9f99be643"}, + {file = "returns-0.22.0-py3-none-any.whl", hash = "sha256:d38d6324692eeb29ec4bd698e1b859ec0ac79fb2c17bf0d302f92c8c42ef35c1"}, + {file = "returns-0.22.0.tar.gz", hash = "sha256:c7bd85bd1e0041b44fe46c7e2f68fcc76a0546142c876229e395174bcd674f37"}, ] [package.dependencies] -httpx = ">=0.21.0" +mypy = {version = ">=1.5,<1.6", optional = true, markers = "extra == \"compatible-mypy\""} +typing-extensions = ">=4.0,<5.0" + +[package.extras] +compatible-mypy = ["mypy (>=1.5,<1.6)"] [[package]] name = "rich" @@ -2364,6 +2383,26 @@ files = [ {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, ] +[[package]] +name = "stamina" +version = "24.2.0" +description = "Production-grade retries made easy." +optional = false +python-versions = ">=3.8" +files = [ + {file = "stamina-24.2.0-py3-none-any.whl", hash = "sha256:4dbd8076d2cb4e228046833e4507af3406cc31b9b6046e8a6729ffde934b2526"}, + {file = "stamina-24.2.0.tar.gz", hash = "sha256:8db72126f2342e428b153cbcf837f8a90f89b783aa55e19d4a17193116ee35ee"}, +] + +[package.dependencies] +tenacity = "*" + +[package.extras] +dev = ["nox", "prometheus-client", "stamina[tests,typing]", "structlog", "tomli", "trio"] +docs = ["furo", "myst-parser", "prometheus-client", "sphinx (>=7.2.2)", "sphinx-copybutton", "sphinx-notfound-page", "structlog"] +tests = ["anyio", "pytest"] +typing = ["mypy (>=1.4)"] + [[package]] name = "starlette" version = "0.35.1" @@ -2395,6 +2434,16 @@ files = [ [package.dependencies] mpmath = ">=0.19" +[[package]] +name = "syncer" +version = "2.0.3" +description = "Async to sync converter" +optional = false +python-versions = "*" +files = [ + {file = "syncer-2.0.3.tar.gz", hash = "sha256:4340eb54b54368724a78c5c0763824470201804fe9180129daf3635cb500550f"}, +] + [[package]] name = "tblib" version = "3.0.0" @@ -2406,6 +2455,20 @@ files = [ {file = "tblib-3.0.0.tar.gz", hash = "sha256:93622790a0a29e04f0346458face1e144dc4d32f493714c6c3dff82a4adb77e6"}, ] +[[package]] +name = "tenacity" +version = "8.2.3" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tenacity-8.2.3-py3-none-any.whl", hash = "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c"}, + {file = "tenacity-8.2.3.tar.gz", hash = "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a"}, +] + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "tokenize-rt" version = "5.2.0" @@ -2787,4 +2850,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "d2bcafb8054d96682e888949f73d47daa491af2e18fc33ffcfaf84221e056254" +content-hash = "329b3ff6e2a7ca8ff985cca96a226ac7f4b3db2b85f473e33b84ee22f17da52f" diff --git a/pyproject.toml b/pyproject.toml index e29db41..4591acc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,17 +30,19 @@ pyhumps = "^3.8" python-magic = "^0.4" python-multipart = "^0.0" redis = "^5.0" +returns = { extras = ["compatible-mypy"], version = "^0.22" } +stamina = "^24.2" uvicorn = "^0.25" vt-py = "^0.18" [tool.poetry.group.dev.dependencies] -aioresponses = "^0.7" black = "^24.1" +ci-py = "^1.0.0" coveralls = "^3.3" -mypy = "^1.8" pytest = "^7.4" pytest-asyncio = "^0.23" pytest-cov = "^4.1" +pytest-docker = "^3.1" pytest-env = "^1.1" pytest-mock = "^3.12" pytest-parallel = "^0.1" @@ -48,13 +50,12 @@ pytest-pretty = "^1.2" pytest-randomly = "^3.15" pytest-timeout = "^2.2" pyupgrade = "^3.15" -respx = "^0.20" ruff = "^0.2" +syncer = "^2.0" vcrpy = "^6.0" [tool.pytest.ini_options] asyncio_mode = "auto" -env = ["VIRUSTOTAL_API_KEY=foo", "INQUEST_API_KEY=bar", "TESTING=True"] [build-system] requires = ["poetry-core"] @@ -78,3 +79,7 @@ select = [ ignore = [ "E501", # line too long ] + +[tool.mypy] +ignore_missing_imports = true +plugins = ["pydantic.mypy", "returns.contrib.mypy.returns_plugin"] diff --git a/scripts/ping.py b/scripts/ping.py new file mode 100644 index 0000000..b06a5cc --- /dev/null +++ b/scripts/ping.py @@ -0,0 +1,18 @@ +import aiospamc +import stamina +from syncer import sync + + +@stamina.retry( + on=Exception, + attempts=10, + wait_initial=5.0, + timeout=60.0, +) +@sync +async def is_spam_assassin_responsive(): + await aiospamc.ping() + + +if __name__ == "__main__": + is_spam_assassin_responsive() # type: ignore diff --git a/test.docker-compose.yml b/test.docker-compose.yml new file mode 100644 index 0000000..f48540f --- /dev/null +++ b/test.docker-compose.yml @@ -0,0 +1,8 @@ +version: "3" +services: + spamassassin: + image: instantlinux/spamassassin:4.0.0-6 + platform: linux/x86_64 + ports: + - ${SPAMASSASSIN_PORT:-783}:783 + restart: always diff --git a/tests/api/endpoints/test_analyze.py b/tests/api/endpoints/test_analyze.py index 239b3ce..bf205f8 100644 --- a/tests/api/endpoints/test_analyze.py +++ b/tests/api/endpoints/test_analyze.py @@ -1,40 +1,32 @@ -import pytest -from httpx import AsyncClient +from fastapi import status +from fastapi.testclient import TestClient -from tests.conftest import read_file - -@pytest.mark.asyncio -async def test_analyze(client: AsyncClient): - payload = {"file": read_file("sample.eml")} - response = await client.post("/api/analyze/", json=payload) +def test_analyze(client: TestClient, sample_eml: bytes): + payload = {"file": sample_eml.decode()} + response = client.post("/api/analyze/", json=payload) json = response.json() assert json.get("eml", {}).get("header", {}).get("subject") == "Winter promotions" assert json.get("eml", {}).get("header", {}).get("from") == "no-reply@example.com" -@pytest.mark.asyncio -async def test_analyze_with_invalid_file(client: AsyncClient): +def test_analyze_with_invalid_file(client: TestClient): payload = {"file": ""} - response = await client.post("/api/analyze/", json=payload) - - assert response.status_code == 422 + response = client.post("/api/analyze/", json=payload) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY -@pytest.mark.asyncio -async def test_analyze_file(client: AsyncClient): - data = {"file": read_file("sample.eml").encode()} - response = await client.post("/api/analyze/file", files=data) +def test_analyze_file(client: TestClient, sample_eml: bytes): + data = {"file": sample_eml} + response = client.post("/api/analyze/file", files=data) json = response.json() assert json.get("eml", {}).get("header", {}).get("subject") == "Winter promotions" assert json.get("eml", {}).get("header", {}).get("from") == "no-reply@example.com" -@pytest.mark.asyncio -async def test_analyze_file_with_invalid_file(client: AsyncClient): +def test_analyze_file_with_invalid_file(client: TestClient): data = {"file": b""} - response = await client.post("/api/analyze/file", files=data) - - assert response.status_code == 422 + response = client.post("/api/analyze/file", files=data) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/tests/api/endpoints/test_submit.py b/tests/api/endpoints/test_submit.py index 55ce506..461e8c2 100644 --- a/tests/api/endpoints/test_submit.py +++ b/tests/api/endpoints/test_submit.py @@ -1,39 +1,27 @@ -import pytest -from pytest_mock import MockerFixture +from fastapi import status +from fastapi.testclient import TestClient -from backend.schemas.eml import Attachment +from backend import schemas -@pytest.mark.asyncio -async def test_submit_to_inquest_without_api_key( - client, docx_attachment: Attachment, mocker: MockerFixture +def test_submit_to_inquest_without_api_key( + client: TestClient, docx_attachment: schemas.Attachment ): - mocker.patch("backend.core.settings.INQUEST_API_KEY", return_value="") + response = client.post("/api/submit/inquest", json=docx_attachment.model_dump()) + assert response.status_code == status.HTTP_403_FORBIDDEN - payload = docx_attachment.dict() - response = await client.post("/api/submit/inquest", json=payload) - assert response.status_code == 403 - - -@pytest.mark.asyncio -async def test_submit_to_inquest_with_invalid_extension( - client, docx_attachment: Attachment +def test_submit_to_inquest_with_invalid_extension( + client: TestClient, docx_attachment: schemas.Attachment ): # change extension of the attachment docx_attachment.extension = "foo" - response = await client.post( - "/api/submit/inquest", json=docx_attachment.model_dump() - ) - assert response.status_code == 415 + response = client.post("/api/submit/inquest", json=docx_attachment.model_dump()) + assert response.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE -@pytest.mark.asyncio -async def test_submit_to_virustotal_without_api_key( - client, docx_attachment: Attachment, mocker: MockerFixture +def test_submit_to_virustotal_without_api_key( + client: TestClient, docx_attachment: schemas.Attachment ): - mocker.patch("backend.core.settings.VIRUSTOTAL_API_KEY", return_value="") - response = await client.post( - "/api/submit/virustotal", json=docx_attachment.model_dump() - ) - assert response.status_code == 403 + response = client.post("/api/submit/virustotal", json=docx_attachment.model_dump()) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/tests/conftest.py b/tests/conftest.py index c0345b5..0965e98 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,59 +1,80 @@ import glob -from pathlib import Path -from typing import Any +import os -import httpx +import aiospamc +import ci import pytest -import pytest_asyncio -from aiospamc.header_values import SpamValue -from aiospamc.responses import Response +from fastapi.testclient import TestClient +from pytest_docker.plugin import Services +from syncer import sync -from backend.factories.eml import EmlFactory +from backend import clients, factories, schemas from backend.main import create_app -from backend.schemas.eml import Attachment -def read_file(filename) -> str: - parent = Path(__file__).parent.absolute() - path = parent / f"fixtures/{filename}" - with open(path) as f: - return f.read() +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return os.path.join(str(pytestconfig.rootdir), "test.docker-compose.yml") -def read_file_as_binary(filename) -> bytes: - parent = Path(__file__).parent.absolute() - path = parent / f"fixtures/{filename}" - with open(path, "rb") as f: - return f.read() +@sync +async def is_spam_assassin_responsive(port: int) -> bool: + try: + res = await aiospamc.ping(port=port) + return res is not None + except Exception: + return False + + +if not ci.is_ci(): + + @pytest.fixture(scope="session", autouse=True) + def docker_compose(docker_ip: str, docker_services: Services): # type: ignore + port = docker_services.port_for("spamassassin", 783) + docker_services.wait_until_responsive( + timeout=60.0, pause=0.1, check=lambda: is_spam_assassin_responsive(port) + ) + +else: + + @pytest.fixture + def docker_compose(): + return + + +@pytest.fixture +def spam_assassin() -> clients.SpamAssassin: + return clients.SpamAssassin() @pytest.fixture def sample_eml() -> bytes: - return read_file("sample.eml").encode() + with open("tests/fixtures/sample.eml", "rb") as f: + return f.read() @pytest.fixture def cc_eml() -> bytes: - return read_file("cc.eml").encode() + with open("tests/fixtures/cc.eml", "rb") as f: + return f.read() @pytest.fixture def multipart_eml() -> bytes: - return read_file("multipart.eml").encode() + with open("tests/fixtures/multipart.eml", "rb") as f: + return f.read() @pytest.fixture def encrypted_docx_eml() -> bytes: - return read_file("encrypted_docx.eml").encode() + with open("tests/fixtures/encrypted_docx.eml", "rb") as f: + return f.read() @pytest.fixture def emails() -> list[bytes]: - parent = str(Path(__file__).parent.absolute()) - path = parent + "/fixtures/emails/**/*.eml" - emails: list[bytes] = [] - for p in glob.glob(path): + for p in glob.glob("tests/fixtures/emails/**/*.eml"): with open(p, "rb") as f: emails.append(f.read()) @@ -62,75 +83,47 @@ def emails() -> list[bytes]: @pytest.fixture def outer_msg() -> bytes: - return read_file_as_binary("outer.msg") + with open("tests/fixtures/outer.msg", "rb") as f: + return f.read() @pytest.fixture def other_msg() -> bytes: - return read_file_as_binary("other.msg") - - -@pytest.fixture -def emailrep_response() -> str: - return read_file("emailrep.json") - - -@pytest.fixture -def urlscan_search_response() -> str: - return read_file("urlscan_search.json") - - -@pytest.fixture -def urlscan_result_response() -> str: - return read_file("urlscan_result.json") - - -@pytest.fixture -def inquest_dfi_details_response() -> str: - return read_file("inquest_dfi_details.json") - - -@pytest.fixture -def inquest_dfi_upload_response() -> str: - return read_file("inquest_dfi_upload.json") + with open("tests/fixtures/other.msg", "rb") as f: + return f.read() @pytest.fixture def encrypted_docx() -> bytes: - return read_file_as_binary("encrypted.docx") + with open("tests/fixtures/encrypted.docx", "rb") as f: + return f.read() @pytest.fixture def xls_with_macro() -> bytes: - return read_file_as_binary("macro.xls") + with open("tests/fixtures/macro.xls", "rb") as f: + return f.read() @pytest.fixture def complete_msg() -> bytes: - return read_file_as_binary("complete.msg") + with open("tests/fixtures/complete.msg", "rb") as f: + return f.read() @pytest.fixture def test_html() -> str: - return read_file("test.html") + with open("tests/fixtures/test.html") as f: + return f.read() @pytest.fixture -def docx_attachment(encrypted_docx_eml: bytes) -> Attachment: - eml = EmlFactory.from_bytes(encrypted_docx_eml) +def docx_attachment(encrypted_docx_eml: bytes) -> schemas.Attachment: + eml = factories.EmlFactory.call(encrypted_docx_eml) return eml.attachments[0] @pytest.fixture -def spamassassin_response() -> Response: - body = read_file("sa.txt").encode() - headers: dict[str, Any] = {} - headers["Spam"] = SpamValue(value=True, score=40, threshold=20) - return Response(headers=headers, body=body) - - -@pytest_asyncio.fixture -async def client(): +def client() -> TestClient: app = create_app() - async with httpx.AsyncClient(app=app, base_url="http://testserver") as c: - yield c + return TestClient(app) diff --git a/tests/factories/test_emailrep.py b/tests/factories/test_emailrep.py deleted file mode 100644 index 268d923..0000000 --- a/tests/factories/test_emailrep.py +++ /dev/null @@ -1,17 +0,0 @@ -import httpx -import pytest -from respx import MockRouter - -from backend.factories.emailrep import EmailRepVerdictFactory - - -@pytest.mark.asyncio -async def test_bill(emailrep_response, respx_mock: MockRouter): - respx_mock.get( - "https://emailrep.io/bill@microsoft.com", - ).mock(return_value=httpx.Response(200, content=emailrep_response)) - - verdict = await EmailRepVerdictFactory.from_email("bill@microsoft.com") - assert verdict.malicious is False - assert len(verdict.details) == 1 - assert "is not suspicious" in verdict.details[0].description diff --git a/tests/factories/test_eml.py b/tests/factories/test_eml.py index d487d81..0c7e0cd 100644 --- a/tests/factories/test_eml.py +++ b/tests/factories/test_eml.py @@ -1,8 +1,11 @@ -from backend.factories.eml import EmlFactory, is_inline_forward_attachment +import pytest +from backend import factories +from backend.factories.eml import is_inline_forward_attachment -def test_sample(sample_eml): - eml = EmlFactory.from_bytes(sample_eml) + +def test_sample(sample_eml: bytes): + eml = factories.EmlFactory.call(sample_eml) assert eml.header.message_id is None assert eml.header.subject == "Winter promotions" assert eml.header.to == ["foo.bar@example.com"] @@ -11,8 +14,8 @@ def test_sample(sample_eml): assert len(eml.bodies) == 2 -def test_cc(cc_eml): - eml = EmlFactory.from_bytes(cc_eml) +def test_cc(cc_eml: bytes): + eml = factories.EmlFactory.call(cc_eml) assert eml.header.message_id == "ecc38b11-aa06-44c9-b8de-283b06a1d89e@example.com" assert eml.header.subject == "To and Cc headers" assert eml.header.to == ["foo.bar@example.com", "info@example.com"] @@ -25,8 +28,8 @@ def test_cc(cc_eml): assert eml.attachments == [] -def test_multipart(multipart_eml): - eml = EmlFactory.from_bytes(multipart_eml) +def test_multipart(multipart_eml: bytes): + eml = factories.EmlFactory.call(multipart_eml) assert eml.attachments is not None assert len(eml.attachments) == 1 @@ -35,8 +38,8 @@ def test_multipart(multipart_eml): assert first.hash.md5 == "f561388f7446cedd5b8b480311744b3c" -def test_encrypted_docx(encrypted_docx_eml): - eml = EmlFactory.from_bytes(encrypted_docx_eml) +def test_encrypted_docx(encrypted_docx_eml: bytes): + eml = factories.EmlFactory.call(encrypted_docx_eml) assert eml.attachments is not None assert len(eml.attachments) == 1 @@ -49,33 +52,38 @@ def test_encrypted_docx(encrypted_docx_eml): def test_emails(emails: list[bytes]): for email in emails: - try: - eml = EmlFactory.from_bytes(email) - assert eml is not None - except Exception: - pass - + eml = factories.EmlFactory.call(email) + assert eml is not None -def test_complete_msg(complete_msg): - eml = EmlFactory.from_bytes(complete_msg) +def test_complete_msg(complete_msg: bytes): + eml = factories.EmlFactory.call(complete_msg) assert eml.header.subject == "Test Multiple attachments complete email!!" -def test_is_inline_forward_attachment(): - inline_forward = { - "content_header": { - "content-type": ['message/rfc822; name="Fwd: foo"'], - "content-disposition": ['inline; filename="Fwd: foo"'], - } - } - assert is_inline_forward_attachment(inline_forward) is True - - zip_ = { - "content_header": { - "content-type": ['application/x-zip-compressed; name="foo.zip"'], - "content-transfer-encoding": ["base64"], - "content-disposition": ['attachment; filename="foo.zip"'], - } - } - assert is_inline_forward_attachment(zip_) is False +@pytest.mark.parametrize( + "attachment,expected", + [ + ( + { + "content_header": { + "content-type": ['message/rfc822; name="Fwd: foo"'], + "content-disposition": ['inline; filename="Fwd: foo"'], + } + }, + True, + ), + ( + { + "content_header": { + "content-type": ['application/x-zip-compressed; name="foo.zip"'], + "content-transfer-encoding": ["base64"], + "content-disposition": ['attachment; filename="foo.zip"'], + } + }, + False, + ), + ], +) +def test_is_inline_forward_attachment(attachment: dict, expected: bool): + assert is_inline_forward_attachment(attachment) is expected diff --git a/tests/factories/test_inquest.py b/tests/factories/test_inquest.py index c1a1fe5..97b0918 100644 --- a/tests/factories/test_inquest.py +++ b/tests/factories/test_inquest.py @@ -1,30 +1,25 @@ -import json - -import httpx import pytest -from respx import MockRouter - -from backend.factories.inquest import InQuestVerdict, InQuestVerdictFactory +import vcr +from starlette.datastructures import Secret +from backend import clients, factories, settings -def test_inquest_verdict(inquest_dfi_details_response: str): - sha256 = "e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149" - dict_ = json.loads(inquest_dfi_details_response) - verdict = InQuestVerdict.build(dict_) - assert verdict.sha256 == sha256 - assert verdict.malicious is True - assert verdict.reference_link == f"https://labs.inquest.net/dfi/sha256/{sha256}" +@pytest.fixture +async def client(): + async with clients.InQuest( + api_key=settings.INQUEST_API_KEY or Secret("") + ) as client: + yield client +@vcr.use_cassette( + "tests/fixtures/vcr_cassettes/inquest.yaml", filter_headers=["authorization"] +) # type: ignore @pytest.mark.asyncio -async def test_inquest(inquest_dfi_details_response: str, respx_mock: MockRouter): - sha256 = "e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149" - respx_mock.get( - f"https://labs.inquest.net/api/dfi/details?sha256={sha256}", - ).mock( - return_value=httpx.Response(200, content=inquest_dfi_details_response), +async def test_inquest_factory(client: clients.InQuest): + verdict = await factories.InQuestVerdictFactory.call( + ["e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149"], + client=client, ) - - verdict = await InQuestVerdictFactory.from_sha256s([sha256]) assert verdict.malicious is True diff --git a/tests/factories/test_oleid.py b/tests/factories/test_oleid.py index 3f4531a..2f633a5 100644 --- a/tests/factories/test_oleid.py +++ b/tests/factories/test_oleid.py @@ -1,24 +1,18 @@ -from backend.factories.eml import EmlFactory -from backend.factories.oldid import OleIDVerdictFactory -from backend.schemas.eml import Attachment +from backend import factories, schemas -def get_attachments(eml_file: bytes) -> list[Attachment]: - eml = EmlFactory.from_bytes(eml_file) +def get_attachments(eml_file: bytes) -> list[schemas.Attachment]: + eml = factories.EmlFactory.call(eml_file) return eml.attachments -def test_encrypted_docx(encrypted_docx_eml): - attachments = get_attachments(encrypted_docx_eml) - - verdict = OleIDVerdictFactory.from_attachments(attachments) +def test_encrypted_docx(encrypted_docx_eml: bytes): + verdict = factories.OleIDVerdictFactory.call(get_attachments(encrypted_docx_eml)) assert verdict.malicious is True assert len(verdict.details) == 1 -def test_sample(sample_eml): - attachments = get_attachments(sample_eml) - - verdict = OleIDVerdictFactory.from_attachments(attachments) +def test_sample(sample_eml: bytes): + verdict = factories.OleIDVerdictFactory.call(get_attachments(sample_eml)) assert verdict.malicious is False assert len(verdict.details) == 1 diff --git a/tests/factories/test_spamassassin.py b/tests/factories/test_spamassassin.py index 17256b3..512081c 100644 --- a/tests/factories/test_spamassassin.py +++ b/tests/factories/test_spamassassin.py @@ -1,16 +1,12 @@ -from unittest.mock import AsyncMock - import pytest -from backend.factories.spamassassin import SpamAssassinVerdictFactory +from backend import clients, factories @pytest.mark.asyncio -async def test_sample(sample_eml: bytes, spamassassin_response, mocker): - mock = AsyncMock() - mock.return_value = spamassassin_response - mocker.patch("aiospamc.report", mock) - - verdict = await SpamAssassinVerdictFactory.from_bytes(sample_eml) - assert verdict.malicious is True +async def test_sample(sample_eml: bytes, spam_assassin: clients.SpamAssassin): + verdict = await factories.SpamAssassinVerdictFactory.call( + sample_eml, client=spam_assassin + ) + assert verdict.malicious is False assert len(verdict.details) > 0 diff --git a/tests/factories/test_urlscan.py b/tests/factories/test_urlscan.py index e42346d..73970c3 100644 --- a/tests/factories/test_urlscan.py +++ b/tests/factories/test_urlscan.py @@ -1,40 +1,24 @@ -import httpx import pytest -from respx import MockRouter +import vcr +from starlette.datastructures import Secret -from backend.factories.urlscan import UrlscanVerdict, UrlscanVerdictFactory +from backend import clients, factories, settings -@pytest.mark.asyncio -async def test_urlscan( - urlscan_search_response: str, urlscan_result_response: str, respx_mock: MockRouter -): - uuid = "3db439ff-036f-409f-96d6-c28da55767f4" - respx_mock.get( - "https://urlscan.io/api/v1/search/?q=task.url%3A%22http%3A%2F%2Frakuten-ia.com%2F%22&size=10", - ).mock(return_value=httpx.Response(200, content=urlscan_search_response)) - respx_mock.get( - f"https://urlscan.io/api/v1/result/{uuid}/", - ).mock(return_value=httpx.Response(200, content=urlscan_result_response)) - - verdict = await UrlscanVerdictFactory.from_urls(["http://rakuten-ia.com/"]) - assert verdict.malicious is True +@pytest.fixture +async def client(): + async with clients.UrlScan( + api_key=settings.URLSCAN_API_KEY or Secret("") + ) as client: + yield client +@vcr.use_cassette( + "tests/fixtures/vcr_cassettes/urlscan.yaml", filter_headers=["api-key"] +) # type: ignore @pytest.mark.asyncio -async def test_urlscan_with_empty_response(respx_mock: MockRouter): - respx_mock.get( - "https://urlscan.io/api/v1/search/?q=task.url%3A%22http%3A%2F%2Frakuten-ia.com%2F%22&size=10", - ).mock( - return_value=httpx.Response(200, content="{}"), +async def test_urlscan_factory(client: clients.UrlScan): + verdict = await factories.UrlScanVerdictFactory.call( + ["http://example.com"], client=client ) - - verdict = await UrlscanVerdictFactory.from_urls(["http://rakuten-ia.com/"]) assert verdict.malicious is False - - -def test_urlscan_verdict(): - verdict = UrlscanVerdict( - score=0, malicious=True, uuid="foo", url="http://example.com" - ) - assert verdict.reference_link == "https://urlscan.io/result/foo/" diff --git a/tests/factories/test_virustotal.py b/tests/factories/test_virustotal.py index 0cdc168..2cfd488 100644 --- a/tests/factories/test_virustotal.py +++ b/tests/factories/test_virustotal.py @@ -1,35 +1,21 @@ import pytest -import vcr -from backend.factories.virustotal import VirusTotalVerdict, VirusTotalVerdictFactory +from backend import clients, factories, settings -@vcr.use_cassette( - "tests/fixtures/vcr_cassettes/vt.yaml", - filter_headers=["x-apikey"], -) -@pytest.mark.asyncio -async def test_virustotal(): - # eicar file - verdict = await VirusTotalVerdictFactory.from_sha256s( - ["275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"] - ) - assert verdict.malicious is True +@pytest.fixture +async def client(): + async with clients.VirusTotal( + apikey=str(settings.VIRUSTOTAL_API_KEY or "") + ) as client: + yield client +@pytest.mark.skip(reason="VCR cannot handle this...") @pytest.mark.asyncio -@vcr.use_cassette( - "tests/fixtures/vcr_cassettes/vt_non_malicious.yaml", - filter_headers=["x-apikey"], -) -async def test_virustotal_with_non_malicious_file(): - # empty file - verdict = await VirusTotalVerdictFactory.from_sha256s( - ["e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"] +async def test_virus_total_factory(client: clients.VirusTotal): + verdict = await factories.VirusTotalVerdictFactory.call( + ["275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"], + client=client, ) - assert verdict.malicious is False - - -def test_virustotal_verdict(): - verdict = VirusTotalVerdict(malicious=True, sha256="foo") - assert verdict.reference_link == "https://www.virustotal.com/gui/file/foo/detection" + assert verdict.malicious is True diff --git a/tests/fixtures/emailrep.json b/tests/fixtures/emailrep.json deleted file mode 100644 index ee73926..0000000 --- a/tests/fixtures/emailrep.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "email": "bill@microsoft.com", - "reputation": "high", - "suspicious": false, - "references": 79, - "details": { - "blacklisted": false, - "malicious_activity": false, - "malicious_activity_recent": false, - "credentials_leaked": true, - "credentials_leaked_recent": false, - "data_breach": true, - "first_seen": "07/01/2008", - "last_seen": "05/24/2019", - "domain_exists": true, - "domain_reputation": "high", - "new_domain": false, - "days_since_domain_creation": 10341, - "suspicious_tld": false, - "spam": false, - "free_provider": false, - "disposable": false, - "deliverable": true, - "accept_all": true, - "valid_mx": true, - "spoofable": false, - "spf_strict": true, - "dmarc_enforced": true, - "profiles": [ - "myspace", - "spotify", - "twitter", - "pinterest", - "flickr", - "linkedin", - "vimeo", - "angellist" - ] - } -} diff --git a/tests/fixtures/inquest_dfi_details.json b/tests/fixtures/inquest_dfi_details.json deleted file mode 100644 index 5855d7a..0000000 --- a/tests/fixtures/inquest_dfi_details.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "data": { - "analysis_completed": true, - "classification": "MALICIOUS", - "ext_code": "Attribute VB_Name = \"C0i6za70b_m8d\"\nAttribute VB_Base = \"0{91E8CCD0-24DD-46D7-84CE-1DBB59437D6C}{2ACD4A5D-E132-449D-A81A-E6D6F3D07A39}\"\nAttribute VB_GlobalNameSpace = False\nAttribute VB_Creatable = False\nAttribute VB_PredeclaredId = True\nAttribute VB_Exposed = False\nAttribute VB_TemplateDerived = False\nAttribute VB_Customizable = False\nFunction Uear8otu6_8c()\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nUf_da0kjip7cd9fz = 100\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nPdq6tkc2kxsy = ChrW(Uf_da0kjip7cd9fz + (15))\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nXksnmq27j8hbxwitrh = \"3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[w3%hs8( 8192&&&21gs [[]asd2[i3%hs8( 8192&&&21gs [[]asd2[nm3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[gm3%hs8( 8192&&&21gs [[]asd2[t3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[\" + Pdq6tkc2kxsy + \"3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[:3%hs8( 8192&&&21gs [[]asd2[w3%hs8( 8192&&&21gs [[]asd2[in3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[33%hs8( 8192&&&21gs [[]asd2[23%hs8( 8192&&&21gs [[]asd2[_3%hs8( 8192&&&21gs [[]asd2[\" + C0i6za70b_m8d.G_i79t6kr2vmmngjaf + \"3%hs8( 8192&&&21gs [[]asd2[ro3%hs8( 8192&&&21gs [[]asd2[3%hs8( 8192&&&21gs [[]asd2[ce3%hs8( 8192&&&21gs [[]asd2[s3%hs8( 8192&&&21gs [[]asd2[s3%hs8( 8192&&&21gs [[]asd2[\"\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nF60gwm8csrd_v3kvm = Zb43wcswx97(Xksnmq27j8hbxwitrh)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nSet Ninn8449hulve = CreateObject(F60gwm8csrd_v3kvm)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nC8bdmegzajch9_pue = C0i6za70b_m8d.Uclknrx3re0.ControlTipText\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nV4tsh1h0dr0517 = Mbpwmk8bkgivi8 + (F60gwm8csrd_v3kvm + Pdq6tkc2kxsy + C0i6za70b_m8d.Dq0e4bk8mepn.ControlTipText + C8bdmegzajch9_pue)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nJzh5n7sdap18_ = V4tsh1h0dr0517 + C0i6za70b_m8d.G_i79t6kr2vmmngjaf\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nSet Kqj3a7q3q2i9y9 = Jkr2hgqs4i897(Jzh5n7sdap18_)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nYkxctoz0wggitg = Array(Cajuznhyzuv6nb1 + \"Nvn5g9xsb4d4 Ebn71chlzhc2x0Yum88ayygmogo3s Goitsi0x0s65oldt_d\", Ninn8449hulve.Create(Sxribdiksv4m570az2, Mrg8psogum75ugk, Kqj3a7q3q2i9y9), Ippb1vz0of6j7i6hl + \"Aqds4bdbplcz Qds3euc683isj Dllrztng6z6wo9yf W16ignr2ximey\")\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nEnd Function\nFunction Jkr2hgqs4i897(Eusvnwn4fja2sa26v)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nSet Jkr2hgqs4i897 = CreateObject(Eusvnwn4fja2sa26v)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nJkr2hgqs4i897. _\nshowwindow = C0i6za70b_m8d.BorderStyle + C0i6za70b_m8d.HelpContextId\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nEnd Function\nFunction Zb43wcswx97(Agu6hsyakouo)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nQ678fwlwfgei8i = Trim(Conversion.CVar((Agu6hsyakouo)))\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nRsq35h30vei1q = Split(Q678fwlwfgei8i, \"3%hs8( 8192&&&21gs [[]asd2[\")\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nL30yptqwotxfwgwu = Ato02e6of9qj58tfeg + Join(Rsq35h30vei1q, Km3pp0951ckacg45)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nZb43wcswx97 = L30yptqwotxfwgwu\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nEnd Function\nFunction Sxribdiksv4m570az2()\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nPma8wwa58s78e89 = C0i6za70b_m8d.Opq5fj8wgfcx2j7.Tabs(1).ControlTipText\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nSxribdiksv4m570az2 = Zb43wcswx97(Pma8wwa58s78e89)\n On Error Resume Next\n uqnsw = (hiuqowgjv / 1 - 334 * CSng( _\n 55 * Tan(vAqg0) * kuhih * 6) * 6 _\n - CBool(Lll / Rnd( _\n uhJLQWIUO)))\n Set kHE = C0i6za70b_m8d\nEnd Function\n\nAttribute VB_Name = \"Pb92o9ip1hjg\"\nAttribute VB_Base = \"1Normal.ThisDocument\"\nAttribute VB_GlobalNameSpace = False\nAttribute VB_Creatable = False\nAttribute VB_PredeclaredId = True\nAttribute VB_Exposed = True\nAttribute VB_TemplateDerived = True\nAttribute VB_Customizable = True\nPrivate Sub _\nDocument_open()\nC0i6za70b_m8d.Uear8otu6_8c\nEnd Sub\n\n\n", - "ext_context": null, - "ext_metadata": "File Name : e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149.stream-5.olefileio.jpg.png\nFile Size : 399 kB\nFile Modification Date/Time : 2020:08:25 01:15:19+00:00\nFile Access Date/Time : 2020:08:25 01:15:19+00:00\nFile Inode Change Date/Time : 2020:08:25 01:15:19+00:00\nFile Permissions : rw-rwxr--\nFile Type : PNG\nFile Type Extension : png\nMIME Type : image/png\nImage Width : 2818\nImage Height : 1248\nBit Depth : 8\nColor Type : RGB\nCompression : Deflate/Inflate\nFilter : Adaptive\nInterlace : Noninterlaced\nBackground Color : 255 255 255\nPixels Per Unit X : 3700\nPixels Per Unit Y : 3700\nPixel Units : meters\nWarning : Install Compress::Zlib to read compressed information\nEXIF Profile : (Binary data 113 bytes, use -b option to extract)\nDatecreate : 2020-08-25T01:15:19+00:00\nDatemodify : 2020-08-25T01:15:19+00:00\nJpegcolorspace : 2\nJpegsampling-factor : 2x1,1x1,1x1\nImage Size : 2818x1248\nMegapixels : 3.5\n\nFile Name : e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149\nFile Size : 231 kB\nFile Modification Date/Time : 2020:08:25 01:15:15+00:00\nFile Access Date/Time : 2020:08:25 01:15:17+00:00\nFile Inode Change Date/Time : 2020:08:25 01:15:15+00:00\nFile Permissions : rw-rwxr--\nFile Type : DOC\nFile Type Extension : doc\nMIME Type : application/msword\nTitle : Fugiat.\nSubject : \nAuthor : Ma|eb|lys Garcia\nKeywords : \nComments : \nTemplate : Normal.dotm\nLast Modified By : \nRevision Number : 1\nSoftware : Microsoft Office Word\nTotal Edit Time : 0\nCreate Date : 2020:08:24 23:02:00\nModify Date : 2020:08:24 23:02:00\nPages : 1\nWords : 4\nCharacters : 23\nSecurity : None\nCompany : \nLines : 1\nParagraphs : 1\nChar Count With Spaces : 26\nApp Version : 15.0000\nScale Crop : No\nLinks Up To Date : No\nShared Doc : No\nHyperlinks Changed : No\nTitle Of Parts : \nHeading Pairs : Title, 1\nCode Page : Unicode UTF-16, little endian\nLocale Indicator : 1033\nComp Obj User Type Len : 32\nComp Obj User Type : Microsoft Word 97-2003 Document\n", - "ext_ocr": "Microsoft Word \n\nIf you are opening the attached file with Microsoft Word and you see a Protected view warning, then no values will be displayed until editing is enabled. \n\n", - "file_type": "DOC", - "first_seen": "Tue, 25 Aug 2020 01:13:10 GMT", - "image": true, - "inquest_alerts": [ - { - "category": "info", - "description": "Document contains between one and three pages of content. Most malicious documents are sparse in page count.", - "reference": null, - "title": "Document With Few Pages" - }, - { - "category": "malicious", - "description": "Detected heuristics indicative of an Emotet macro.", - "reference": null, - "title": "Emotet Macro Dropper 2020" - }, - { - "category": "suspicious", - "description": "Detected a macro that references a suspicious number of tersely named variables.", - "reference": null, - "title": "Suspicious Document Variables" - } - ], - "inquest_dfi_size": 729984, - "last_inquest_dfi": "Tue, 25 Aug 2020 01:15:27 GMT", - "last_inquest_featext": "Tue, 25 Aug 2020 01:17:15 GMT", - "last_updated": "Tue, 25 Aug 2020 01:17:16 GMT", - "len_code": 6699, - "len_context": 0, - "len_metadata": 3240, - "len_ocr": 195, - "md5": "b8ec0f5d681426b6dee1ff1398bac0f4", - "mime_type": "application/msword", - "sha1": "1e7e850f1352b330403bc2d683f6d05dec7aec74", - "sha256": "e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149", - "sha512": "5c323ed84a0d75549f7e850a0c42fcbc80c72bb729c1077cad4d3ca6ea047c15e1de923b854619a55f9cebb42e6b8c1519f1ac18002ecb39bae036134e1eea0b", - "size": 236541, - "subcategory": "macro_hunter", - "subcategory_url": "https://github.com/InQuest/yara-rules/blob/master/labs.inquest.net/macro_hunter.rule", - "virus_total": "https://www.virustotal.com/gui/file/e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149", - "vt_positives": 24, - "vt_weight": 10.699999809265137 - }, - "success": true -} diff --git a/tests/fixtures/inquest_dfi_upload.json b/tests/fixtures/inquest_dfi_upload.json deleted file mode 100644 index c5e87db..0000000 --- a/tests/fixtures/inquest_dfi_upload.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "data": "539e4557975be726d0fc8d7813dba5470dd703272957b4476296f976b5678900", - "success": true -} diff --git a/tests/fixtures/sa.txt b/tests/fixtures/sa.txt deleted file mode 100644 index 002b96f..0000000 --- a/tests/fixtures/sa.txt +++ /dev/null @@ -1,28 +0,0 @@ -SPAMD/1.1 0 EX_OK -Spam: False ; 2.2 / 5.0 - -Spam detection software, running on the system "9004c0f783a6", -has NOT identified this incoming email as spam. The original -message has been attached to this so you can view it or label -similar future email. If you have any questions, see -the administrator of that system for details. - -Content preview: Test Test - -Content analysis details: (2.2 points, 5.0 required) - - pts rule name description ----- ---------------------- -------------------------------------------------- - 0.0 FREEMAIL_FROM Sender email is commonly abused enduser mail - provider (foo[at]gmail.com) --0.0 NO_RELAYS Informational: message was not relayed via SMTP - 1.0 FORGED_GMAIL_RCVD 'From' gmail.com does not match 'Received' - headers - 0.0 DKIM_ADSP_CUSTOM_MED No valid author signature, adsp_override is - CUSTOM_MED - 0.0 HTML_MESSAGE BODY: HTML included in message --0.0 NO_RECEIVED Informational: message has no Received headers - 1.2 NML_ADSP_CUSTOM_MED ADSP custom_med hit, and not from a mailing - list - 0.0 T_FREEMAIL_DOC_PDF MS document or PDF attachment, from freemail - diff --git a/tests/fixtures/urlscan_result.json b/tests/fixtures/urlscan_result.json deleted file mode 100644 index 8d439bb..0000000 --- a/tests/fixtures/urlscan_result.json +++ /dev/null @@ -1,1313 +0,0 @@ -{ - "data": { - "requests": [ - { - "request": { - "requestId": "FBBFF1BB5C51082D0C247C55CD138D96", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/", - "method": "GET", - "headers": { - "Upgrade-Insecure-Requests": "1", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "VeryHigh", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.345207, - "wallTime": 1598146371.203674, - "initiator": { - "type": "other" - }, - "type": "Document", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 2024, - "dataLength": 4042, - "requestId": "FBBFF1BB5C51082D0C247C55CD138D96", - "type": "Document", - "response": { - "url": "http://rakuten-ia.com/", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Type": "text/html; charset=UTF-8", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "Set-Cookie": "__cfduid=d119bec6b0e8c631b943bdc18797d23941598146371; expires=Tue, 22-Sep-20 01:32:51 GMT; path=/; domain=.rakuten-ia.com; HttpOnly; SameSite=Lax", - "X-Frame-Options": "SAMEORIGIN", - "cf-request-id": "04ba8b76930000c2a457b03200000001", - "Vary": "Accept-Encoding", - "Server": "cloudflare", - "CF-RAY": "5c7115041bb2c2a4-FRA", - "Content-Encoding": "gzip" - }, - "mimeType": "text/html", - "requestHeaders": { - "Host": "rakuten-ia.com", - "Connection": "keep-alive", - "Pragma": "no-cache", - "Cache-Control": "no-cache", - "Upgrade-Insecure-Requests": "1", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", - "Accept-Encoding": "gzip, deflate", - "Accept-Language": "en-US" - }, - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 482, - "timing": { - "requestTime": 9360446.34566, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": 0.473, - "dnsEnd": 9.867, - "connectStart": 9.867, - "connectEnd": 15.017, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 15.07, - "sendEnd": 15.105, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 28.289 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "hash": "ca4cd26fdfe666a6da4b852f939c50b480e8748bf53524830762fe30ea50707b", - "size": 4042, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - } - }, - { - "request": { - "requestId": "21414.2", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "method": "GET", - "headers": { - "Referer": "http://rakuten-ia.com/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "VeryHigh", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.377392, - "wallTime": 1598146371.235851, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/", - "lineNumber": 12 - }, - "type": "Stylesheet", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 5295, - "dataLength": 28004, - "requestId": "21414.2", - "type": "Stylesheet", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "X-Frame-Options": "SAMEORIGIN", - "ETag": "W/\"5f3ab77a-6d64\"", - "Vary": "Accept-Encoding", - "Content-Type": "text/css", - "Cache-Control": "max-age=7200, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115043bc8c2a4-FRA", - "cf-request-id": "04ba8b76a50000c2a457b04200000001", - "Expires": "Sun, 23 Aug 2020 03:32:51 GMT" - }, - "mimeType": "text/css", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 470, - "timing": { - "requestTime": 9360446.377593, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": -1, - "dnsEnd": -1, - "connectStart": -1, - "connectEnd": -1, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 0.735, - "sendEnd": 0.766, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 7.57 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "hash": "ff5b724501640c081ba873f3d27b9f547b62ce5a4ef5d594ff630f00ba1eea7e", - "size": 28004, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.3", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/scripts/zepto.min.js", - "method": "GET", - "headers": { - "Referer": "http://rakuten-ia.com/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "High", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.377505, - "wallTime": 1598146371.235964, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/", - "lineNumber": 17 - }, - "type": "Script", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 9840, - "dataLength": 24975, - "requestId": "21414.3", - "type": "Script", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/scripts/zepto.min.js", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Vary": "Accept-Encoding", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "ETag": "W/\"5f3ab77a-618f\"", - "X-Frame-Options": "SAMEORIGIN", - "Content-Type": "application/javascript", - "Cache-Control": "max-age=172800, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115044907e00b-FRA", - "cf-request-id": "04ba8b76ac0000e00b8782d200000001", - "Expires": "Tue, 25 Aug 2020 01:32:51 GMT" - }, - "mimeType": "application/javascript", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 486, - "timing": { - "requestTime": 9360446.377767, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": 0.422, - "dnsEnd": 0.429, - "connectStart": 0.429, - "connectEnd": 5.573, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 5.608, - "sendEnd": 5.664, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 22.696 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "hash": "cdb3d0c8bdaa4ff0e4808dd9f53c33f0898fd934c3df605368b82a92c88ec049", - "size": 24975, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.4", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/scripts/cf.common.js", - "method": "GET", - "headers": { - "Referer": "http://rakuten-ia.com/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "High", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.377599, - "wallTime": 1598146371.236058, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/", - "lineNumber": 18 - }, - "type": "Script", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 2488, - "dataLength": 4408, - "requestId": "21414.4", - "type": "Script", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/scripts/cf.common.js", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Vary": "Accept-Encoding", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "ETag": "W/\"5f3ab77a-1138\"", - "X-Frame-Options": "SAMEORIGIN", - "Content-Type": "application/javascript", - "Cache-Control": "max-age=172800, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115044d1d05e4-FRA", - "cf-request-id": "04ba8b76aa000005e481bf8200000001", - "Expires": "Tue, 25 Aug 2020 01:32:51 GMT" - }, - "mimeType": "application/javascript", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 486, - "timing": { - "requestTime": 9360446.377908, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": 0.37, - "dnsEnd": 0.375, - "connectStart": 0.375, - "connectEnd": 5.536, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 5.56, - "sendEnd": 5.587, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 16.178 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "hash": "393c14162b5472e48358ba027ef7fc321d7761e6f4a86ea909b58ad9839177c4", - "size": 4408, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.20", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/images/icon-exclamation.png?1376755637", - "method": "GET", - "headers": { - "Referer": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "Low", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.40964, - "wallTime": 1598146371.268098, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css" - }, - "type": "Image", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 911, - "dataLength": 452, - "requestId": "21414.20", - "type": "Image", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/images/icon-exclamation.png?1376755637", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "X-Frame-Options": "SAMEORIGIN", - "ETag": "\"5f3ab77a-1c4\"", - "Vary": "Accept-Encoding", - "Content-Type": "image/png", - "Cache-Control": "max-age=7200, public", - "Connection": "keep-alive", - "Accept-Ranges": "bytes", - "CF-RAY": "5c7115046933e00b-FRA", - "Content-Length": "452", - "cf-request-id": "04ba8b76c50000e00b87832200000001", - "Expires": "Sun, 23 Aug 2020 03:32:51 GMT" - }, - "mimeType": "image/png", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 459, - "timing": { - "requestTime": 9360446.409887, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": -1, - "dnsEnd": -1, - "connectStart": -1, - "connectEnd": -1, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 0.303, - "sendEnd": 0.341, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 6.925 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "hash": "f1591a5221136c49438642155691ae6c68e25b7241f3d7ebe975b09a77662016", - "size": 604, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.8", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-300.woff", - "method": "GET", - "headers": { - "Origin": "http://rakuten-ia.com", - "Referer": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "VeryHigh", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.411132, - "wallTime": 1598146371.26959, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css" - }, - "type": "Font", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 15153, - "dataLength": 15868, - "requestId": "21414.8", - "type": "Font", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-300.woff", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "X-Frame-Options": "SAMEORIGIN", - "ETag": "W/\"5f3ab77a-3dfc\"", - "Vary": "Accept-Encoding", - "Content-Type": "application/font-woff", - "Cache-Control": "max-age=7200, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115047d6505e4-FRA", - "cf-request-id": "04ba8b76c6000005e481bf9200000001", - "Expires": "Sun, 23 Aug 2020 03:32:51 GMT" - }, - "mimeType": "application/font-woff", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 483, - "timing": { - "requestTime": 9360446.411399, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": -1, - "dnsEnd": -1, - "connectStart": -1, - "connectEnd": -1, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 0.251, - "sendEnd": 0.285, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 6.769 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.10", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-400.woff", - "method": "GET", - "headers": { - "Origin": "http://rakuten-ia.com", - "Referer": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "VeryHigh", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.411866, - "wallTime": 1598146371.270325, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css" - }, - "type": "Font", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 15227, - "dataLength": 15936, - "requestId": "21414.10", - "type": "Font", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-400.woff", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "X-Frame-Options": "SAMEORIGIN", - "ETag": "W/\"5f3ab77a-3e40\"", - "Vary": "Accept-Encoding", - "Content-Type": "application/font-woff", - "Cache-Control": "max-age=7200, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115047bf2c2a4-FRA", - "cf-request-id": "04ba8b76c70000c2a457b0b200000001", - "Expires": "Sun, 23 Aug 2020 03:32:51 GMT" - }, - "mimeType": "application/font-woff", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 483, - "timing": { - "requestTime": 9360446.412095, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": -1, - "dnsEnd": -1, - "connectStart": -1, - "connectEnd": -1, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 0.237, - "sendEnd": 0.27, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 6.775 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "host": "rakuten-ia.com", - "type": "parser" - } - }, - { - "request": { - "requestId": "21414.12", - "loaderId": "FBBFF1BB5C51082D0C247C55CD138D96", - "documentURL": "http://rakuten-ia.com/", - "request": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-600.woff", - "method": "GET", - "headers": { - "Origin": "http://rakuten-ia.com", - "Referer": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "mixedContentType": "none", - "initialPriority": "VeryHigh", - "referrerPolicy": "no-referrer-when-downgrade" - }, - "timestamp": 9360446.412646, - "wallTime": 1598146371.271104, - "initiator": { - "type": "parser", - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css" - }, - "type": "Font", - "frameId": "CF2B9C62484FDC896E4D53C3AACA7D22", - "hasUserGesture": false - }, - "response": { - "encodedDataLength": 15347, - "dataLength": 16056, - "requestId": "21414.12", - "type": "Font", - "response": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-600.woff", - "status": 200, - "statusText": "OK", - "headers": { - "Date": "Sun, 23 Aug 2020 01:32:51 GMT", - "Content-Encoding": "gzip", - "Last-Modified": "Mon, 17 Aug 2020 16:59:38 GMT", - "Server": "cloudflare", - "X-Frame-Options": "SAMEORIGIN", - "ETag": "W/\"5f3ab77a-3eb8\"", - "Vary": "Accept-Encoding", - "Content-Type": "application/font-woff", - "Cache-Control": "max-age=7200, public", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "CF-RAY": "5c7115047943e00b-FRA", - "cf-request-id": "04ba8b76cc0000e00b87833200000001", - "Expires": "Sun, 23 Aug 2020 03:32:51 GMT" - }, - "mimeType": "application/font-woff", - "remoteIPAddress": "[2606:4700:3031::6818:7bee]", - "remotePort": 80, - "fromPrefetchCache": false, - "encodedDataLength": 483, - "timing": { - "requestTime": 9360446.412931, - "proxyStart": -1, - "proxyEnd": -1, - "dnsStart": -1, - "dnsEnd": -1, - "connectStart": -1, - "connectEnd": -1, - "sslStart": -1, - "sslEnd": -1, - "workerStart": -1, - "workerReady": -1, - "sendStart": 4.476, - "sendEnd": 4.52, - "pushStart": 0, - "pushEnd": 0, - "receiveHeadersEnd": 11.332 - }, - "protocol": "http/1.1", - "securityState": "insecure", - "securityHeaders": [ - { - "name": "X-Frame-Options", - "value": "SAMEORIGIN" - } - ] - }, - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - }, - "initiatorInfo": { - "url": "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "host": "rakuten-ia.com", - "type": "parser" - } - } - ], - "cookies": [ - { - "name": "__cfduid", - "value": "d119bec6b0e8c631b943bdc18797d23941598146371", - "domain": ".rakuten-ia.com", - "path": "/", - "expires": 1600738371.232418, - "size": 51, - "httpOnly": true, - "secure": false, - "session": false, - "sameSite": "Lax", - "priority": "Medium" - } - ], - "console": [], - "links": [ - { - "href": "https://www.cloudflare.com/5xx-error-landing?utm_source=error_footer", - "text": "Cloudflare" - } - ], - "timing": { - "beginNavigation": "2020-08-23T01:32:51.203Z", - "frameStartedLoading": "2020-08-23T01:32:51.234Z", - "frameNavigated": "2020-08-23T01:32:51.235Z", - "domContentEventFired": "2020-08-23T01:32:51.272Z", - "loadEventFired": "2020-08-23T01:32:51.318Z", - "frameStoppedLoading": "2020-08-23T01:32:51.319Z" - }, - "globals": [ - { - "prop": "trustedTypes", - "type": "object" - }, - { - "prop": "Zepto", - "type": "function" - }, - { - "prop": "$", - "type": "function" - }, - { - "prop": "Polyglot", - "type": "function" - }, - { - "prop": "polyglot", - "type": "object" - }, - { - "prop": "_cf_translation", - "type": "object" - } - ] - }, - "stats": { - "resourceStats": [ - { - "count": 3, - "size": 47860, - "encodedSize": 45727, - "latency": 0, - "countries": ["US"], - "ips": ["[2606:4700:3031::6818:7bee]"], - "type": "Font", - "compression": "1.0", - "percentage": null - }, - { - "count": 2, - "size": 29383, - "encodedSize": 12328, - "latency": 0, - "countries": ["US"], - "ips": ["[2606:4700:3031::6818:7bee]"], - "type": "Script", - "compression": "2.4", - "percentage": null - }, - { - "count": 1, - "size": 452, - "encodedSize": 911, - "latency": 0, - "countries": ["US"], - "ips": ["[2606:4700:3031::6818:7bee]"], - "type": "Image", - "compression": "0.5", - "percentage": null - }, - { - "count": 1, - "size": 28004, - "encodedSize": 5295, - "latency": 0, - "countries": ["US"], - "ips": ["[2606:4700:3031::6818:7bee]"], - "type": "Stylesheet", - "compression": "5.3", - "percentage": null - }, - { - "count": 1, - "size": 4042, - "encodedSize": 2024, - "latency": 0, - "countries": ["US"], - "ips": ["[2606:4700:3031::6818:7bee]"], - "type": "Document", - "compression": "2.0", - "percentage": null - } - ], - "protocolStats": [ - { - "count": 8, - "size": 109741, - "encodedSize": 66285, - "ips": ["[2606:4700:3031::6818:7bee]"], - "countries": ["US"], - "securityState": {}, - "protocol": "http/1.1" - } - ], - "tlsStats": [ - { - "count": 8, - "size": 109741, - "encodedSize": 66285, - "ips": ["[2606:4700:3031::6818:7bee]"], - "countries": ["US"], - "protocols": {}, - "securityState": "insecure" - } - ], - "serverStats": [ - { - "count": 8, - "size": 109741, - "encodedSize": 66285, - "ips": ["[2606:4700:3031::6818:7bee]"], - "countries": ["US"], - "server": "cloudflare" - } - ], - "domainStats": [ - { - "count": 8, - "ips": ["2606:4700:3031::6818:7bee", "[2606:4700:3031::6818:7bee]"], - "domain": "rakuten-ia.com", - "size": 109741, - "encodedSize": 66285, - "countries": ["US"], - "index": 0, - "initiators": ["rakuten-ia.com"], - "redirects": 0 - } - ], - "regDomainStats": [ - { - "count": 8, - "ips": ["2606:4700:3031::6818:7bee", "[2606:4700:3031::6818:7bee]"], - "regDomain": "rakuten-ia.com", - "size": 109741, - "encodedSize": 66285, - "countries": [], - "index": 0, - "subDomains": [], - "redirects": 0 - } - ], - "secureRequests": 0, - "securePercentage": 0, - "IPv6Percentage": 100, - "uniqCountries": 1, - "totalLinks": 1, - "malicious": 0, - "adBlocked": 0, - "ipStats": [ - { - "requests": 8, - "domains": ["rakuten-ia.com"], - "ip": "2606:4700:3031::6818:7bee", - "asn": { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - }, - "dns": {}, - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - }, - "size": 109741, - "encodedSize": 66285, - "countries": ["US", "US", "US", "US", "US", "US", "US", "US"], - "index": 0, - "ipv6": true, - "redirects": 0, - "count": null - } - ] - }, - "meta": { - "processors": { - "rdns": { - "state": "done", - "data": [] - }, - "geoip": { - "state": "done", - "data": [ - { - "ip": "2606:4700:3031::6818:7bee", - "geoip": { - "range": "", - "country": "US", - "region": "", - "city": "", - "ll": [37.751, -97.822], - "metro": 0, - "area": 100, - "eu": "0", - "timezone": "America/Chicago", - "country_name": "United States" - } - } - ] - }, - "wappa": { - "state": "done", - "data": [ - { - "app": "CloudFlare", - "confidence": [ - { - "pattern": "headers server /^cloudflare$/i", - "confidence": 100 - } - ], - "confidenceTotal": 100, - "icon": "CloudFlare.svg", - "website": "http://www.cloudflare.com", - "categories": [ - { - "name": "CDN", - "priority": 9 - } - ] - } - ] - }, - "asn": { - "state": "done", - "data": [ - { - "ip": "2606:4700:3031::6818:7bee", - "asn": "13335", - "country": "US", - "registrar": "arin", - "date": "2010-07-14", - "description": "CLOUDFLARENET, US", - "route": "2606:4700:3031::/48", - "name": "CLOUDFLARENET" - } - ] - }, - "done": { - "state": "done", - "data": { - "state": "done" - } - } - } - }, - "task": { - "uuid": "3db439ff-036f-409f-96d6-c28da55767f4", - "time": "2020-08-23T01:32:51.069Z", - "url": "http://rakuten-ia.com", - "visibility": "public", - "options": { - "useragent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" - }, - "method": "api", - "source": "c708bdbe", - "tags": [], - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", - "reportURL": "https://urlscan.io/result/3db439ff-036f-409f-96d6-c28da55767f4/", - "screenshotURL": "https://urlscan.io/screenshots/3db439ff-036f-409f-96d6-c28da55767f4.png", - "domURL": "https://urlscan.io/dom/3db439ff-036f-409f-96d6-c28da55767f4/" - }, - "page": { - "url": "http://rakuten-ia.com/", - "domain": "rakuten-ia.com", - "country": "US", - "city": "", - "server": "cloudflare", - "ip": "2606:4700:3031::6818:7bee", - "asn": "AS13335", - "asnname": "CLOUDFLARENET, US" - }, - "lists": { - "ips": ["2606:4700:3031::6818:7bee"], - "countries": ["US"], - "asns": ["13335"], - "domains": ["rakuten-ia.com"], - "servers": ["cloudflare"], - "urls": [ - "http://rakuten-ia.com/", - "http://rakuten-ia.com/cdn-cgi/styles/cf.errors.css", - "http://rakuten-ia.com/cdn-cgi/scripts/zepto.min.js", - "http://rakuten-ia.com/cdn-cgi/scripts/cf.common.js", - "http://rakuten-ia.com/cdn-cgi/images/icon-exclamation.png?1376755637", - "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-300.woff", - "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-400.woff", - "http://rakuten-ia.com/cdn-cgi/styles/fonts/opensans-600.woff" - ], - "linkDomains": ["www.cloudflare.com"], - "certificates": [], - "hashes": [ - "ca4cd26fdfe666a6da4b852f939c50b480e8748bf53524830762fe30ea50707b", - "ff5b724501640c081ba873f3d27b9f547b62ce5a4ef5d594ff630f00ba1eea7e", - "cdb3d0c8bdaa4ff0e4808dd9f53c33f0898fd934c3df605368b82a92c88ec049", - "393c14162b5472e48358ba027ef7fc321d7761e6f4a86ea909b58ad9839177c4", - "f1591a5221136c49438642155691ae6c68e25b7241f3d7ebe975b09a77662016" - ] - }, - "verdicts": { - "overall": { - "score": 100, - "categories": ["phishing"], - "brands": ["generic cloudflare"], - "tags": ["phishing"], - "malicious": true, - "hasVerdicts": 100 - }, - "urlscan": { - "score": 100, - "categories": ["phishing"], - "brands": [ - { - "key": "genericcloudflare", - "name": "Generic CloudFlare", - "country": ["us"], - "vertical": ["Online"] - } - ], - "tags": ["phishing"], - "detectionDetails": [], - "malicious": true - }, - "engines": { - "score": 0, - "malicious": [], - "benign": [], - "maliciousTotal": 0, - "benignTotal": 0, - "verdicts": [], - "enginesTotal": 0 - }, - "community": { - "score": 0, - "votes": [], - "votesTotal": 0, - "votesMalicious": 0, - "votesBenign": 0, - "tags": [], - "categories": [] - }, - "raw": [ - null, - { - "key": "genericcloudflare", - "name": "Generic CloudFlare", - "country": ["us"], - "vertical": ["Online"] - } - ] - } -} diff --git a/tests/fixtures/urlscan_search.json b/tests/fixtures/urlscan_search.json deleted file mode 100644 index f79ea6e..0000000 --- a/tests/fixtures/urlscan_search.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "results": [ - { - "indexedAt": "2020-08-23T01:33:08.108Z", - "task": { - "visibility": "public", - "method": "api", - "domain": "rakuten-ia.com", - "time": "2020-08-23T01:32:51.069Z", - "uuid": "3db439ff-036f-409f-96d6-c28da55767f4", - "url": "http://rakuten-ia.com" - }, - "stats": { - "uniqIPs": 1, - "consoleMsgs": 0, - "uniqCountries": 1, - "dataLength": 109741, - "encodedDataLength": 66285, - "requests": 8 - }, - "page": { - "country": "US", - "server": "cloudflare", - "domain": "rakuten-ia.com", - "ip": "2606:4700:3031::6818:7bee", - "mimeType": "text/html", - "asnname": "CLOUDFLARENET, US", - "asn": "AS13335", - "url": "http://rakuten-ia.com/", - "status": "200" - }, - "_id": "3db439ff-036f-409f-96d6-c28da55767f4", - "sort": [1598146371069, "3db439ff-036f-409f-96d6-c28da55767f4"], - "result": "https://urlscan.io/api/v1/result/3db439ff-036f-409f-96d6-c28da55767f4/", - "screenshot": "https://urlscan.io/screenshots/3db439ff-036f-409f-96d6-c28da55767f4.png" - } - ], - "total": 5, - "took": 52, - "has_more": false -} diff --git a/tests/fixtures/vcr_cassettes/inquest.yaml b/tests/fixtures/vcr_cassettes/inquest.yaml new file mode 100644 index 0000000..7edc1c5 --- /dev/null +++ b/tests/fixtures/vcr_cassettes/inquest.yaml @@ -0,0 +1,103 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + host: + - labs.inquest.net + user-agent: + - python-httpx/0.26.0 + method: GET + uri: https://labs.inquest.net/api/dfi/details?sha256=e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149 + response: + body: + string: !!binary | + H4sIAAAAAAAAA9RXW2+jOBT+KxbSVnSbdLmFS6V9SCHtptvLzLSd7mwzQgY74AA2wYYQqvnv66RT + adPs8DDSPoQXZL5jn3O+zz7HvCgICqicvSiQwnzNCQ9jVpQ5FhgpZ6Kq8UCJc8g5mZMYCsKocqbc + jK+n/vTu8V6RIMtzHG8Arpw9fx0oiK1oziCCUY7fVsCtkMsiOVbGQlQkqgUGn8/DW1hg8DuYKb5G + 7A46WhQWLpopM7pjdg75q5n24ukT1/cDbWhYQTC07MAZupY/GerB+fnIs0wnsP1vL8bYD6zxKBhO + dNMYWpYXDMeuPh5O7MC+MAPNGZvetz03lzmLYL6J6b6E8cbjBcw5fheMX2HJmEzuB/iHCiMsKZOv + KZI2D5LDd0tM2pJxvAH/y8EDlvxDgQNckeaHVn7NBStItxvJRU23WoBHDCuXidoO3Vg9nlGwfe4o + mFQVq8AnzGtJ/a0UZovVS8pXMh41JfWSrZJFA34DOhgC07TAr8C/p4kKwq3paCQ/PECqNuNloh3L + QVanJJVvezOwv5sNgX/OWK5e57lc6hNFb/Pr9Or649P08e74+DWueyxA9sdEet/dBfRxHiKoZQtS + OjHy5p200DXtMHP5gJa2yGIja/l6k2laPal7+Z0AVR99Z0XqdVhq/ZVxWiwNZ+GmUbsiokplnjPF + /CXlrgpc3TOOjo4MPeHg+fkr5Mh47oFWPRjpwWjRA/ZASd880TOxB5op4ATs6H7y03yc9fjp5Yr2 + TOyDejCjBwt7sC0fO0f89DIkjifsrDKaoqDJAs4lZb17pmI9HnqgGPeA/CexTQc5xMJ6YWvJqnBj + XqGwMbOmkCf178gyVzFftZ6j7h/lA+0gm9ZySyh15RUgrfNm07O3/RvfRQt5a1H3mDjQRH03QgVO + OriIUy8s622i/75TnT7GeUar1qywduozKiqWP5Dy4e0CcHDt5rMleKqnGqq0ke5IXW+iclVkbpQl + pCGurCP74u7X491yFCw1bEWZW+CSviNJTt3j+ED3ylWXjqjDESx1N5TEvWNSJrqzcfZr9GEWvU0t + +HO5MKGzNJcG8daezP1Ktp40WXKLuLLu7TBzoOp+ydpYsE5bJQkRiUxxXFVwrfpwUXc0XXd1Y9NI + 37bZ24aOEq/lkYUsMImoo8dp3qWx0Wpf6sJ14XqdFCxhJgeXjAhOtFbj9ojlSITyJ22wW1pPXwur + et/K/ztEMt5YxcjRYGcMwE2VuCVnSV04ozrJBu+EOB6AaVlGetNpbG4vHGKn+TbC8RJxK0JRmccd + +Ii4ievYdk3CFyDI86oTNLE7e8W89Rw86TZJaGW0pMDrmfK/y/cPAAAA///cmeFzmjAUwP+VXD+5 + 22oBi2K+aWs7dmvHqlu7nXc71ABRSDAE0f31e8HSCdT2025H9RTElwfv5b0kL79/U/OM2AIVRdyU + FWeVOB2lyYZl7NxbukbiGt1NQ41VKVnKQAjX0vT8VgwtGdlWFXIS8CyjDDZLlM2lAXfIxYKIsdzB + FkN1MP5IwljN3jBt24tmjsNHIvxw/Tnw026Q7NwVT3lDQ/trt2d5WZh5PqEWhT6eCBq1oOs2RCSw + Wda++O6KVqtkaVMr/7tk3TGDjrYhVF+DqeM4pLJV9sCHl8u6xo7XnzvaLpbrjMutl/lZCuYPJNcM + 0uVef700LekRH9L4E6esVXIUTIJRJ461vqnPV+7cPzcbGuoHmQvWVx3ypgap+tqmqXurTuRaWeaa + VtKziKWWwuVJ6Eu8Nr2llfnefGsse+2JO0ta+rtKVdTMzq33Iph/EMWtinMampflibaCIJ7IizPr + G7xPYz1Y+jUi8gRe9FsuIjdsTwKaXPI5sAMma9L/gZ88A1fq+OQZoQo92Us4wFuAu6BxOlNLtMLO + XzwmTOV5OUMOCcuUKWdDuylTb8Bie+KVL9ROMEvDcH8pIhLImAJuJ1cU1ndj+pvsdxBr3xh1+n20 + GkIVoCRv+OIJwaFLeMyzCRQ6eSuMDM3QsGZhw0SajnUT6/33GlwBWJI3HsznJEkqzfZ3fL2xzYDc + AbFwmU8qKl5v7BARUaCHgAerFmIkslORbcXp6eNjTnbxcW84t9eHYqOtJEzpLanFKGb+lN3YNyP0 + ojYauT45y4VtdYru6UIGJV3FDzDS0q0p28t9JNQPZPHf4REj3TgHuSGV6JLER7QhhBEIXfAQMNzx + R8To7hq6/gJYrICuq9pZ3BfDnTwFC89slh9zF0kiCoH6EaMBbP1IQItgEgSoCBXrfP6F0S1ntBCC + gmPozle+4ClE+96CSjvwFfDBx8+UOXRLwgRBEKBvDNzyUBPv9FSQVuV+vCSXq6pFU94CI0gwWGBP + 2b0r4MH9ip6/PzGyWSJdgJOFizH+GdIZkhwB4F0gRcGV5wHAUuap0U8xbkj0B/sKOYJ7fwAAAP// + rFrfc9s2DH7fX8HL0x4sW9QvW35su67d0qVb0z3lzkdRlM2rLakiFce35n/fB8pW5NhJ262++CJZ + AAGCwAeQAgXlmc+c/fxCl6LZMYpzxnnIsp1VZsRavMT2MlbB9vBayAFGNEJaIAvFs6T3ymeHJJeh + CPf8mRfE148jnJg3hA67M+rgp2eZf6vVEq/wq8a4V94nI4D5piQiI/BWGhb1CqgM1x1+QHTHR7z7 + HgLlaWgDOQLqrouWd2op6s5NhkP214DBcUyg+nW8DEL+n/Ey/j94OT1i/k68PJb84/Dy1dXLb8HL + vJLfgpeixup3DSCTjdniiOKmvNb2fAzsF2+Ok6ylFnZ8UyI10vuWflUfX8wZ6pPWrh451hHZnL0T + X1T2BV0q7FfRSC1uyt/VjnQ5jwbgpnER4FSuPEtzKBqOBA5uCAhdAZRXdnNTXgpj9xkZ+PDiJPBI + 7F/qVjvc/qPdZGcAGekCdqkKu0WryEDU0SXmrGVTGZCxqwItOEhUne0rgBf7JQeo9nXAkHPOAKvd + WZrL2cNn/XWHDK5wiFgQzv3AlQyu1Nh9N9t7pNEnjQyZbsKk/fNEEfReCQJGAHmv6tEF9A5hOyXb + RtsT4x9IXfJCkiOAF+WTZM5HLnX5dd3fQ6tlI+rVk2phRUl35JS2tCgpUAO4VqIBA3RP4Ot1zf7u + ToIO6h79h63isY8PpikFouxlU9VHFP0NTRMeqctPhn2s2XV1duE6qg9QDg6LyrZnP7roqN6gDGuA + 9Riwq/ryIyIYjCR20X9VMJjlXHBRDLxBJqUs/F7os2s5h/MCQ0YUCi+pyiQfOhbW380p71MTGft4 + /drjyYjhlIkASJW5FkgQl5Wz1FvcAqhOkQQ29UP4DfkDw/tf9tEgLF0JdqkGZSQyDnLeGapeFboY + RiY5NUunXuD7IRm32x4dtgGVbKhnro9jR00Z7W3BdlXLKPxpi0GGsivFhLVCrrBMrsRAJ81qAAKO + WaD8Ik6jQE3FCKoH9OuxW622DHBCQ41orJKVFbsV6xbOvdUodjLFcm3QXrYDOXxUr5kChpBobWBJ + 6ibLx1g5t4khBRYW3gD9kU6wrSmwjnYBudQG+KgmCefcB4mrrA+Nf7r8DNl2AReGk6BB8J8LrI1a + Vs0OA1BhBY5cGdloVxaRoL39UIGVVmjsGzJlt5DIqhLTxdTtqsHEawc3VeHogO9jADJQGbtULXXV + GobM5hbCOAujvmlQgOnSMYIJsx9DeKMK1ahSYo7dNs2SRw71cGH8GpZ1AHdxPzqaQy/vdCIoRd2y + rBRwylgtDcQ750TxzaC5QDveBotHWgPln1VnT/iOCNkroAFi1FWFjxUyrak7CzytkegEwpLCotzd + WwCGYg/crOzyFvQkJFbrHSvRGgkvE40mPzHP6vuhV6OPCIYj747z4h6NogffyAu9MNgHX8ynQZrO + oocnm/XCyKrplwZ9qHYxYDv1wXgeTDHtI8KC6uo7e0o8xU75QNzWKNep5fWxV4MoISJVLroO1iRJ + 08P9fn/vd/cPm/swiPa/ueDnaTy6gKNQpl+sRabWkLO/92xVa+MZeHNpvc959snudttGrD2zqavt + Gp20WRZ5aBg1Gkb3Pqu72430QB1x7lWtzbBv+OTB2fM19NzkMcbOZkr6RZwnMx4FSZbkSvGi4GE6 + ywQeRESI04NDbJ8WdyDA8i90JRG0nI8uzEpwDMzVVM1iH0PFQRaGfuSHmQwgJyyS3I/R+DoV+JIA + cARxAh41S2Sc4nVyKJIk8oss9d2fSoGXKsLRhA+bonkXE42jIEiCII2SwJ9J7BB4lHZjxTzAWLEM + g1Dls0j4+TSOo7Rw+ghfRkEhMznz5TTIMjiS5P50KkUe5aEUiRJ+NJU8VjyH2DCbxVHCUxHHRSoV + 7Ivz+gzyYp4WXEg+83GCL7MwzYTyw4SHkeIKY2Ski3PVIEziiOzSZgNAc2G8WAFaVEOkDw8XbUOL + vrK2NvPJBO/mV202ltVm8rb8kyByskN14TUt4mqSoRF5soGzq2YCdzHjvc+PS2XxOxBgL2NM9BBk + xXLffX2rm9YsLFWHA3Hb7XbsnrgHTuyy1RPC98kPWJ5bu4B/Io3cKqgRIILxy9YdlMB7/DEiBp+Z + nwZJzMPpPVnGnUh1aeL+p38BAAD//wMAfu5QSoYuAAA= + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sun, 04 Feb 2024 09:45:33 GMT + Server: + - nginx + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/fixtures/vcr_cassettes/urlscan.yaml b/tests/fixtures/vcr_cassettes/urlscan.yaml new file mode 100644 index 0000000..ddd34bf --- /dev/null +++ b/tests/fixtures/vcr_cassettes/urlscan.yaml @@ -0,0 +1,76 @@ +interactions: +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + host: + - urlscan.io + user-agent: + - python-httpx/0.26.0 + method: GET + uri: https://urlscan.io/api/v1/search/?q=task.url%3A%22http%3A%2F%2Fexample.com%22%20AND%20task.domain%3A%22example.com%22%20AND%20verdicts.malicious%3Atrue&size=1 + response: + body: + string: !!binary | + H4sIAAAAAAAAA6vmUlBQKkotLs0pKVayUoiO1QEJlOSXJOYAuQZQXn42kGNmBOZlJBbH5+YXpQJF + 0hJzilO5agHQgV8gRAAAAA== + headers: + Cache-Control: + - private, max-age=10 + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - 'default-src ''self'' data:; script-src ''self'' data: developers.google.com + www.google.com www.gstatic.com; style-src ''self'' fonts.googleapis.com www.google.com; + img-src * data:; font-src ''self'' fonts.gstatic.com; child-src ''self''; + frame-src https://www.google.com/recaptcha/; form-action ''self''; connect-src + ''self''; upgrade-insecure-requests; frame-ancestors ''none''' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 04 Feb 2024 10:03:02 GMT + ETag: + - W/"44-94b5SgxOGo7bqoy7V0TqAHYl4XI" + Referrer-Policy: + - unsafe-url + Server: + - nginx + Strict-Transport-Security: + - max-age=63072000; includeSubdomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - DENY + X-Proxy-Cache: + - MISS + X-Rate-Limit-Action: + - search + X-Rate-Limit-Limit: + - '50000' + X-Rate-Limit-Remaining: + - '49996' + X-Rate-Limit-Reset: + - '2024-02-04T11:00:00.000Z' + X-Rate-Limit-Reset-After: + - '3417' + X-Rate-Limit-Scope: + - team + X-Rate-Limit-Window: + - hour + X-Robots-Tag: + - all + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/tests/fixtures/vcr_cassettes/vt.yaml b/tests/fixtures/vcr_cassettes/vt.yaml deleted file mode 100644 index 564947b..0000000 --- a/tests/fixtures/vcr_cassettes/vt.yaml +++ /dev/null @@ -1,443 +0,0 @@ -interactions: - - request: - body: null - headers: - Accept-Encoding: - - gzip - User-Agent: - - unknown; vtpy 0.7.2; gzip - method: GET - uri: https://www.virustotal.com/api/v3/files/275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f - response: - body: - string: - "{\n \"data\": {\n \"attributes\": {\n \"type_description\": - \"Text\",\n \"tlsh\": \"T141A022003B0EEE2BA20B00200032E8B00808020E2CE00A3820A020B8C83308803EC228\",\n - \ \"trid\": [\n {\n \"file_type\": - \"EICAR antivirus test file\",\n \"probability\": 100.0\n - \ }\n ],\n \"antiy_info\": \"Trojan/Generic.ASBOL.2A\",\n - \ \"crowdsourced_yara_results\": [\n {\n \"description\": - \"Rule to detect the EICAR pattern\",\n \"source\": \"https://github.com/advanced-threat-research/Yara-Rules\",\n - \ \"author\": \"Marc Rivero | McAfee ATR Team\",\n \"ruleset_name\": - \"MALW_Eicar\",\n \"rule_name\": \"malw_eicar\",\n \"ruleset_id\": - \"0019ab4291\"\n },\n {\n \"description\": - \"Just an EICAR test file - this is boring but users asked for it\",\n \"source\": - \"https://github.com/Neo23x0/signature-base\",\n \"author\": - \"Florian Roth\",\n \"ruleset_name\": \"gen_suspicious_strings\",\n - \ \"rule_name\": \"SUSP_Just_EICAR\",\n \"ruleset_id\": - \"000ae70a1a\"\n }\n ],\n \"names\": - [\n \"eicar.com-27284\",\n \"eicar.com-32493\",\n - \ \"eicar.com-47683\",\n \"eicar.com-5597\",\n - \ \"eicar.com-23657\",\n \"eicar.com-31458\",\n - \ \"eicar.com-45199\",\n \"eicar1.com\",\n \"eicar.com-188637\",\n - \ \"eicar.com-17027\",\n \"eicar.com-28897\",\n - \ \"eicar.com-42683\",\n \"eicar.com-186284\",\n - \ \"eicar.com-13339\",\n \"eicar.com-26405\",\n - \ \"eicar.com-22086\",\n \"eicar.com-11617\",\n - \ \"eicar.com-9601\",\n \"eicar.com-15195\",\n - \ \"eicar.com-37607\",\n \"eicar.com-6452\",\n - \ \"eicar.com-4705\",\n \"eicar.com-21508\",\n - \ \"eicar.com-35115\",\n \"eicar.com-2213\",\n - \ \"eicar.com-861\",\n \"eicar.com-47856\",\n - \ \"eicar.com-32610\",\n \"eicar.com-29851\",\n - \ \"eicar.com-942\",\n \"eicar.com-16558\",\n - \ \"eicar.com-1533\",\n \"eicar.com-40143\",\n - \ \"eicar.com-31585\",\n \"eicar.com-27597\",\n - \ \"eicar.com-28096\",\n \"eicar.com-12402\",\n - \ \"eicar.com-30522\",\n \"eicar.com-25110\",\n - \ \"eicar.com-12124\",\n \"eicar.com-39474\",\n - \ \"eicar.com-22236\",\n \"eicar.com-87556\",\n - \ \"eicar.com-521\",\n \"eicar.com-33229\",\n - \ \"eicar.com-84808\",\n \"eicar.com-15499\",\n - \ \"eicar.com-25594\",\n \"eicar.com-15706\",\n - \ \"eicar.com-3326\",\n \"eicar.com-15874\",\n - \ \"eicar.com-12091\",\n \"eicar.com-30953\",\n - \ \"eicar.com-32110\",\n \"eicar.com-20030\",\n - \ \"eicar.com-80093\",\n \"eicar.com-24621\",\n - \ \"eicar.com-14817\",\n \"eicar.com-79355\",\n - \ \"eicar.com-28787\",\n \"eicar.com-29588\",\n - \ \"eicar.com-11674\",\n \"eicar.com-29945\",\n - \ \"eicar.com-6565\",\n \"eicar.com-15928\",\n - \ \"eicar.com-3869\",\n \"eicar.com-6872\",\n - \ \"eicar.com-89354\",\n \"eicar.com-6102\",\n - \ \"eicar.com-15163\",\n \"eicar.com-571\",\n - \ \"eicar.com-41412\",\n \"eicar.com-48986\",\n - \ \"eicar.com-8113\",\n \"eicar.com-23492\",\n - \ \"eicar.com-37784\",\n \"eicar.com-4593\",\n - \ \"eicar.com-77055\",\n \"eicar.com-12330\",\n - \ \"eicar.com-46601\",\n \"eicar.com-112838\",\n - \ \"eicar.com-51461\",\n \"eicar.com-21573\",\n - \ \"eicar.com-107519\",\n \"eicar.com-7963\",\n - \ \"eicar.com-69494\",\n \"eicar.com-218017\",\n - \ \"eicar.com-27317\",\n \"eicar.com-22158\",\n - \ \"eicar.com-59775\",\n \"eicar.com-96714\",\n - \ \"eicar.com-3248\",\n \"eicar.com-16560\",\n - \ \"eicar.com-61854\",\n \"eicar.com-40412\",\n - \ \"eicar.com-13710\",\n \"eicar.com-42560\",\n - \ \"eicar.com-26183\",\n \"eicar.com-19184\",\n - \ \"eicar.com-10911\"\n ],\n \"last_modification_date\": - 1628462426,\n \"type_tag\": \"text\",\n \"times_submitted\": - 861771,\n \"total_votes\": {\n \"harmless\": 2019,\n - \ \"malicious\": 359\n },\n \"size\": - 68,\n \"popular_threat_classification\": {\n \"suggested_threat_label\": - \"virus.eicar/test\",\n \"popular_threat_category\": [\n {\n - \ \"count\": 12,\n \"value\": - \"virus\"\n },\n {\n \"count\": - 2,\n \"value\": \"trojan\"\n }\n - \ ],\n \"popular_threat_name\": [\n {\n - \ \"count\": 54,\n \"value\": - \"eicar\"\n },\n {\n \"count\": - 45,\n \"value\": \"test\"\n },\n - \ {\n \"count\": 34,\n \"value\": - \"file\"\n }\n ]\n },\n \"last_submission_date\": - 1628462426,\n \"meaningful_name\": \"eicar.com-27284\",\n \"sandbox_verdicts\": - {\n \"Lastline\": {\n \"category\": \"malicious\",\n - \ \"sandbox_name\": \"Lastline\",\n \"malware_classification\": - [\n \"MALWARE\",\n \"TROJAN\"\n - \ ]\n },\n \"OS X Sandbox\": - {\n \"category\": \"malicious\",\n \"sandbox_name\": - \"OS X Sandbox\",\n \"malware_classification\": [\n \"EVADER\"\n - \ ]\n }\n },\n \"sha256\": - \"275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f\",\n \"type_extension\": - \"txt\",\n \"tags\": [\n \"text\",\n \"attachment\",\n - \ \"via-tor\"\n ],\n \"last_analysis_date\": - 1628462046,\n \"unique_sources\": 3557,\n \"first_submission_date\": - 1148301722,\n \"ssdeep\": \"3:a+JraNvsgzsVqSwHq9:tJuOgzsko\",\n - \ \"md5\": \"44d88612fea8a8f36de82e1278abb02f\",\n \"sha1\": - \"3395856ce81f2b7382dee72602f798b642f14140\",\n \"magic\": \"ASCII - text, with no line terminators\",\n \"last_analysis_stats\": {\n - \ \"harmless\": 0,\n \"type-unsupported\": 9,\n - \ \"suspicious\": 0,\n \"confirmed-timeout\": - 0,\n \"timeout\": 2,\n \"failure\": 0,\n \"malicious\": - 59,\n \"undetected\": 5\n },\n \"last_analysis_results\": - {\n \"Bkav\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Bkav\",\n \"engine_version\": - \"1.3.0.9899\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Lionic\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Lionic\",\n \"engine_version\": - \"4.2\",\n \"result\": \"Test.File.EICAR.y\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Elastic\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Elastic\",\n \"engine_version\": - \"4.0.27\",\n \"result\": \"eicar\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210805\"\n },\n - \ \"MicroWorld-eScan\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"MicroWorld-eScan\",\n - \ \"engine_version\": \"14.0.409.0\",\n \"result\": - \"EICAR-Test-File\",\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"CMC\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"CMC\",\n \"engine_version\": - \"2.10.2019.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210624\"\n },\n - \ \"CAT-QuickHeal\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"CAT-QuickHeal\",\n \"engine_version\": - \"14.00\",\n \"result\": \"EICAR.TestFile\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"McAfee\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"McAfee\",\n \"engine_version\": - \"6.0.6.653\",\n \"result\": \"EICAR test file\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Malwarebytes\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Malwarebytes\",\n \"engine_version\": - \"4.2.2.27\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Zillya\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Zillya\",\n \"engine_version\": - \"2.0.0.4424\",\n \"result\": \"EICAR.TestFile\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210806\"\n },\n - \ \"Paloalto\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Paloalto\",\n \"engine_version\": - \"1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Sangfor\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Sangfor\",\n \"engine_version\": - \"2.9.0.0\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210625\"\n },\n - \ \"K7AntiVirus\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"K7AntiVirus\",\n \"engine_version\": - \"11.202.37928\",\n \"result\": \"EICAR_Test_File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Alibaba\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Alibaba\",\n \"engine_version\": - \"0.3.0.5\",\n \"result\": \"Trojan:MacOS/eicar.com\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20190527\"\n },\n \"K7GW\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"K7GW\",\n \"engine_version\": - \"11.202.37928\",\n \"result\": \"EICAR_Test_File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"CrowdStrike\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"CrowdStrike\",\n - \ \"engine_version\": \"1.0\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210203\"\n },\n \"Baidu\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Baidu\",\n \"engine_version\": - \"1.0.0.2\",\n \"result\": \"Win32.Test.Eicar.a\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20190318\"\n },\n - \ \"Cyren\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Cyren\",\n \"engine_version\": - \"6.3.0.2\",\n \"result\": \"EICAR_Test_File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"SymantecMobileInsight\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"SymantecMobileInsight\",\n - \ \"engine_version\": \"2.0\",\n \"result\": - \"ALG:EICAR Test String\",\n \"method\": \"blacklist\",\n - \ \"engine_update\": \"20210126\"\n },\n - \ \"Symantec\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Symantec\",\n \"engine_version\": - \"1.15.0.0\",\n \"result\": \"EICAR Test String\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ESET-NOD32\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"ESET-NOD32\",\n \"engine_version\": - \"23761\",\n \"result\": \"Eicar test file\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"APEX\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"APEX\",\n \"engine_version\": - \"6.195\",\n \"result\": \"EICAR Anti-Virus Test File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210807\"\n },\n \"Avast\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Avast\",\n \"engine_version\": - \"21.1.5827.0\",\n \"result\": \"EICAR Test-NOT virus!!!\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"ClamAV\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"ClamAV\",\n \"engine_version\": - \"0.103.3.0\",\n \"result\": \"Win.Test.EICAR_HDB-1\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Kaspersky\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Kaspersky\",\n \"engine_version\": - \"21.0.1.45\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"BitDefender\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"BitDefender\",\n \"engine_version\": - \"7.2\",\n \"result\": \"EICAR-Test-File (not a virus)\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"NANO-Antivirus\": {\n - \ \"category\": \"malicious\",\n \"engine_name\": - \"NANO-Antivirus\",\n \"engine_version\": \"1.0.146.25311\",\n - \ \"result\": \"Marker.Dos.EICAR-Test-File.dyb\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"SUPERAntiSpyware\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"SUPERAntiSpyware\",\n - \ \"engine_version\": \"5.6.0.1032\",\n \"result\": - \"NotAThreat.EICAR[TestFile]\",\n \"method\": \"blacklist\",\n - \ \"engine_update\": \"20210807\"\n },\n - \ \"Rising\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Rising\",\n \"engine_version\": - \"25.0.0.26\",\n \"result\": \"EICAR-Test-File (CLASSIC)\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Ad-Aware\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Ad-Aware\",\n \"engine_version\": - \"3.0.21.179\",\n \"result\": \"EICAR-Test-File (not a - virus)\",\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Trustlook\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"Trustlook\",\n - \ \"engine_version\": \"1.0\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"TACHYON\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"TACHYON\",\n \"engine_version\": - \"2021-08-08.02\",\n \"result\": \"EICAR-Test-File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Sophos\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Sophos\",\n \"engine_version\": - \"1.3.0.0\",\n \"result\": \"EICAR-AV-Test\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Comodo\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Comodo\",\n \"engine_version\": - \"33784\",\n \"result\": \"Malware@#2975xfk8s2pq1\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"F-Secure\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"F-Secure\",\n \"engine_version\": - \"12.0.86.52\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"DrWeb\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"DrWeb\",\n \"engine_version\": - \"7.0.49.9080\",\n \"result\": \"EICAR Test File (NOT a - Virus!)\",\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"VIPRE\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"VIPRE\",\n \"engine_version\": - \"94606\",\n \"result\": \"EICAR (v)\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"TrendMicro\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"TrendMicro\",\n \"engine_version\": - \"11.0.0.1006\",\n \"result\": \"Eicar_test_file\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"McAfee-GW-Edition\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"McAfee-GW-Edition\",\n - \ \"engine_version\": \"v2019.1.2+3728\",\n \"result\": - \"EICAR test file\",\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Trapmine\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"Trapmine\",\n - \ \"engine_version\": \"3.5.0.1023\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20200727\"\n },\n \"FireEye\": {\n \"category\": - \"timeout\",\n \"engine_name\": \"FireEye\",\n \"engine_version\": - \"32.44.1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Emsisoft\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Emsisoft\",\n \"engine_version\": - \"2021.4.0.5819\",\n \"result\": \"EICAR-Test-File (not - a virus) (B)\",\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Ikarus\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Ikarus\",\n \"engine_version\": - \"0.1.5.2\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Avast-Mobile\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Avast-Mobile\",\n \"engine_version\": - \"210808-00\",\n \"result\": \"Eicar\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Jiangmin\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Jiangmin\",\n \"engine_version\": - \"16.0.100\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210807\"\n },\n - \ \"Webroot\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Webroot\",\n \"engine_version\": - \"1.0.0.403\",\n \"result\": \"W32.Eicar.Testvirus.Gen\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Avira\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Avira\",\n \"engine_version\": - \"8.3.3.12\",\n \"result\": \"Eicar-Test-Signature\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"eGambit\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"eGambit\",\n - \ \"engine_version\": null,\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Antiy-AVL\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Antiy-AVL\",\n \"engine_version\": - \"3.0.0.1\",\n \"result\": \"Trojan/Generic.ASMalwRG.118\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Kingsoft\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Kingsoft\",\n \"engine_version\": - \"2017.9.26.565\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Microsoft\": {\n \"category\": \"timeout\",\n - \ \"engine_name\": \"Microsoft\",\n \"engine_version\": - \"1.1.18400.4\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Gridinsoft\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Gridinsoft\",\n \"engine_version\": - \"1.0.51.144\",\n \"result\": \"PUP.U.EICAR_Test_File.dd\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Arcabit\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Arcabit\",\n \"engine_version\": - \"1.0.0.886\",\n \"result\": \"EICAR-Test-File (not a virus)\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"ViRobot\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"ViRobot\",\n \"engine_version\": - \"2014.3.20.0\",\n \"result\": \"EICAR-test\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ZoneAlarm\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"ZoneAlarm\",\n \"engine_version\": - \"1.0\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"GData\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"GData\",\n \"engine_version\": - \"A:25.30525B:27.24020\",\n \"result\": \"EICAR_TEST_FILE\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Cynet\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Cynet\",\n \"engine_version\": - \"4.0.0.27\",\n \"result\": \"Malicious (score: 99)\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"BitDefenderFalx\": {\n - \ \"category\": \"type-unsupported\",\n \"engine_name\": - \"BitDefenderFalx\",\n \"engine_version\": \"2.0.936\",\n - \ \"result\": null,\n \"method\": \"blacklist\",\n - \ \"engine_update\": \"20210610\"\n },\n - \ \"AhnLab-V3\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"AhnLab-V3\",\n \"engine_version\": - \"3.20.4.10148\",\n \"result\": \"Virus/EICAR_Test_File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Acronis\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"Acronis\",\n - \ \"engine_version\": \"1.1.1.82\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210512\"\n },\n \"BitDefenderTheta\": {\n - \ \"category\": \"malicious\",\n \"engine_name\": - \"BitDefenderTheta\",\n \"engine_version\": \"7.2.37796.0\",\n - \ \"result\": \"EICAR-Test-File (not a virus)\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210803\"\n },\n - \ \"ALYac\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"ALYac\",\n \"engine_version\": - \"1.1.3.1\",\n \"result\": \"Misc.Eicar-Test-File\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"MAX\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"MAX\",\n \"engine_version\": - \"2019.9.16.1\",\n \"result\": \"malware (ai score=100)\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"VBA32\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"VBA32\",\n \"engine_version\": - \"5.0.0\",\n \"result\": \"EICAR-Test-File\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210806\"\n },\n - \ \"Cylance\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Cylance\",\n \"engine_version\": - \"2.3.1.101\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Zoner\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Zoner\",\n \"engine_version\": - \"0.0.0.0\",\n \"result\": \"EICAR.Test.File-NoVirus.250\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"TrendMicro-HouseCall\": - {\n \"category\": \"malicious\",\n \"engine_name\": - \"TrendMicro-HouseCall\",\n \"engine_version\": \"10.0.0.1040\",\n - \ \"result\": \"Eicar_test_file\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Tencent\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Tencent\",\n \"engine_version\": - \"1.0.0.1\",\n \"result\": \"EICAR.TEST.NOT-A-VIRUS\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Yandex\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Yandex\",\n \"engine_version\": - \"5.5.2.24\",\n \"result\": \"EICAR_test_file\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"SentinelOne\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"SentinelOne\",\n \"engine_version\": - \"6.1.0.4\",\n \"result\": \"Static AI - Malicious COM\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210805\"\n },\n \"MaxSecure\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"MaxSecure\",\n \"engine_version\": - \"1.0.0.1\",\n \"result\": \"VIRUS.EICAR.TEST\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210807\"\n },\n - \ \"Fortinet\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Fortinet\",\n \"engine_version\": - \"6.2.142.0\",\n \"result\": \"EICAR_TEST_FILE\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"AVG\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"AVG\",\n \"engine_version\": - \"21.1.5827.0\",\n \"result\": \"EICAR Test-NOT virus!!!\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Cybereason\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"Cybereason\",\n - \ \"engine_version\": \"1.2.449\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210330\"\n },\n \"Panda\": {\n \"category\": - \"malicious\",\n \"engine_name\": \"Panda\",\n \"engine_version\": - \"4.6.4.2\",\n \"result\": \"EICAR-AV-TEST-FILE\",\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Qihoo-360\": {\n \"category\": \"malicious\",\n - \ \"engine_name\": \"Qihoo-360\",\n \"engine_version\": - \"1.0.0.1300\",\n \"result\": \"qex.eicar.gen.gen\",\n - \ \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n }\n },\n \"reputation\": - 3457,\n \"first_seen_itw_date\": 1276250738\n },\n \"type\": - \"file\",\n \"id\": \"275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f\",\n - \ \"links\": {\n \"self\": \"https://www.virustotal.com/api/v3/files/275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f\"\n - \ }\n }\n}" - headers: - Cache-Control: - - no-cache - Content-Encoding: - - gzip - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 08 Aug 2021 22:40:51 GMT - Server: - - Google Frontend - Vary: - - Accept-Encoding - X-Cloud-Trace-Context: - - 12dd672d524a26dc459e994cb10a47d6 - status: - code: 200 - message: OK - url: https://www.virustotal.com/api/v3/files/275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f -version: 1 diff --git a/tests/fixtures/vcr_cassettes/vt_non_malicious.yaml b/tests/fixtures/vcr_cassettes/vt_non_malicious.yaml deleted file mode 100644 index 877b763..0000000 --- a/tests/fixtures/vcr_cassettes/vt_non_malicious.yaml +++ /dev/null @@ -1,625 +0,0 @@ -interactions: - - request: - body: null - headers: - Accept-Encoding: - - gzip - User-Agent: - - unknown; vtpy 0.7.2; gzip - method: GET - uri: https://www.virustotal.com/api/v3/files/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 - response: - body: - string: - "{\n \"data\": {\n \"attributes\": {\n \"type_description\": - \"unknown\",\n \"tlsh\": \"TNULL\",\n \"nsrl_info\": - {\n \"products\": [\n \"DRAW (Corel Corporation)\",\n - \ \"Photo-Paint (Corel Corporation)\",\n \"Commerce - Server Developer Edition (Microsoft)\",\n \"Exchange Server - Enterprise Edition (Microsoft)\",\n \"eMbedded Visual Tools - (Microsoft)\",\n \"Internet Security and Acceleration Server - - Enterprise Edition (Microsoft)\",\n \"Commerce Server - - Developer Edition (Microsoft)\",\n \"Linux (Corel Corporation)\",\n - \ \"Yourideallink.com (Ideal link Inc.)\",\n \"NSRL - Test (NIST)\",\n \"Visio (Microsoft)\",\n \"Visio - Enterprise Edition (Microsoft)\",\n \"EarthLink (Earthlink - Inc.)\",\n \"Riven (Red Orb)\",\n \"Quicken - (Intuit Inc.)\",\n \"Get Set to Learn (Creative Wonders)\",\n - \ \"MySQL (NuSphere Corporation)\",\n \"Windows - (Microsoft)\",\n \"QuickBooks (Intuit Inc.)\",\n \"Tivoli - Manager (Tivoli)\"\n ],\n \"filenames\": [\n - \ \"1, Augustin, Butterfield, Cook, Copperplate Gothic (1, - Copperplate Gothic (8, Drummer, Erickson, Eurostile (1, Eurostile 2 (3, FJSV, - FMI, Flynn, Gorman, Holmes, Ivey, Jirik, Koval, Lovitz, MAHJONGG.{EASY, Met - Turn, Midstokke, NATE, Nipstad, Oak, Papenfuss, Quigley, Rada, Ross, SUNW, - Schue, Sorry, TI, Thuen, Uglem, Univers (1-5, Univers Condensed (2, Vorhees, - Wicker, Xanadu, Yaeger, Zimmerman, btmgr.spec, nasm.vim, sunw\",\n \"iesetup.dir\",\n - \ \"BLANK.TXT, blogo.gi!, blogo.gi_\",\n \"ROUTE.TBL\",\n - \ \"BLANK DOCUMENT.PSW, BLANK NOTE.PWI, CD1.INF, FILEOSP.RC, - chat.adm\",\n \"cdrom_sp.tst\",\n \".FVWM95, - .FVWM95RC, .TEXTSWRC, .TEXT_EXTRAS_MENU, .TTYSWRC, ADDGROUP, ANSI, AWK, AWK.1, - CAPTOINFO, CBB-MAN, COMPILED, CONFIG, DIGITAL, DUMB, DYNALOADER, EDITOR, EDITOR.1, - FDLIST, FDMOUNT.CONF, FDMOUNTD, FDUMOUNT, FUJITSU, GENKSYMS, INFOTOCAP, INIT-RESTART.HOOK, - INIT.HOOK, IO, IO.BS, LASTB, LD-LINUX.000, LD-LINUX.SO, LIBAPT-PKG.001, LIBAPT-PKG.SO, - LIBATTRGLYPH.001, LIBATTRGLYPH.SO, LIBATTRIBUTE.001, LIBATTRIBUTE.SO, LIBBROKENLOCALE.SO, - LIBC.SO, LIBCOMGLYPH.001, LIBCOMGLYPH.SO, LIBCOMTERP.001, LIBCOMTERP.SO, LIBCOMUNIDRAW.001, - LIBCOMUNIDRAW.SO, LIBCOMUTIL.001, LIBCOMUTIL.SO, LIBCOM_ERR.000, LIBCRYPT.SO, - LIBDB.SO, LIBDL.000, LIBDL.SO, LIBDND++.SO, LIBDND.SO, LIBDPKG.000, LIBDPKG.001, - LIBDRAWSERV.001, LIBDRAWSERV.SO, LIBE2P.000, LIBEXT2FS.000, LIBFORM.000, LIBFRAMEUNIDRAW.001, - LIBFRAMEUNIDRAW.SO, LIBGDBM.000, LIBGDBM.001, LIBGIF.000, LIBGIF.SO, LIBGRAPHUNIDRAW.001, - LIBGRAPHUNIDRAW.SO, LIBHISTORY.000, LIBICE.001, LIBICE.SO, LIBIV-COMMON.001, - LIBIV-COMMON.SO, LIBIV.001, LIBIV.SO, LIBIVGLYPH.001, LIBIVGLYPH.SO, LIBJPEG.000, - LIBJPEG.SO, LIBM.SO, LIBMAGICK.SO, LIBMENU.000, LIBMRM.001, LIBMRM.SO, LIBNSL.SO, - LIBNSS_COMPAT.SO, LIBNSS_DB.SO, LIBNSS_DNS.SO, LIBNSS_FILES.SO, LIBNSS_NIS.SO, - LIBOLGX.SO, LIBOVERLAYUNIDRAW.001, LIBOVERLAYUNIDRAW.SO, LIBPANEL.000, LIBPEX5.001, - LIBPEX5.SO, LIBPTHREAD.SO, LIBQT.001, LIBQT.SO, LIBRESOLV.SO, LIBSLANG.000, - LIBSM.001, LIBSM.SO, LIBSS.000, LIBSTDC++-LIBC6.0-1, LIBSTDC++-LIBC6.1-1, - LIBSTDC++.001, LIBSTDC++.SO, LIBTIFF.SO, LIBTIME.001, LIBTIME.SO, LIBTOPOFACE.001, - LIBTOPOFACE.SO, LIBUNGIF.SO, LIBUNIDRAW-COMMON.001, LIBUNIDRAW-COMMON.SO, - LIBUNIDRAW.001, LIBUNIDRAW.SO, LIBUNIIDRAW.001, LIBUNIIDRAW.SO, LIBUTIL.SO, - LIBUUID.000, LIBWRASTER.SO, LIBWXGRID_XT.SO, LIBWXTAB_XT.SO, LIBWX_XT.SO, - LIBWX_XTTHREAD.SO, LIBWX_XTWIDGETS.SO, LIBX11.001, LIBX11.SO, LIBXAW.001, - LIBXAW.SO, LIBXAW3D.001, LIBXAW3D.SO, LIBXEXT.001, LIBXEXT.SO, LIBXI.001, - LIBXI.SO, LIBXIE.001, LIBXIE.SO, LIBXM.001, LIBXM.SO, LIBXMU.001, LIBXMU.SO, - LIBXP.001, LIBXP.SO, LIBXPM.000, LIBXPM.SO, LIBXT.001, LIBXT.SO, LIBXTST.001, - LIBXTST.SO, LIBXVIEW.SO, LIBZ.001, LIBZ.SO, LOCALE.ALIAS, MACINTOSH, MAIN-MENU-PRE.HOOK, - MAIN-MENU.HOOK, MENUDEFS.HOOK, NAWK, NAWK.1, NEC, NEWXSERVER.XSERVER-VGA16, - PAGER, PIDOF, POST.HOOK, POWEROFF, RAMSIZE, RBASH, RCLOCK, REBOOT, RESET, - RMMOD, ROOTFLAGS, RXVT, RXVT-M, SCREEN, SCREEN-W, SECURITYPOLICY, SG, SGI, - SHELLTOOL, SOCKET, SOCKET.BS, SONY, SUN, SWAPDEV, SWAPOFF, TABSET, TELINIT, - TERMINFO, VI.1, VIDMODE, VIGR, VT100, VT102, VT220, VT52, W.1, X11R6, XDFFORMAT, - XDM-CONFIG, XDVI, XF86CONFIG, XFTP, XINITRC, XKBCOMP, XSCREENSAVER, XSERVERRC, - XSETBG, XSYSINFO, XTERM, XTERM-DEBIAN, XTERM-XFREE86\",\n \"rfc779.htm\",\n - \ \"test1.txt, test1.z\",\n \"INSTALL.LOG\",\n - \ \"Drafts, Inbox, Sent, Templates, Trash, Unsent_Messages, - blogo.gi!, blogo.gi_, ns45_drafts, ns45_inbox, ns45_sent, ns45_templates, - ns45_trash, ns45_unsent_messages, phonepref.txt\",\n \"MSDN332.INF\",\n - \ \"PREFREPT.BMP, PREFRPT2.BMP, PREFSMOD.BMP, PREFSWIN.BMP, - PROGGRP1.BMP, PROGGRP2.BMP, PROGRUN.BMP, QCARD01.BMP, QCARD06.BMP, UGCHAP9.BMP\",\n - \ \"BD.CON, BF.CON, BG.CON, BL.CON, BN.CON, BNCON.WRI, CC.CON, - CD.CON, DISK1, DISK2, DISK3, WOW.DRV\",\n \".exists, API.bs, - B.bs, Base64.bs, ByteLoader.bs, ChangeNotify.bs, Clipboard.bs, Console.bs, - DBI.bs, DB_File.bs, DProf.bs, Dumper.bs, Embperl.bs, Event.bs, EventLog.bs, - Fcntl.bs, FileSecurity.bs, GDBM_File.bs, Glob.bs, Hostname.bs, IO.bs, IPC.bs, - Internet.bs, Leak.bs, MD2.bs, MD5.bs, Mutex.bs, NDBM_File.bs, Net.bs, NetAdmin.bs, - NetResource.bs, ODBC.bs, ODBM_File.bs, OLE.bs, Opcode.bs, Oracle.bs, POSIX.bs, - Peek.bs, PerfLib.bs, Pipe.bs, Process.bs, Registry.bs, SDBM_File.bs, SHA1.bs, - Semaphore.bs, Service.bs, Shortcut.bs, Socket.bs, Sound.bs, Storable.bs, Symbol.bs, - SysV.bs, Syslog.bs, Thread.bs, Win32.bs, WinError.bs, attrs.bs, carts.MYD, - columns_priv.MYD, comments, host.MYD, images.MYD, mail, mrbs_entry.MYD, mrbs_repeat.MYD, - mysql.bs, nomail, sessions.MYD, tables_priv.MYD, users.MYD, zlib.bs\",\n \"empty.htm, - logagent.exe, quartz.dll, tvxdup.001, vnetsup.vxd, xeno.avb\",\n \"blogo.gi!, - blogo.gi_\",\n \"MessagesD.properties, MessagesF.properties, - MessagesJA.properties, access_log\",\n \"CUSTOMERSERVICE.RESX, - CUSTOMERSERVICES.CUSTOMERSERVICE.RESOURCES, DEFAULT.ASPX.RESX, EXCEPTIONHANDLING.EXCEPTIONHANDLINGFORM.RESOURCES, - EXCEPTIONHANDLINGFORM.RESX, FRMPOORUPGRADE.RESX, GLOBAL.ASAX.RESX, LOGIN.ASPX.RESX, - MAINFORM.RESX, MOBILEWEBFORM1.ASPX.RESX, README.ASPX.RESX, SERVICE.LCK, SERVICE1.ASMX.RESX, - VB6POOREXAMPLE.FRMPOORUPGRADE.RESOURCES, WEBAPPLICATION3.GLOBAL.RESOURCES, - WEBAPPLICATION3.WEBFORM1.RESOURCES, _11EVENTLOGGINGDEMO.README.RESOURCES, - _MYHEADER.ASCX.RESX\",\n \"DECSCSI, DISK1, DISK103, PLANGEOAREA.BCP, - SPCDROM.40, TAGFILE.1\"\n ]\n },\n \"names\": - [\n \"dmlconf.dat\",\n \"WUGkjpjuxkULbv.dex.flock - (deleted)\",\n \"593d33644823373f0bfa00ae3cc7330e.dex.flock - (deleted)\",\n \"output.16409235.txt\",\n \"torrc\",\n - \ \"StartupFaster.ini\",\n \"cmonitor.dll\",\n - \ \"cals-150213151ses.dex.flock (deleted)\",\n \"d.dex.flock - (deleted)\",\n \"audience_network.dex.flock (deleted)\",\n - \ \"1628460935099s.dex.flock (deleted)\",\n \"eicar.com-19653\",\n - \ \"Python27 .exe\",\n \"output.16421589.txt\",\n - \ \"anYCnqqS.dex.flock (deleted)\",\n \"Dokumente.mydocs\",\n - \ \"soft.lnk\",\n \"present.txt\",\n \"com.google.android.gms.appid-no-backup\",\n - \ \"testdisk-7.0.win.zip\",\n \"dex.dex.flock - (deleted)\",\n \"svchost\",\n \"output.16428080.txt\",\n - \ \"apkprotect-v1.dex.flock (deleted)\",\n \"mry7x1kr\",\n - \ \"honz3tmxf\",\n \"9wxvaxc5\",\n \"lxfnqxfrnsezce\",\n - \ \"iztm3cub0b4lg3yq\",\n \"4qan0kxd\",\n \"lsx1497ktenl\",\n - \ \"o1m9q6dh\",\n \"m35tzztrsk9slja\",\n \"0sglucg9x\",\n - \ \"rmpont09br3\",\n \"8tlqysqchgstz\",\n \"f0vf18kw7ynlxe3y\",\n - \ \"cvux8xfrqhuwrl\",\n \"dbwuupnc1fyiw5hf\",\n - \ \"1w3yny72j2fre\",\n \"dtpfdk6fl3wdbnf\",\n - \ \"zUlXgEzJfLyBmDrXaJp\",\n \"an35uxlwzqp9znh0\",\n - \ \"4pm7jcnqxof\",\n \"gytczzifc9qexoc\",\n \"sgtnj3j6x09v\",\n - \ \"7fxyc4k7jede\",\n \"dnuninst.exe\",\n \"tSfFxMoThOoT\",\n - \ \"eicar.com-13030\",\n \"qpath.ini\",\n \"Sysqemsjdro.exe\",\n - \ \"state.rsm\",\n \"nn.dex.flock (deleted)\",\n - \ \"eicar.com-9786\",\n \"HPJumpStartBridge.exe\",\n - \ \"autorun.inf\",\n \"credentials.txt\",\n \"Registry.pol\",\n - \ \"ntkrnlmp.exe\",\n \"abort_hook.health\",\n - \ \"cals-259835210ses.dex.flock (deleted)\",\n \"eicar.com-6481\",\n - \ \"Documents.mydocs\",\n \"abc.jpg\",\n \"uBPYgN.dll\",\n - \ \"acinggameksjdshd\",\n \".nomedia\",\n \"eicar.com-3098\",\n - \ \"CTS.exe\",\n \"21.129.0627.0002\",\n \"MlvxvZlTareCEsr.dex.flock - (deleted)\",\n \"EPJvHRoZFJewvL.dex.flock (deleted)\",\n \"eicar.com-32213\",\n - \ \"rufus-3.15(1).exe\",\n \"manager.php\",\n - \ \"ueiMMwpt.dex.flock (deleted)\",\n \"fj4ghga23_fsa.txt\",\n - \ \"Start\",\n \"eicar.com-28908\",\n \"h7osut4d\",\n - \ \"u6yf9edg\",\n \"jzguxmrpmy\",\n \"7xuu0o9man69ip\",\n - \ \"iy890gq0oyys\"\n ],\n \"last_modification_date\": - 1628462399,\n \"times_submitted\": 28772,\n \"size\": - 0,\n \"total_votes\": {\n \"harmless\": 7776,\n - \ \"malicious\": 1887\n },\n \"last_submission_date\": - 1628462386,\n \"known_distributors\": {\n \"filenames\": - [\n \"pp.dll-5c8ccd7a7c4e045b186a1d13aa6a89747b7f4956289dc0ddd3e26afb493e63fe\",\n - \ \"ScheduledTask_DialogTrigger.xml\",\n \"stub32i.exe - \ -0bf48b2b5cf2a88226358d9bf7b3d00333559306760bd555058b0168d8bc89c0\",\n - \ \"qdds.dll-7f542b5d1ce1a5c5c75a016e031c4c245ef43c75a2be5fc26d96f88db77b84e4\",\n - \ \"{F3225FA7-7989-4EF5-8E7B-81952DCC0E9F}\",\n \"hpsoftpaqwrapper.exe-fddab04ea9cb4e08278c3e004dd3712527eccc9d1763ec95a216b1225b136780\",\n - \ \"666db2d1546c3ac1bb524a058e5350fc8c197cb167104e4ff1b13e572deff0bd\",\n - \ \"SurfaceVirtualFunctionEnum.sys.lastcodeanalysissucceeded\",\n - \ \"vietool.exe-e3444c771cafe93bb2666990108bd216212c2bb4c8b923a5dd95158b00700af8\",\n - \ \"cteng_index.lck\",\n \"dwrite.dll-75af0e2bf98ebcc64f8d9cacde2c226f5a5784406523a5d3241495f7e007bc83\",\n - \ \"ReplaceSupportFilesNamesInISM.txt\",\n \"nestedMasterDetail.theme.css\",\n - \ \"d3e1c832-3bca-401d-b11e-bf9417df3343-e1355518f0f7752f725b021fa08e3ad84956e1b1866e0616b77531b39cdf0d83\",\n - \ \"66aa9e8b3a049566950d5154f4b7b25e3e2f7043bfccd39a0f35be1073f32a12\",\n - \ \"9f1d612c-077f-44e1-bb4c-95e4380fbb29\",\n \"install.exe-ad18fcd479609dfa41d79225fb31021054b34e9961d23017bf142ee9af74a120\",\n - \ \"9fedb224-bf78-4a38-b7a7-87d64ff2c091\",\n \"Colorado - Player Location Check.exe\",\n \"stub32i.exe -03a6f6c6a74c039afa1763acd47694a708bb6ac5978fb490f9a7b4f30a3b6a1e\",\n - \ \"2303a6a2-836a-4843-a1ee-eff233a88b40-33b0d7dcbea7d88b5ac9654b00fd61d964b0e50edcfaad26012b6ed71518bf08\",\n - \ \"9f438706-d713-4351-8ee7-6a461ca86e81\",\n \"4bf4fdf43925f6152769817b98fdafd621af2b615e85df6b0a54bf8b70306a1a\",\n - \ \"a230fd30914a79613f0d1c4a167e6fba362b8c641433266e7522267c64de46d5\",\n - \ \"receive_data.txt\",\n \"GATDirectConnect.dll.lastcodeanalysissucceeded\",\n - \ \"9fd1ca0b-3a99-4773-917b-63439a2951f6\",\n \"9f5c2cce-13d6-4b70-814d-ec7e1106be04\",\n - \ \"50b0a47de5aa86a7926ea2b21403833a323840b4a3829f045a43fd8bb3545290\",\n - \ \"7f2c72b0b972e850cf67a82d402e31ea2a5653a9fa68cd9b65047b4ad11a54f7\",\n - \ \"9f13b694-0b01-4d0b-aaf8-d07c62912d3c\",\n \"4ce11e99ef4fe3b259e0bc1ae9ecc82ec724e537be7eab0716dc6ef5e8f47250\",\n - \ \"tsworkspace.dll-3d2a89140b020219ffad00d285d4320703eebfda652e491b3cbd765d021cec3a\",\n - \ \"bulkusb.bmf\",\n \"60e1b14f486c394b03a5c0f8c3144e33b5fd076d7f287ff8cf10d5fed0226bf4\",\n - \ \"9fff6daf-eafb-4e35-91d1-39205be4e6e0\",\n \"810c22cc2e7dd1f521e380d3e62a9e992caeea9a1081a2c1abf7ffc41b45e962\",\n - \ \"\\u0001\",\n \"9f33f224-e31e-4bfc-98af-a9dd19fb46e2\",\n - \ \"Siemens.Siport.Common.dll.lastcodeanalysissucceeded\",\n - \ \"ReplaceAgentExeName.txt\",\n \"tuMCFPlg.dll-6e352d0de058882a7db46b9b791d409b81117a0dd3188fba4d8688631123c8d7\",\n - \ \"hpsoftpaqwrapper.exe-fd36ad6058183d16ec9cc97488498b44b69ade376c2ae6523c7150be40d49222\",\n - \ \"5d53eac41c5c63b5af3c81e8de56687f0bf4b38e942837a7b60030dd37963742\",\n - \ \"SurfaceServiceSensors.dll.lastcodeanalysissucceeded\",\n - \ \"485de75f40ece6b228d12240549483d3550a6a6d94525377bfd42ac8b4fb6382\",\n - \ \"prcs32.exe-70e075cb1ec4bdf0a952c2d3d292ecaaf1cbfddb0f5bdf30c3479521554d8a71\",\n - \ \"main.css\",\n \"tsworkspace.dll-7f970189d7fbe228901b089162ce3232656625428334fdd185d0a139b1611259\",\n - \ \"stub32i.exe -e22c0b45c667988c8c773af0262e3b60e92976143a60368ff997efade60d04ab\",\n - \ \"si8_fifo.c\",\n \"HP_SLICE_FWU.inf.cab\",\n - \ \"USB_Write_Enforcer.ps1\",\n \"Dummy.txt\",\n - \ \"hpsoftpaqwrapper.exe-ffdf807f29ec54abb4675cad7aa6bcb0439cf63d6150d9aa7a55e0f0c2bc7981\",\n - \ \"rotatelogs.exe-7cd4ec84b2ee92e338d4b290d9128c80caae117374c7c440bfbe47a83c741943\",\n - \ \"SurfaceService.exe.lastcodeanalysissucceeded\",\n \"DSE_dialog.ps1\",\n - \ \"RDID1110.WPRP\",\n \"hpsoftpaqwrapper.exe-fc5f37367f40d885c05ddb743db663b52050a0e5013b7e0d88c73f6bff06d451\",\n - \ \"GoogleUpdateSetup.exe-93c6cc759757abd4af17a57b59812e1be23d47cc750464ba152c4a2a040ef9f1\",\n - \ \"tseRes.dll-dca5a52d4be382e259aecb66110441fbc133a30978008a9c5a15c60ea5d90290\",\n - \ \"9f9f5312-8d34-4b13-84b9-f488e06712ae\",\n \"SurfSvcRpcClientDll.dll.lastcodeanalysissucceeded\",\n - \ \"RunPowershell.vbs\",\n \"mpavdlta.vdm-a4e04bc7fc8e64a9b40858586f9cb9b75a73916a40bc27c8da6be68e5ea1a8c7\",\n - \ \"rtc.dll-289befbde6186db2a7e220f580fbe53afe6727f6a412a8786ae59de9c1bf2794\",\n - \ \"Siemens.Siport.Common.TestHelper.dll.lastcodeanalysissucceeded\",\n - \ \"suf80_launch.exe-cfc507fafd9e1dc327768da8a816607d9021ea65d823c1bd668c042cb6222b6a\",\n - \ \"uiAlert.dll-8cc4847890e998695defdd598ab0f2c33704834d10044c2b918db16c134a749b\",\n - \ \"SurfaceBtleLcPenTelemetry.dll.lastcodeanalysissucceeded\",\n - \ \"SurfaceUcmUcsiHidClient.sys.lastcodeanalysissucceeded\",\n - \ \"6e7b8c38f6514c586a92cfa3a9149f522007a3bca0c8bc711b972678c1400088\",\n - \ \"7c0f98e68c7c436ceceaa421dd09e39a7af8020d9f713ca7669aab1ec734fcbc\",\n - \ \"policy.8.1.Altiris.Common.UI.dll-c95a065b9115ec14774c80373fec4b9b9e324c8c52fea1e3954f1b772b74ef67\",\n - \ \"SurfaceTconDriver.sys.lastcodeanalysissucceeded\",\n - \ \"dcd1fb77-fb9d-438e-ad2a-268570a54b7f-78f6727431f9065eb3dc3297ffd5cf2822d8866e51a8915a3853633bcea32e8f\",\n - \ \"9fdd526e-5129-4753-afa6-ef92c9b9749f\",\n \"rgsender_gui.exe-21f89e4a4204d7a91b2c87dddd3188e64f334eb8231a9846748ec0f6d962546e\",\n - \ \"Siemens.Siport.LockerService.exe.lastcodeanalysissucceeded\",\n - \ \"SurfaceThermalPolicy.sys.lastcodeanalysissucceeded\",\n - \ \"ScheduledTask_DSE_policy_check.xml\"\n ],\n - \ \"products\": [\n \"Sound Pool 18 DVD Collection\",\n - \ \"centos-cloud-centos-7-v20210721\",\n \"8A4F7B4302826FD8876897FAA4AF9B5A12544CD9\",\n - \ \"rhel-cloud-rhel-7-v20210721\",\n \"E9D8B00EB09652557A4C3E46AEC459D06EA5471C\",\n - \ \"centos-cloud-centos-stream-8-v20210721\",\n \"B736D6800E2687DCD3639757977B6AFB766807B7\",\n - \ \"1418196ADC9C9C13FFB8BA01E3B281D2B0CB5EFC\",\n \"CDB02F81E9388E33590F8DF10439981092641892\",\n - \ \"cos-cloud-cos-89-16108-470-11\",\n \"6C7543515299BC5053894D03E410F338C8C5BDD3\",\n - \ \"debian-cloud-debian-9-stretch-v20210721\",\n \"071-71342-11.5-20G71\",\n - \ \"BC7A78EC9B2BA33E20C665E4F2880BB69FC5E1EE\",\n \"rhel-cloud-rhel-8-v20210721\",\n - \ \"071-72781-11.5.1-20G80\",\n \"FBEB73BAECCE163B0F1C3F2DF5FE469B9E6D4A5E\",\n - \ \"cos-cloud-cos-81-12871-1290-20\",\n \"C5D4DB32129330441474A50B3C63B17E87AA3795\",\n - \ \"cos-cloud-cos-81-12871-1317-1\",\n \"071-14766-11.2.3-20D91\",\n - \ \"suse-cloud-sles-15-sp3-v20210727\",\n \"debian-cloud-debian-10-buster-v20210721\",\n - \ \"46D0156AC25B61F75457FFDA0F0CEC78120B1EB7\",\n \"275AA4413943A4AFED9A918DD72C43D704A1D586\",\n - \ \"E2C68590CD5588BE630D659088937C6323F1F681\",\n \"53060D393277BDA58DCBB45F66BF6B4319F9EADE\",\n - \ \"cos-cloud-cos-85-13310-1308-6\",\n \"593C226110CBA10F8874715AB06BADABED435061\",\n - \ \"907F228EB56EB859D5D57196739F3E5AC543E4B1\",\n \"D9F0C33D583D2DF53F31699F8018E26C41517861\"\n - \ ],\n \"distributors\": [\n \"Microsoft\",\n - \ \"MAGIX Computer Products\",\n \"Oracle\",\n - \ \"F. Hoffmann-La Roche Ltd.\",\n \"G - DATA\",\n \"Google\",\n \"IAR Systems - AB\",\n \"Siemens\",\n \"Geocomply\",\n - \ \"HP\",\n \"Symantec\",\n \"ObserveIT\",\n - \ \"Electric Quilt\"\n ],\n \"data_sources\": - [\n \"HashDB\",\n \"National Software - Reference Library (NSRL)\",\n \"monitor_oracle\",\n \"monitor_roche\",\n - \ \"monitor_gdata\",\n \"monitor_google\",\n - \ \"monitor_iar\",\n \"monitor_siemens\",\n - \ \"monitor_geocomply\",\n \"monitor_hp\",\n - \ \"monitor_microsoft_updates\",\n \"monitor_symantec\",\n - \ \"monitor_observeit\",\n \"monitor_microsoft\",\n - \ \"monitor_electricquilt\"\n ]\n },\n - \ \"last_analysis_results\": {\n \"Bkav\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Bkav\",\n \"engine_version\": - \"1.3.0.9899\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Lionic\": {\n \"category\": \"timeout\",\n - \ \"engine_name\": \"Lionic\",\n \"engine_version\": - \"4.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Elastic\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Elastic\",\n \"engine_version\": - \"4.0.27\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210805\"\n },\n - \ \"DrWeb\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"DrWeb\",\n \"engine_version\": - \"7.0.49.9080\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"MicroWorld-eScan\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"MicroWorld-eScan\",\n - \ \"engine_version\": \"14.0.409.0\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"FireEye\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"FireEye\",\n \"engine_version\": - \"32.44.1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"CAT-QuickHeal\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"CAT-QuickHeal\",\n \"engine_version\": - \"14.00\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"McAfee\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"McAfee\",\n \"engine_version\": - \"6.0.6.653\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Malwarebytes\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Malwarebytes\",\n \"engine_version\": - \"4.2.2.27\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Zillya\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Zillya\",\n \"engine_version\": - \"2.0.0.4424\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210806\"\n },\n - \ \"Sangfor\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Sangfor\",\n \"engine_version\": - \"2.9.0.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210625\"\n },\n - \ \"K7AntiVirus\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"K7AntiVirus\",\n \"engine_version\": - \"11.202.37928\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Alibaba\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Alibaba\",\n \"engine_version\": - \"0.3.0.5\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20190527\"\n },\n - \ \"K7GW\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"K7GW\",\n \"engine_version\": - \"11.202.37928\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Trustlook\": {\n \"category\": \"timeout\",\n - \ \"engine_name\": \"Trustlook\",\n \"engine_version\": - \"1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Arcabit\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Arcabit\",\n \"engine_version\": - \"1.0.0.886\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"BitDefenderTheta\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"BitDefenderTheta\",\n - \ \"engine_version\": \"7.2.37796.0\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210803\"\n },\n \"Cyren\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Cyren\",\n \"engine_version\": - \"6.3.0.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"SymantecMobileInsight\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"SymantecMobileInsight\",\n - \ \"engine_version\": \"2.0\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210126\"\n },\n \"Symantec\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Symantec\",\n \"engine_version\": - \"1.15.0.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ESET-NOD32\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"ESET-NOD32\",\n \"engine_version\": - \"23761\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"APEX\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"APEX\",\n \"engine_version\": - \"6.195\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210807\"\n },\n - \ \"Avast\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Avast\",\n \"engine_version\": - \"21.1.5827.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ClamAV\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"ClamAV\",\n \"engine_version\": - \"0.103.3.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Kaspersky\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Kaspersky\",\n \"engine_version\": - \"21.0.1.45\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"BitDefender\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"BitDefender\",\n \"engine_version\": - \"7.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"NANO-Antivirus\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"NANO-Antivirus\",\n \"engine_version\": - \"1.0.146.25311\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"SUPERAntiSpyware\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"SUPERAntiSpyware\",\n - \ \"engine_version\": \"5.6.0.1032\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210807\"\n },\n \"Tencent\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Tencent\",\n \"engine_version\": - \"1.0.0.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Ad-Aware\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Ad-Aware\",\n \"engine_version\": - \"3.0.21.179\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Emsisoft\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Emsisoft\",\n \"engine_version\": - \"2021.4.0.5819\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Comodo\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Comodo\",\n \"engine_version\": - \"33784\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"F-Secure\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"F-Secure\",\n \"engine_version\": - \"12.0.86.52\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Baidu\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Baidu\",\n \"engine_version\": - \"1.0.0.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20190318\"\n },\n - \ \"VIPRE\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"VIPRE\",\n \"engine_version\": - \"94606\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"TrendMicro\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"TrendMicro\",\n \"engine_version\": - \"11.0.0.1006\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"McAfee-GW-Edition\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"McAfee-GW-Edition\",\n - \ \"engine_version\": \"v2019.1.2+3728\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"SentinelOne\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"SentinelOne\",\n - \ \"engine_version\": \"6.1.0.4\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210805\"\n },\n \"Trapmine\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"Trapmine\",\n - \ \"engine_version\": \"3.5.0.1023\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20200727\"\n },\n \"CMC\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"CMC\",\n \"engine_version\": - \"2.10.2019.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210624\"\n },\n - \ \"Sophos\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Sophos\",\n \"engine_version\": - \"1.3.0.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Paloalto\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Paloalto\",\n \"engine_version\": - \"1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Avast-Mobile\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Avast-Mobile\",\n \"engine_version\": - \"210808-00\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Jiangmin\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Jiangmin\",\n \"engine_version\": - \"16.0.100\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210807\"\n },\n - \ \"Webroot\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Webroot\",\n \"engine_version\": - \"1.0.0.403\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Avira\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Avira\",\n \"engine_version\": - \"8.3.3.12\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"eGambit\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"eGambit\",\n \"engine_version\": - null,\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"MAX\": {\n \"category\": \"confirmed-timeout\",\n - \ \"engine_name\": \"MAX\",\n \"engine_version\": - \"2019.9.16.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Antiy-AVL\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Antiy-AVL\",\n \"engine_version\": - \"3.0.0.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Kingsoft\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Kingsoft\",\n \"engine_version\": - \"2017.9.26.565\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Gridinsoft\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Gridinsoft\",\n \"engine_version\": - \"1.0.51.144\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Microsoft\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Microsoft\",\n \"engine_version\": - \"1.1.18400.4\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ViRobot\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"ViRobot\",\n \"engine_version\": - \"2014.3.20.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"ZoneAlarm\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"ZoneAlarm\",\n \"engine_version\": - \"1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"GData\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"GData\",\n \"engine_version\": - \"A:25.30525B:27.24020\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Cynet\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Cynet\",\n \"engine_version\": - \"4.0.0.27\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"BitDefenderFalx\": {\n \"category\": - \"type-unsupported\",\n \"engine_name\": \"BitDefenderFalx\",\n - \ \"engine_version\": \"2.0.936\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210610\"\n },\n \"AhnLab-V3\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"AhnLab-V3\",\n \"engine_version\": - \"3.20.4.10148\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Acronis\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Acronis\",\n \"engine_version\": - \"1.1.1.82\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210512\"\n },\n - \ \"VBA32\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"VBA32\",\n \"engine_version\": - \"5.0.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210806\"\n },\n - \ \"ALYac\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"ALYac\",\n \"engine_version\": - \"1.1.3.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"TACHYON\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"TACHYON\",\n \"engine_version\": - \"2021-08-08.02\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Cylance\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Cylance\",\n \"engine_version\": - \"2.3.1.101\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Zoner\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Zoner\",\n \"engine_version\": - \"0.0.0.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"TrendMicro-HouseCall\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"TrendMicro-HouseCall\",\n - \ \"engine_version\": \"10.0.0.1040\",\n \"result\": - null,\n \"method\": \"blacklist\",\n \"engine_update\": - \"20210808\"\n },\n \"Rising\": {\n \"category\": - \"undetected\",\n \"engine_name\": \"Rising\",\n \"engine_version\": - \"25.0.0.26\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Yandex\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Yandex\",\n \"engine_version\": - \"5.5.2.24\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Ikarus\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Ikarus\",\n \"engine_version\": - \"0.1.5.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"MaxSecure\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"MaxSecure\",\n \"engine_version\": - \"1.0.0.1\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210807\"\n },\n - \ \"Fortinet\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Fortinet\",\n \"engine_version\": - \"6.2.142.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"Cybereason\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"Cybereason\",\n \"engine_version\": - \"1.2.449\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210330\"\n },\n - \ \"Panda\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Panda\",\n \"engine_version\": - \"4.6.4.2\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n },\n - \ \"CrowdStrike\": {\n \"category\": \"type-unsupported\",\n - \ \"engine_name\": \"CrowdStrike\",\n \"engine_version\": - \"1.0\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210203\"\n },\n - \ \"Qihoo-360\": {\n \"category\": \"undetected\",\n - \ \"engine_name\": \"Qihoo-360\",\n \"engine_version\": - \"1.0.0.1300\",\n \"result\": null,\n \"method\": - \"blacklist\",\n \"engine_update\": \"20210808\"\n }\n - \ },\n \"sandbox_verdicts\": {\n \"C2AE\": - {\n \"category\": \"undetected\",\n \"sandbox_name\": - \"C2AE\",\n \"malware_classification\": [\n \"UNKNOWN_VERDICT\"\n - \ ]\n },\n \"Yomi Hunter\": - {\n \"category\": \"malicious\",\n \"sandbox_name\": - \"Yomi Hunter\",\n \"malware_classification\": [\n \"MALWARE\"\n - \ ]\n },\n \"ReaQta-Hive\": - {\n \"category\": \"malicious\",\n \"sandbox_name\": - \"ReaQta-Hive\",\n \"malware_classification\": [\n \"MALWARE\"\n - \ ]\n }\n },\n \"sha256\": - \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\n \"trusted_verdict\": - {\n \"link\": \"https://dl.google.com/dl/android/cts/android-cts-7.1_r6-linux_x86-arm.zip\",\n - \ \"organization\": \"Google\",\n \"verdict\": - \"goodware\",\n \"filename\": \"android-cts-7.1_r6-linux_x86-arm.zip\"\n - \ },\n \"tags\": [\n \"nsrl\",\n \"zero-filled\",\n - \ \"via-tor\",\n \"known-distributor\",\n \"trusted\",\n - \ \"software-collection\"\n ],\n \"last_analysis_date\": - 1628461968,\n \"unique_sources\": 3756,\n \"first_submission_date\": - 1158564375,\n \"ssdeep\": \"3::\",\n \"oldapps_info\": - {\n \"website\": \"http://oldapps.com/blender.php?old_blender=7584\",\n - \ \"oldapps\": \"http://oldapps.com/blender.php?old_blender=7584?download\",\n - \ \"product\": \"Blender 2.63 (x64)\",\n \"developer\": - \"The Blender Foundation\"\n },\n \"md5\": \"d41d8cd98f00b204e9800998ecf8427e\",\n - \ \"sha1\": \"da39a3ee5e6b4b0d3255bfef95601890afd80709\",\n \"magic\": - \"empty\",\n \"last_analysis_stats\": {\n \"harmless\": - 0,\n \"type-unsupported\": 14,\n \"suspicious\": - 0,\n \"confirmed-timeout\": 1,\n \"timeout\": - 2,\n \"failure\": 0,\n \"malicious\": 0,\n \"undetected\": - 57\n },\n \"meaningful_name\": \"android-cts-7.1_r6-linux_x86-arm.zip\",\n - \ \"reputation\": 1313,\n \"first_seen_itw_date\": 1127126949,\n - \ \"monitor_info\": {\n \"organizations\": [\n \"ObserveIT\",\n - \ \"Google\",\n \"Electric Quilt\",\n - \ \"HP\",\n \"Siemens\",\n \"Geocomply\",\n - \ \"G DATA\",\n \"F. Hoffmann-La Roche - Ltd.\",\n \"Oracle\",\n \"Symantec\",\n - \ \"Microsoft\"\n ],\n \"filenames\": - [\n \"33506fc4-c1d0-46a0-92d4-047ce0d07634\",\n \"68bae406-c776-4959-82a4-cb1fdae972a8\",\n - \ \"Iron.dll-1d30936a490b8e8499467272760d38f3d234b9544b1589bc3e47ea9285e59453\",\n - \ \"fsdui.exe-0237ac5da79ecec5af54fc0e28aaea5da5690ba93f36d0f3c6b005fe0a7dffda\",\n - \ \"fsdui.exe-ed3d6532815513ef074e122f893d800347075f038046de3a9028e4231328f9ff\",\n - \ \"bbRGen.dll-16d707bcf5a581a03e1d965022ac83ba0afa3dc5169695ac4a2c6bc1fda3300f\",\n - \ \"Setup.exe-94802d243f960f849f40fc437b53bf89b92f14a91b65fef6bb0eb1ceaaf0b53c\",\n - \ \"18fee64a-1e7f-4932-8158-e8ab241f3a1a\",\n \"Symantec_Agent_setup.exe-03bc90030768e6733a78240f0b5705d31b13b0cb5a7577cf2d375f2a55303476\",\n - \ \"NFT.exe-65d67829fe5c1a08444d1d3b0dcf76304454dd383e9d2385c421286f3f856169\",\n - \ \"fsdui.exe-c7df0544d8212d40735d3aa5e7ef953472279fdf0cc4c64230d37e281a1924af\",\n - \ \"fsdui.exe-6df75472c30b3b1a7517cf9255666195da0d032360324bbe5d6f7bc073618447\",\n - \ \"fsdui.exe-32e6163b611c1cbac85769addfe94ac081c7dd1786146522733c054d26d61ff2\",\n - \ \"ca3af54a-b237-4268-8cbc-45c2fdd762e9\",\n \"b42ad700-47d3-4b30-ae9c-7f1328ea9232\",\n - \ \"9f9f5312-8d34-4b13-84b9-f488e06712ae\",\n \"4596e1fa-32c3-4000-be44-3182e41b236c\",\n - \ \"b73edfa0-ab54-410b-8d3d-f69b6245fb79\",\n \"BUShell.dll-0b932bd4bcde42bede793db9397a9ddbfa867e4dc68df6dce408e369c8858d84\",\n - \ \"4bf8f4c8-2812-448e-9988-0dd3695741c6\",\n \"fsdui.exe-68fbb12e2be2b5cabf952c82241fa67483cf6b4f3ad339e15820b160bd658a91\",\n - \ \"hpsoftpaqwrapper.exe-a0419c2754398d2d144da8adbf9c084576c97a768979074af82c96421915b7e0\",\n - \ \"ee0e1764-6dc5-4b0b-b9cc-a342d9818a1c\",\n \"2b3a456c-6fa1-4154-8ac2-eb81e5484659\",\n - \ \"fsdui.exe-5869a7509cd7234629877e7ab2644ba5e58b736c4c47d5fdbf8060d5d194db31\",\n - \ \"2229eeeb-7151-4870-8d88-b670c9638054\",\n \"9f5c2cce-13d6-4b70-814d-ec7e1106be04\",\n - \ \"SymHelp.exe-803494eb0408221dac2abed266c6bb8c2409fc3e10d73529103ceac5844a5c5f\",\n - \ \"14aef02b-32ec-431e-87e1-9ed84e3855f8\",\n \"7bbb49e8-94d6-463d-9e39-435c467efb8c\",\n - \ \"Symantec_Agent_setup.exe-d7467080d5d0073dea71e017d03a4f630476acb04e4c814276e704d8fca902a2\",\n - \ \"AeXClientUpgrade.exe-57f39f752ac0eacb7165c05544389ea39545ee3ef9a0ac669c2253a468daf140\",\n - \ \"fsdui.exe-1ecac0bec11c59b9ea26dd854b6dce414a22f0ca62abefecbb01f002f0c4ab4b\",\n - \ \"eaa60501-e1aa-43de-ad86-1f78439e2125\",\n \"SISIPSService.exe-c286b214de6e7821c02055386d707ed9386d4c2b0bf636a74feb9d1985a5f177\",\n - \ \"980c8146-2c94-424a-af83-ef8b02166fba\",\n \"c42d391d-6af6-4cdb-b1d5-b8ca07a76e58\",\n - \ \"LiveUpdate.exe-98b0003756978b45845938f9af4d6e85cd05b53d0f86596df28645b70f97c9bd\",\n - \ \"608e2c2b-37f7-4c2e-b1c6-be5dc0a01be3\",\n \"fsdui.exe-ef719e9b143fd2b7ef4486d264ff227bd560277b2f1d20dc4b8372e0d7b40ab9\",\n - \ \"fsdui.exe-8812249287a7cd82ebc171c47f3a92c5c02ea842ea17dd26222d946fc7f5d8ba\",\n - \ \"4d178c36-c32b-4e9e-84ae-cdc47a131789\",\n \"JAWTAccessBridge-32.dll-2c0c398ac2cff2f075679d3c03228246a9841897b32901019b733625fd57d53a\",\n - \ \"fsdui.exe-4d4aa42e59b95c93b2c0644f5b27257004e8ad1b811e8ecbdadd49689c9e29e6\",\n - \ \"fsdui.exe-3401b9dc2e1000b4fa64453afe7c4aa897d997afc37585910e81760e838f5545\",\n - \ \"fsdui.exe-f9fc86c86cebccae5318cfaa1131ea2669dc74ba3b4d45a715ed6d2dfc45ef08\",\n - \ \"fsdui.exe-183fdc6338ff35d5ae75069a9f0b8c6b18b4ee427488d1a95bc8ea8e5d1c122a\",\n - \ \"online_wrapper-cab.exe-e0d7fc83fac4a4c4c6e646f5c0b31b5aa3973625fc466ebeaa4ea86f2339009a\",\n - \ \"SETDAD.CredManagerDarkCorner.dll-c07a22d5a2c06d628625e68a585bb99bd4c619f3f1ee2ea9d7811796cf5740f1\",\n - \ \"3f1cb173-bbc2-44ff-960b-9beb296ce30b\"\n ]\n - \ }\n },\n \"type\": \"file\",\n \"id\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\n - \ \"links\": {\n \"self\": \"https://www.virustotal.com/api/v3/files/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"\n - \ }\n }\n}" - headers: - Cache-Control: - - no-cache - Content-Encoding: - - gzip - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 08 Aug 2021 22:40:51 GMT - Server: - - Google Frontend - Vary: - - Accept-Encoding - X-Cloud-Trace-Context: - - afbfa6f1391e1c798463ecc794ec2804 - status: - code: 200 - message: OK - url: https://www.virustotal.com/api/v3/files/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -version: 1 diff --git a/tests/schemas/test_payload.py b/tests/schemas/test_payload.py index 1f97df5..51819f1 100644 --- a/tests/schemas/test_payload.py +++ b/tests/schemas/test_payload.py @@ -4,19 +4,19 @@ def test_sample_eml(sample_eml: bytes): - FilePayload(file=sample_eml) + assert FilePayload(file=sample_eml) is not None def test_multipart_eml(multipart_eml: bytes): - FilePayload(file=multipart_eml) + assert FilePayload(file=multipart_eml) is not None def test_encrypted_docx_eml(encrypted_docx_eml: bytes): - FilePayload(file=encrypted_docx_eml) + assert FilePayload(file=encrypted_docx_eml) is not None def test_cc_eml(cc_eml: bytes): - FilePayload(file=cc_eml) + assert FilePayload(file=cc_eml) is not None def test_invalid_eml_file(): diff --git a/tests/services/__init__.py b/tests/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/services/test_emailrep.py b/tests/services/test_emailrep.py deleted file mode 100644 index 7ed2399..0000000 --- a/tests/services/test_emailrep.py +++ /dev/null @@ -1,15 +0,0 @@ -import httpx -import pytest -from respx import MockRouter - -from backend.services.emailrep import EmailRep - - -@pytest.mark.asyncio -async def test_get(emailrep_response, respx_mock: MockRouter): - respx_mock.get("https://emailrep.io/bill@microsoft.com").mock( - return_value=httpx.Response(200, content=emailrep_response), - ) - emailrep = EmailRep() - res = await emailrep.get("bill@microsoft.com") - assert res.email == "bill@microsoft.com" diff --git a/tests/services/test_extractor.py b/tests/services/test_extractor.py deleted file mode 100644 index 390555e..0000000 --- a/tests/services/test_extractor.py +++ /dev/null @@ -1,36 +0,0 @@ -from backend.services.extractor import parse_urls_from_body - - -def test_parse_urls_from_body_with_html(test_html: str): - urls = parse_urls_from_body(test_html, "text/html") - - assert len(urls) > 0 - assert "http://www.w3.org/TR/html4/loose.dtd" not in urls - assert "http://example.com" in urls - - # check whether urls are unique or not - assert len(set(urls)) == len(urls) - - -def test_parse_urls_from_body_with_text(): - urls = parse_urls_from_body("[http://example.com]", "text/plain") - assert len(urls) == 1 - assert "http://example.com" in urls - - urls = parse_urls_from_body("", "text/plain") - assert len(urls) == 1 - assert "http://example.com" in urls - - urls = parse_urls_from_body( - " [http://example.com]", "text/plain" - ) - assert len(urls) == 1 - assert "http://example.com" in urls - - -def test_parse_urls_with_safelinks(): - urls = parse_urls_from_body( - "https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.google.com%2F", - "text/plain", - ) - assert "https://www.google.com/" in urls diff --git a/tests/services/test_inquest.py b/tests/services/test_inquest.py deleted file mode 100644 index 03767a2..0000000 --- a/tests/services/test_inquest.py +++ /dev/null @@ -1,53 +0,0 @@ -from io import BytesIO - -import httpx -import pytest -from respx import MockRouter - -from backend.services.inquest import InQuest - - -@pytest.mark.asyncio -async def test_dfi_details(inquest_dfi_details_response: str, respx_mock: MockRouter): - sha256 = "e86c5988a3a6640fb90b90b9e9200e4cce0669594dbb5422622946208c124149" - respx_mock.get(f"https://labs.inquest.net/api/dfi/details?sha256={sha256}").mock( - return_value=httpx.Response(200, content=inquest_dfi_details_response), - ) - - api = InQuest() - res = await api.dfi_details(sha256) - assert res is not None - data = res.get("data", {}) - assert data.get("classification", "") == "MALICIOUS" - - -@pytest.mark.asyncio -async def test_dfi_details_with_eicar(respx_mock: MockRouter): - sha256 = "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f" - respx_mock.get( - f"https://labs.inquest.net/api/dfi/details?sha256={sha256}", - ).mock(return_value=httpx.Response(404, content="")) - - api = InQuest() - res = await api.dfi_details(sha256) - assert res is None - - -@pytest.mark.asyncio -async def test_dfi_upload( - encrypted_docx: bytes, inquest_dfi_upload_response: str, respx_mock: MockRouter -): - respx_mock.post("https://labs.inquest.net/api/dfi/upload").mock( - return_value=httpx.Response(200, content=inquest_dfi_upload_response) - ) - - file_ = BytesIO(encrypted_docx) - file_.name = "encrypted.docx" - - api = InQuest() - res = await api.dfi_upload(file_) - assert res is not None - assert ( - res.get("data", "") - == "539e4557975be726d0fc8d7813dba5470dd703272957b4476296f976b5678900" - ) diff --git a/tests/services/test_oleid.py b/tests/services/test_oleid.py deleted file mode 100644 index 1c3b263..0000000 --- a/tests/services/test_oleid.py +++ /dev/null @@ -1,17 +0,0 @@ -from backend.services.oleid import OleID - - -def test_encrypted_docx(encrypted_docx: bytes): - oid = OleID(encrypted_docx) - - assert oid.is_encrypted() is True - assert oid.has_flash_objects() is False - assert oid.has_vba_macros() is False - - -def test_xls_with_macro(xls_with_macro: bytes): - oid = OleID(xls_with_macro) - - assert oid.is_encrypted() is True - assert oid.has_flash_objects() is False - assert oid.has_vba_macros() is True diff --git a/tests/services/test_outlookmsgfile.py b/tests/services/test_outlookmsgfile.py deleted file mode 100644 index 67f199c..0000000 --- a/tests/services/test_outlookmsgfile.py +++ /dev/null @@ -1,27 +0,0 @@ -from io import BytesIO - -from backend.services.outlookmsgfile import Message - - -def test_other_msg(other_msg: bytes): - file = BytesIO(other_msg) - message = Message(file) - email = message.to_email() - - assert email["Subject"] == "投递状态通知 (Failure Notice)" - assert email["To"] == "yosipnps@model.com" - - attachments = list(email.iter_attachments()) - assert len(attachments) == 1 - - -def test_outer_msg(outer_msg: bytes): - file = BytesIO(outer_msg) - message = Message(file) - email = message.to_email() - - assert email["Subject"] == "outer subject" - assert email["To"] == "outer@foo.bar" - - attachments = list(email.iter_attachments()) - assert len(attachments) == 1 diff --git a/tests/services/test_urlscan.py b/tests/services/test_urlscan.py deleted file mode 100644 index 466a7f6..0000000 --- a/tests/services/test_urlscan.py +++ /dev/null @@ -1,31 +0,0 @@ -import httpx -import pytest -from respx import MockRouter - -from backend.services.urlscan import Urlscan - - -@pytest.mark.asyncio -async def test_search(urlscan_search_response: str, respx_mock: MockRouter): - respx_mock.get( - "https://urlscan.io/api/v1/search/?q=task.url%3A%22http%3A%2F%2Frakuten-ia.com%2F%22&size=10", - ).mock( - return_value=httpx.Response(200, content=urlscan_search_response), - ) - api = Urlscan() - res = await api.search("http://rakuten-ia.com/") - results = res.get("results") - assert isinstance(results, list) is True - - -@pytest.mark.asyncio -async def test_result(urlscan_result_response: str, respx_mock: MockRouter): - uuid = "3db439ff-036f-409f-96d6-c28da55767f4" - respx_mock.get( - "https://urlscan.io/api/v1/result/3db439ff-036f-409f-96d6-c28da55767f4/", - ).mock( - return_value=httpx.Response(200, content=urlscan_result_response), - ) - api = Urlscan() - res = await api.result(uuid) - assert res.get("task", {}).get("uuid", "") == uuid diff --git a/tests/submitters/__init__.py b/tests/submitters/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/submitters/test_inquest.py b/tests/submitters/test_inquest.py deleted file mode 100644 index c3da5ac..0000000 --- a/tests/submitters/test_inquest.py +++ /dev/null @@ -1,25 +0,0 @@ -import httpx -import pytest -from respx import MockRouter - -from backend.schemas.eml import Attachment -from backend.submitters.inquest import InQuestSubmitter - - -@pytest.mark.asyncio -async def test_inquest( - docx_attachment: Attachment, - inquest_dfi_upload_response: str, - respx_mock: MockRouter, -): - respx_mock.post("https://labs.inquest.net/api/dfi/upload").mock( - return_value=httpx.Response(200, content=inquest_dfi_upload_response), - ) - - submitter = InQuestSubmitter(docx_attachment) - - result = await submitter.submit() - assert ( - result.reference_url - == "https://labs.inquest.net/dfi/sha256/539e4557975be726d0fc8d7813dba5470dd703272957b4476296f976b5678900" - ) diff --git a/tests/submitters/test_virustotal.py b/tests/submitters/test_virustotal.py deleted file mode 100644 index eeef305..0000000 --- a/tests/submitters/test_virustotal.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -from aioresponses import aioresponses - -from backend.schemas.eml import Attachment -from backend.submitters.virustotal import VirusTotalSubmitter - - -@pytest.mark.asyncio -async def test_virustotal(docx_attachment: Attachment): - submitter = VirusTotalSubmitter(docx_attachment) - - with aioresponses() as aiomock: - aiomock.get( - "https://www.virustotal.com/api/v3/files/upload_url", - payload={"data": "https://www.virustotal.com/_ah/upload/foo"}, - ) - - analysis_data = {"type": "analysis", "id": "foo"} - aiomock.post( - "https://www.virustotal.com/_ah/upload/foo", - payload={"data": analysis_data}, - ) - - result = await submitter.submit() - assert ( - result.reference_url - == "https://www.virustotal.com/gui/file/28df2d6dfa10dc85c8ebb5defffcb15c196dca7b26d4fd6859b9ec75ac60cf9e/detection" - )