From 39784af727a373b020cbbaa0286aac5cf3e460f8 Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 1/7] Updates API v2 GET payloads Adds optional parameters: - sort: Sorts the returned list, - exclude_plugins: Excludes payloads of plugins (only retains data/payloads ones). --- app/api/v2/handlers/payload_api.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index 6ffe14274..cafd71055 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -3,14 +3,17 @@ import aiohttp_apispec from aiohttp import web -import marshmallow as ma +from marshmallow import fields, schema from app.api.v2.handlers.base_api import BaseApi -from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema -class PayloadSchema(ma.Schema): - payloads = ma.fields.List(ma.fields.String()) +class PayloadQuerySchema(schema.Schema): + sort = fields.Boolean(required=False, default=False) + exclude_plugins = fields.Boolean(required=False, default=False) + +class PayloadSchema(schema.Schema): + payloads = fields.List(fields.String()) class PayloadApi(BaseApi): @@ -26,18 +29,28 @@ def add_routes(self, app: web.Application): @aiohttp_apispec.docs(tags=['payloads'], summary='Retrieve payloads', description='Retrieves all stored payloads.') - @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema) + @aiohttp_apispec.querystring_schema(PayloadQuerySchema) @aiohttp_apispec.response_schema(PayloadSchema(), description='Returns a list of all payloads in PayloadSchema format.') async def get_payloads(self, request: web.Request): + sort: bool = request['querystring'].get('sort') + exclude_plugins: bool = request['querystring'].get('exclude_plugins') + cwd = pathlib.Path.cwd() payload_dirs = [cwd / 'data' / 'payloads'] - payload_dirs.extend(cwd / 'plugins' / plugin.name / 'payloads' - for plugin in await self.data_svc.locate('plugins') if plugin.enabled) + + if not exclude_plugins: + payload_dirs.extend(cwd / 'plugins' / plugin.name / 'payloads' + for plugin in await self.data_svc.locate('plugins') if plugin.enabled) + payloads = { self.file_svc.remove_xored_extension(p.name) for p in itertools.chain.from_iterable(p_dir.glob('[!.]*') for p_dir in payload_dirs) if p.is_file() } - return web.json_response(list(payloads)) + payloads = list(payloads) + if sort: + payloads.sort() + + return web.json_response(payloads) From 037a18061e6bee2880508decf5c35c9b2c5a33ac Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 2/7] Updates API v2 GET payloads Adds the optional parameter: add_path: Adds the relative path to the payload, from the Caldera root directory. --- app/api/v2/handlers/payload_api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index cafd71055..422378f90 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -11,6 +11,7 @@ class PayloadQuerySchema(schema.Schema): sort = fields.Boolean(required=False, default=False) exclude_plugins = fields.Boolean(required=False, default=False) + add_path = fields.Boolean(required=False, default=False) class PayloadSchema(schema.Schema): payloads = fields.List(fields.String()) @@ -35,6 +36,7 @@ def add_routes(self, app: web.Application): async def get_payloads(self, request: web.Request): sort: bool = request['querystring'].get('sort') exclude_plugins: bool = request['querystring'].get('exclude_plugins') + add_path: bool = request['querystring'].get('add_path') cwd = pathlib.Path.cwd() payload_dirs = [cwd / 'data' / 'payloads'] @@ -44,7 +46,9 @@ async def get_payloads(self, request: web.Request): for plugin in await self.data_svc.locate('plugins') if plugin.enabled) payloads = { - self.file_svc.remove_xored_extension(p.name) + str(p.parent.relative_to(cwd) / self.file_svc.remove_xored_extension(p.name)) + if add_path + else self.file_svc.remove_xored_extension(p.name) for p in itertools.chain.from_iterable(p_dir.glob('[!.]*') for p_dir in payload_dirs) if p.is_file() } From e4fe33bd78be47fe6161ab65cbc3f504e0837909 Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 3/7] Adds API v2 POST payloads --- app/api/v2/handlers/payload_api.py | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index 422378f90..b4151b748 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -1,5 +1,8 @@ +import asyncio import itertools +import os import pathlib +from io import IOBase import aiohttp_apispec from aiohttp import web @@ -16,6 +19,9 @@ class PayloadQuerySchema(schema.Schema): class PayloadSchema(schema.Schema): payloads = fields.List(fields.String()) +class PayloadCreateRequestSchema(schema.Schema): + file = fields.Raw(type="file", required=True) + class PayloadApi(BaseApi): def __init__(self, services): @@ -26,6 +32,7 @@ def __init__(self, services): def add_routes(self, app: web.Application): router = app.router router.add_get('/payloads', self.get_payloads) + router.add_post("/payloads", self.post_payloads) @aiohttp_apispec.docs(tags=['payloads'], summary='Retrieve payloads', @@ -58,3 +65,49 @@ async def get_payloads(self, request: web.Request): payloads.sort() return web.json_response(payloads) + + @aiohttp_apispec.docs( + tags=['payloads'], + summary='Create a payload', + description='Uploads a payload.') + @aiohttp_apispec.form_schema(PayloadCreateRequestSchema) + @aiohttp_apispec.response_schema( + PayloadSchema(), + description="The created payload in a list in PayloadSchema format (with name changed in case of a duplicate).") + async def post_payloads(self, request: web.Request): + # As aiohttp_apispec.form_schema already calls request.multipart(), + # accessing the file using the prefilled request["form"] dictionary. + file_field: web.FileField = request["form"]["file"] + + file_name_candidate: str = file_field.filename + file_path: str = os.path.join('data/payloads/', file_name_candidate) + suffix: int = 1 + + # Generating a file suffix in the case it already exists. + while os.path.exists(file_path): + file_name_candidate = f"{pathlib.Path(file_field.filename).stem}_" \ + f"{suffix}{pathlib.Path(file_field.filename).suffix}" + file_path = os.path.join('data/payloads/', file_name_candidate) + suffix += 1 + + file_name: str = file_name_candidate + + # The file_field.file is of type IOBase: It uses blocking methods. + # Putting blocking code into a dedicated method and thread... + def save_file(target_file_path: str, io_base_src: IOBase): + size: int = 0 + read_chunk: bool = True + with open(target_file_path, 'wb') as buffered_io_base_dest: + while read_chunk: + chunk: bytes = io_base_src.read(8192) + if chunk: + size += len(chunk) + buffered_io_base_dest.write(chunk) + else: + read_chunk = False + + loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() + await loop.run_in_executor(None, save_file, file_path, file_field.file) + + body: dict[list[str]] = {"payloads": [file_name]} + return web.json_response(body) From d1eeba700c14ffd6625bb4c99cfa1142a7f23dd5 Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 4/7] Adds API v2 DELETE payloads --- app/api/v2/handlers/payload_api.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index b4151b748..f482e20bf 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -22,6 +22,9 @@ class PayloadSchema(schema.Schema): class PayloadCreateRequestSchema(schema.Schema): file = fields.Raw(type="file", required=True) +class PayloadDeleteRequestSchema(schema.Schema): + name = fields.String(required=True) + class PayloadApi(BaseApi): def __init__(self, services): @@ -33,6 +36,7 @@ def add_routes(self, app: web.Application): router = app.router router.add_get('/payloads', self.get_payloads) router.add_post("/payloads", self.post_payloads) + router.add_delete("/payloads/{name}", self.delete_payloads) @aiohttp_apispec.docs(tags=['payloads'], summary='Retrieve payloads', @@ -111,3 +115,24 @@ def save_file(target_file_path: str, io_base_src: IOBase): body: dict[list[str]] = {"payloads": [file_name]} return web.json_response(body) + + @aiohttp_apispec.docs( + tags=['payloads'], + summary='Delete a payload', + description='Deletes a given payload.', + responses = { + 204: {"description": "Payload has been properly deleted."}, + 404: {"description": "Payload not found."}, + }) + @aiohttp_apispec.match_info_schema(PayloadDeleteRequestSchema) + async def delete_payloads(self, request: web.Request): + file_name: str = request.match_info.get("name") + file_path: str = os.path.join('data/payloads/', file_name) + + response: web.HTTPException = None + try: + os.remove(file_path) + response = web.HTTPNoContent() + except FileNotFoundError: + response = web.HTTPNotFound() + return response From e7ac1390ae642c5c6e97bf5eb817320482d563d7 Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 5/7] Moves payload API schemas into a dedicated module --- app/api/v2/handlers/payload_api.py | 54 +++++++++++++-------------- app/api/v2/schemas/payload_schemas.py | 19 ++++++++++ 2 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 app/api/v2/schemas/payload_schemas.py diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index f482e20bf..e0c5c5cc3 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -6,24 +6,10 @@ import aiohttp_apispec from aiohttp import web -from marshmallow import fields, schema from app.api.v2.handlers.base_api import BaseApi - - -class PayloadQuerySchema(schema.Schema): - sort = fields.Boolean(required=False, default=False) - exclude_plugins = fields.Boolean(required=False, default=False) - add_path = fields.Boolean(required=False, default=False) - -class PayloadSchema(schema.Schema): - payloads = fields.List(fields.String()) - -class PayloadCreateRequestSchema(schema.Schema): - file = fields.Raw(type="file", required=True) - -class PayloadDeleteRequestSchema(schema.Schema): - name = fields.String(required=True) +from app.api.v2.schemas.payload_schemas import PayloadQuerySchema, PayloadSchema, PayloadCreateRequestSchema, \ + PayloadDeleteRequestSchema class PayloadApi(BaseApi): @@ -83,18 +69,7 @@ async def post_payloads(self, request: web.Request): # accessing the file using the prefilled request["form"] dictionary. file_field: web.FileField = request["form"]["file"] - file_name_candidate: str = file_field.filename - file_path: str = os.path.join('data/payloads/', file_name_candidate) - suffix: int = 1 - - # Generating a file suffix in the case it already exists. - while os.path.exists(file_path): - file_name_candidate = f"{pathlib.Path(file_field.filename).stem}_" \ - f"{suffix}{pathlib.Path(file_field.filename).suffix}" - file_path = os.path.join('data/payloads/', file_name_candidate) - suffix += 1 - - file_name: str = file_name_candidate + file_name, file_path = await self.__generate_file_name_and_path(file_field) # The file_field.file is of type IOBase: It uses blocking methods. # Putting blocking code into a dedicated method and thread... @@ -136,3 +111,26 @@ async def delete_payloads(self, request: web.Request): except FileNotFoundError: response = web.HTTPNotFound() return response + + @classmethod + async def __generate_file_name_and_path(cls, file_field: web.FileField) -> [str, str]: + """ + Finds whether an uploaded file already exists in the payload directory. + In the case, generates a new file name with an incremental suffix to avoid overriding the existing one. + Otherwise, the original file name is used. + + :param file_field: The upload payload object. + :return: A tuple containing the generated file name and path for future storage. + """ + file_name_candidate: str = file_field.filename + file_path: str = os.path.join('data/payloads/', file_name_candidate) + suffix: int = 1 + + # Generating a file suffix in the case it already exists. + while os.path.exists(file_path): + file_name_candidate = f"{pathlib.Path(file_field.filename).stem}_" \ + f"{suffix}{pathlib.Path(file_field.filename).suffix}" + file_path = os.path.join('data/payloads/', file_name_candidate) + suffix += 1 + file_name: str = file_name_candidate + return file_name, file_path diff --git a/app/api/v2/schemas/payload_schemas.py b/app/api/v2/schemas/payload_schemas.py new file mode 100644 index 000000000..99b8213e6 --- /dev/null +++ b/app/api/v2/schemas/payload_schemas.py @@ -0,0 +1,19 @@ +from marshmallow import fields, schema + + +class PayloadQuerySchema(schema.Schema): + sort = fields.Boolean(required=False, default=False) + exclude_plugins = fields.Boolean(required=False, default=False) + add_path = fields.Boolean(required=False, default=False) + + +class PayloadSchema(schema.Schema): + payloads = fields.List(fields.String()) + + +class PayloadCreateRequestSchema(schema.Schema): + file = fields.Raw(type="file", required=True) + + +class PayloadDeleteRequestSchema(schema.Schema): + name = fields.String(required=True) From d8b03fcad3762633413bd3d72870600caf01cf7d Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Fri, 7 Jun 2024 21:20:23 +0200 Subject: [PATCH 6/7] Moves a payload file upload function at class level --- app/api/v2/handlers/payload_api.py | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/app/api/v2/handlers/payload_api.py b/app/api/v2/handlers/payload_api.py index e0c5c5cc3..51fee89cd 100644 --- a/app/api/v2/handlers/payload_api.py +++ b/app/api/v2/handlers/payload_api.py @@ -73,20 +73,8 @@ async def post_payloads(self, request: web.Request): # The file_field.file is of type IOBase: It uses blocking methods. # Putting blocking code into a dedicated method and thread... - def save_file(target_file_path: str, io_base_src: IOBase): - size: int = 0 - read_chunk: bool = True - with open(target_file_path, 'wb') as buffered_io_base_dest: - while read_chunk: - chunk: bytes = io_base_src.read(8192) - if chunk: - size += len(chunk) - buffered_io_base_dest.write(chunk) - else: - read_chunk = False - loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() - await loop.run_in_executor(None, save_file, file_path, file_field.file) + await loop.run_in_executor(None, self.__save_file, file_path, file_field.file) body: dict[list[str]] = {"payloads": [file_name]} return web.json_response(body) @@ -134,3 +122,23 @@ async def __generate_file_name_and_path(cls, file_field: web.FileField) -> [str, suffix += 1 file_name: str = file_name_candidate return file_name, file_path + + @staticmethod + def __save_file(target_file_path: str, io_base_src: IOBase): + """ + Save an uploaded file content into a targeted file path. + Note this method calls blocking methods and must be run into a dedicated thread. + + :param target_file_path: The destination path to write to. + :param io_base_src: The stream with file content to read from. + """ + size: int = 0 + read_chunk: bool = True + with open(target_file_path, 'wb') as buffered_io_base_dest: + while read_chunk: + chunk: bytes = io_base_src.read(8192) + if chunk: + size += len(chunk) + buffered_io_base_dest.write(chunk) + else: + read_chunk = False From 06262853c13cf5a1c8e8db2380f72e5bb82186cc Mon Sep 17 00:00:00 2001 From: jean-baptiste-perez-bib Date: Tue, 4 Jun 2024 17:39:16 +0200 Subject: [PATCH 7/7] Uses a dedicated setting to generate the magma VITE_CALDERA_URL variable The generated .env file used "host" and "port" settings which were not suitable for client URL generation (they are HTTP server listening settings, not client URL components). --- conf/default.yml | 1 + server.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/default.yml b/conf/default.yml index afae89d9d..4e3e45488 100644 --- a/conf/default.yml +++ b/conf/default.yml @@ -22,6 +22,7 @@ app.contact.ftp.user: caldera_user app.contact.tcp: 0.0.0.0:7010 app.contact.udp: 0.0.0.0:7011 app.contact.websocket: 0.0.0.0:7012 +app.frontend.api_base_url: http://localhost:8888 objects.planners.default: atomic crypt_salt: REPLACE_WITH_RANDOM_VALUE encryption_key: ADMIN123 diff --git a/server.py b/server.py index a759e7f55..7fc4d5b40 100644 --- a/server.py +++ b/server.py @@ -149,10 +149,9 @@ async def start_vue_dev_server(): def configure_magma_env_file(): logging.info("Setting VueJS environment file.") - host = BaseWorld.get_config("host") - port = BaseWorld.get_config("port") + url = BaseWorld.get_config("app.frontend.api_base_url") with open(f"{MAGMA_PATH}/.env", "w") as fp: - fp.write(f"VITE_CALDERA_URL=http://{host}:{port}") + fp.write(f"VITE_CALDERA_URL={url}") def _get_parser():