Merge pull request #498 from OpenMined/eelco/fix-deployment

Migrate before running the server
OpenMined · Dec 12, 2024 · 3fb2b64 · 3fb2b64
2 parents c70c984 + 91e9548
commit 3fb2b64
Show file tree

Hide file tree

Showing 10 changed files with 116 additions and 262 deletions.
diff --git a/docker/syftbox.dockerfile b/docker/syftbox.dockerfile
@@ -14,4 +14,4 @@ RUN uv pip install --no-cache syftbox==${SYFT_VERSION}
 
 EXPOSE 8000
 
-CMD ["uv", "run", "uvicorn", "syftbox.server.server:app", "--host=0.0.0.0", "--port=8000"]
+CMD ["uv", "run", "gunicorn", "syftbox.server.server:app", "--bind=0.0.0.0:8000"]
diff --git a/justfile b/justfile
@@ -22,6 +22,7 @@ _nc := '\033[0m'
 # Aliases
 
 alias rs := run-server
+alias rsu := run-server-uvicorn
 alias rc := run-client
 alias rj := run-jupyter
 alias b := build
@@ -35,11 +36,22 @@ alias b := build
 
 # Run a local syftbox server on port 5001
 [group('server')]
-run-server port="5001" uvicorn_args="":
+run-server port="5001" gunicorn_args="":
     #!/bin/bash
-    mkdir -p .server/data
-    SYFTBOX_DATA_FOLDER=.server/data SYFTBOX_ENV=${SYFTBOX_ENV:-DEV}  SYFTBOX_OTEL_ENABLED=${SYFTBOX_OTEL_ENABLED:-0} \
-    uv run uvicorn syftbox.server.server:app --reload --reload-dir ./syftbox --port {{ port }} {{ uvicorn_args }}
+    set -eou pipefail
+
+    export SYFTBOX_DATA_FOLDER=.server/data
+    uv run syftbox server migrate
+    uv run gunicorn syftbox.server.server:app -k uvicorn.workers.UvicornWorker --bind 127.0.0.1:{{ port }} --reload {{ gunicorn_args }}
+
+[group('server')]
+run-server-uvicorn port="5001" uvicorn_args="":
+    #!/bin/bash
+    set -eou pipefail
+
+    export SYFTBOX_DATA_FOLDER=.server/data
+    uv run syftbox server migrate
+    uv run uvicorn syftbox.server.server:app --host 127.0.0.1 --port {{ port }} --reload --reload-dir ./syftbox {{ uvicorn_args }}
 
 # ---------------------------------------------------------------------------------------------------------------------
 
@@ -66,7 +78,7 @@ run-client name port="auto" server="http://localhost:5001":
     echo -e "Server     : {{ _cyan }}{{ server }}{{ _nc }}"
     echo -e "Data Dir   : $DATA_DIR"
 
-    uv run syftbox/client/cli.py --config=$DATA_DIR/config.json --data-dir=$DATA_DIR --email=$EMAIL --port=$PORT --server={{ server }} --no-open-dir
+    uv run syftbox client --config=$DATA_DIR/config.json --data-dir=$DATA_DIR --email=$EMAIL --port=$PORT --server={{ server }} --no-open-dir
 
 # ---------------------------------------------------------------------------------------------------------------------
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -7,6 +7,7 @@ requires-python = ">=3.9"
 dependencies = [
     "fastapi==0.115.6",
     "uvicorn==0.32.1",
+    "gunicorn==23.0.0",
     "jinja2==3.1.4",
     "typing-extensions==4.12.2",
     "pydantic-settings==2.6.1",
@@ -33,12 +34,10 @@ dependencies = [
 # Local dependencies for development
 # Add using `uv add --group <group> <pip package>`
 [dependency-groups]
-ds = ["pandas>=2.2.3"]
 
 # Published optional dependencies, or "extras". Will be referenced in the built wheel
 # add using `uv add --optional <group> <pip package>`
 [project.optional-dependencies]
-server = ["gunicorn>=23.0.0"]
 
 [project.scripts]
 syftbox = "syftbox.main:main"

diff --git a/syftbox/server/cli.py b/syftbox/server/cli.py
@@ -1,10 +1,12 @@
-from pathlib import Path
-from typing import Annotated, Optional
+from loguru import logger
+from typer import Exit, Option, Typer
 
-from typer import Context, Option, Typer
+from syftbox.server.migrations import run_migrations
+from syftbox.server.settings import ServerSettings
 
 app = Typer(
     name="SyftBox Server",
+    no_args_is_help=True,
     pretty_exceptions_enable=False,
     add_completion=False,
     context_settings={"help_option_names": ["-h", "--help"]},
@@ -16,75 +18,27 @@
 SERVER_PANEL = "Server Options"
 SSL_PANEL = "SSL Options"
 
-PORT_OPTS = Option(
-    "-p", "--port",
-    rich_help_panel=SERVER_PANEL,
-    help="Local port for the SyftBox client",
-)
-WORKERS_OPTS = Option(
-    "-w", "--workers",
-    rich_help_panel=SERVER_PANEL,
-    help="Number of worker processes",
-)
-VERBOSE_OPTS = Option(
+EXAMPLE_OPTS = Option(
     "-v", "--verbose",
     is_flag=True,
     rich_help_panel=SERVER_PANEL,
     help="Enable verbose mode",
 )
-RELOAD_OPTS = Option(
-    "--reload", "--debug",
-    rich_help_panel=SERVER_PANEL,
-    help="Enable debug mode",
-)
-EMAIL_OPTS = Option(
-    "-e", "--email",
-    rich_help_panel=SERVER_PANEL,
-    help="Email to ban/unban",
-)
-SSL_KEY_OPTS = Option(
-    "--key", "--ssl-keyfile",
-    exists=True, file_okay=True, readable=True,
-    rich_help_panel=SSL_PANEL,
-    help="Path to SSL key file",
-)
-SSL_CERT_OPTS = Option(
-    "--cert", "--ssl-certfile",
-    exists=True, file_okay=True, readable=True,
-    rich_help_panel=SSL_PANEL,
-    help="Path to SSL certificate file",
-)
 # fmt: on
 
 
-@app.callback(invoke_without_command=True)
-def server(
-    ctx: Context,
-    port: Annotated[int, PORT_OPTS] = 5001,
-    workers: Annotated[int, WORKERS_OPTS] = 1,
-    verbose: Annotated[bool, VERBOSE_OPTS] = False,
-    ssl_key: Annotated[Optional[Path], SSL_KEY_OPTS] = None,
-    ssl_cert: Annotated[Optional[Path], SSL_CERT_OPTS] = None,
-):
-    """Run the SyftBox server"""
-
-    if ctx.invoked_subcommand is not None:
-        return
-
-    # lazy import to improve CLI startup performance
-    import uvicorn
-
-    from syftbox.server.server import app as fastapi_app
+@app.command()
+def migrate():
+    """Run database migrations"""
 
-    uvicorn.run(
-        app=fastapi_app,
-        host="0.0.0.0",
-        port=port,
-        log_level="debug" if verbose else "info",
-        workers=workers,
-        ssl_keyfile=ssl_key,
-        ssl_certfile=ssl_cert,
-    )
+    try:
+        settings = ServerSettings()
+        run_migrations(settings)
+        logger.info("Migrations completed successfully")
+    except Exception as e:
+        logger.error("Migrations failed")
+        logger.exception(e)
+        raise Exit(1)
 
 
 def main():

diff --git a/syftbox/server/migrations.py b/syftbox/server/migrations.py
@@ -0,0 +1,61 @@
+import yaml
+from loguru import logger
+
+from syftbox import __version__
+from syftbox.lib.constants import PERM_FILE
+from syftbox.lib.hash import collect_files, hash_files
+from syftbox.lib.permissions import SyftPermission, migrate_permissions
+from syftbox.server.db import db
+from syftbox.server.db.schema import get_db
+from syftbox.server.server import create_folders
+from syftbox.server.settings import ServerSettings
+
+
+def run_migrations(settings: ServerSettings):
+    logger.info("Creating folders")
+    create_folders(settings.folders)
+    logger.info("Initializing DB")
+    init_db(settings)
+
+
+def init_db(settings: ServerSettings) -> None:
+    # remove this after the upcoming release
+    if __version__ in ["0.2.11", "0.2.12"]:
+        # Delete existing DB to avoid conflicts
+        db_path = settings.file_db_path.absolute()
+        if db_path.exists():
+            db_path.unlink()
+    migrate_permissions(settings.snapshot_folder)
+
+    # might take very long as snapshot folder grows
+    logger.info(f"> Collecting Files from {settings.snapshot_folder.absolute()}")
+    files = collect_files(settings.snapshot_folder.absolute())
+    logger.info("> Hashing files")
+    metadata = hash_files(files, settings.snapshot_folder)
+    logger.info(f"> Updating file hashes at {settings.file_db_path.absolute()}")
+    con = get_db(settings.file_db_path.absolute())
+    cur = con.cursor()
+    for m in metadata:
+        db.save_file_metadata(cur, m)
+
+    # remove files that are not in the snapshot folder
+    all_metadata = db.get_all_metadata(cur)
+    for m in all_metadata:
+        abs_path = settings.snapshot_folder / m.path
+        if not abs_path.exists():
+            logger.info(f"{m.path} not found in {settings.snapshot_folder}, deleting from db")
+            db.delete_file_metadata(cur, m.path.as_posix())
+
+    # fill the permission tables
+    for file in settings.snapshot_folder.rglob(PERM_FILE):
+        content = file.read_text()
+        rule_dicts = yaml.safe_load(content)
+        perm_file = SyftPermission.from_rule_dicts(
+            permfile_file_path=file.relative_to(settings.snapshot_folder), rule_dicts=rule_dicts
+        )
+        db.set_rules_for_permfile(con, perm_file)
+        db.link_existing_rules_to_file(con, file.relative_to(settings.snapshot_folder))
+
+    cur.close()
+    con.commit()
+    con.close()
diff --git a/syftbox/server/server.py b/syftbox/server/server.py
@@ -4,7 +4,6 @@
 from datetime import datetime
 from pathlib import Path
 
-import yaml
 from fastapi import Depends, FastAPI, Request
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import (
@@ -22,8 +21,6 @@
 from typing_extensions import Any, Optional, Union
 
 from syftbox import __version__
-from syftbox.lib.constants import PERM_FILE
-from syftbox.lib.hash import collect_files, hash_files
 from syftbox.lib.http import (
     HEADER_OS_ARCH,
     HEADER_OS_NAME,
@@ -35,10 +32,7 @@
 from syftbox.lib.lib import (
     get_datasites,
 )
-from syftbox.lib.permissions import SyftPermission, migrate_permissions
 from syftbox.server.analytics import log_analytics_event
-from syftbox.server.db import db
-from syftbox.server.db.schema import get_db
 from syftbox.server.logger import setup_logger
 from syftbox.server.middleware import LoguruMiddleware
 from syftbox.server.settings import ServerSettings, get_server_settings
@@ -66,49 +60,6 @@ def create_folders(folders: list[str]) -> None:
             os.makedirs(folder, exist_ok=True)
 
 
-def init_db(settings: ServerSettings) -> None:
-    # remove this after the upcoming release
-    if __version__ in ["0.2.11", "0.2.12"]:
-        # Delete existing DB to avoid conflicts
-        db_path = settings.file_db_path.absolute()
-        if db_path.exists():
-            db_path.unlink()
-    migrate_permissions(settings.snapshot_folder)
-
-    # might take very long as snapshot folder grows
-    logger.info(f"> Collecting Files from {settings.snapshot_folder.absolute()}")
-    files = collect_files(settings.snapshot_folder.absolute())
-    logger.info("> Hashing files")
-    metadata = hash_files(files, settings.snapshot_folder)
-    logger.info(f"> Updating file hashes at {settings.file_db_path.absolute()}")
-    con = get_db(settings.file_db_path.absolute())
-    cur = con.cursor()
-    for m in metadata:
-        db.save_file_metadata(cur, m)
-
-    # remove files that are not in the snapshot folder
-    all_metadata = db.get_all_metadata(cur)
-    for m in all_metadata:
-        abs_path = settings.snapshot_folder / m.path
-        if not abs_path.exists():
-            logger.info(f"{m.path} not found in {settings.snapshot_folder}, deleting from db")
-            db.delete_file_metadata(cur, m.path.as_posix())
-
-    # fill the permission tables
-    for file in settings.snapshot_folder.rglob(PERM_FILE):
-        content = file.read_text()
-        rule_dicts = yaml.safe_load(content)
-        perm_file = SyftPermission.from_rule_dicts(
-            permfile_file_path=file.relative_to(settings.snapshot_folder), rule_dicts=rule_dicts
-        )
-        db.set_rules_for_permfile(con, perm_file)
-        db.link_existing_rules_to_file(con, file.relative_to(settings.snapshot_folder))
-
-    cur.close()
-    con.commit()
-    con.close()
-
-
 def server_request_hook(span: Span, scope: dict[str, Any]):
     if not span.is_recording():
         return
@@ -139,13 +90,6 @@ async def lifespan(app: FastAPI, settings: Optional[ServerSettings] = None):
     else:
         logger.info("OTel Exporter is DISABLED")
 
-    logger.info("Creating Folders")
-    create_folders(settings.folders)
-
-    logger.info("> Loading Users")
-
-    init_db(settings)
-
     yield {
         "server_settings": settings,
     }

diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
@@ -126,10 +126,10 @@ async def start_server(self, server: Server) -> Process:
         env.update(server.env)
 
         process = await asyncio.create_subprocess_exec(
-            "syftbox",
-            "server",
-            "--port",
-            f"{server.port}",
+            "gunicorn",
+            "syftbox.server.server:app",
+            "-k=uvicorn.workers.UvicornWorker",
+            f"--bind=127.0.0.1:{server.port}",
             stdout=open(logs_dir / "server.log", "w"),
             stderr=asyncio.subprocess.STDOUT,
             env=env,

diff --git a/tests/integration/sync/conftest.py b/tests/integration/sync/conftest.py
@@ -11,6 +11,7 @@
 from syftbox.lib.client_config import SyftClientConfig
 from syftbox.lib.datasite import create_datasite
 from syftbox.lib.workspace import SyftWorkspace
+from syftbox.server.migrations import run_migrations
 from syftbox.server.server import app as server_app
 from syftbox.server.server import lifespan as server_lifespan
 from syftbox.server.settings import ServerSettings
@@ -61,9 +62,10 @@ def server_app_with_lifespan(tmp_path: Path) -> FastAPI:
     path.mkdir()
     settings = ServerSettings.from_data_folder(path)
     settings.auth_enabled = False
+    settings.otel_enabled = False
     lifespan_with_settings = partial(server_lifespan, settings=settings)
     server_app.router.lifespan_context = lifespan_with_settings
-
+    run_migrations(settings)
     return server_app
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,4 +14,4 @@ RUN uv pip install --no-cache syftbox==${SYFT_VERSION}

		EXPOSE 8000

		CMD ["uv", "run", "uvicorn", "syftbox.server.server:app", "--host=0.0.0.0", "--port=8000"]
		CMD ["uv", "run", "gunicorn", "syftbox.server.server:app", "--bind=0.0.0.0:8000"]