From 418589d8c9a21ff2a2146cda574a1076b73d996c Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 31 Oct 2024 20:30:07 +0900 Subject: [PATCH 01/91] Feat: Starting upgrading Pydantic using dump-pyndatic Started the migration using the tool dump-pydantic. It only do some parts, but there are mch to do, not finished. Now, have to change and update manually. --- pyproject.toml | 4 ++-- src/aleph/vm/conf.py | 15 ++++++------ src/aleph/vm/controllers/configuration.py | 8 +++---- .../vm/hypervisors/firecracker/config.py | 6 +++-- src/aleph/vm/orchestrator/README.md | 4 ++-- src/aleph/vm/orchestrator/chain.py | 5 ++-- .../vm/orchestrator/views/authentication.py | 23 ++++++++++++------- 7 files changed, 37 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c4efc54a..1005752a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "aioredis==1.3.1", "aiosqlite==0.19", "alembic==1.13.1", - "aleph-message==0.4.9", + "aleph-message @ git+https://github.com/aleph-im/aleph-message#egg=main", "aleph-superfluid~=0.2.1", "dbus-python==1.3.2", "eth-account~=0.10", @@ -47,7 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", - "pydantic[dotenv]~=1.10.13", + "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", "pyyaml==6.0.1", diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index a1737b2b6..013449704 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -13,8 +13,8 @@ from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType -from pydantic import BaseSettings, Field, HttpUrl -from pydantic.env_settings import DotenvType, env_file_sentinel +from pydantic import Field, HttpUrl +from dotenv import load_dotenv from pydantic.typing import StrPath from aleph.vm.orchestrator.chain import STREAM_CHAINS @@ -24,6 +24,9 @@ file_hashes_differ, is_command_available, ) +from pydantic_settings import BaseSettings, SettingsConfigDict + +load_dotenv() logger = logging.getLogger(__name__) @@ -460,7 +463,7 @@ def display(self) -> str: def __init__( self, - _env_file: DotenvType | None = env_file_sentinel, + _env_file: str | Path | None = None, _env_file_encoding: str | None = None, _env_nested_delimiter: str | None = None, _secrets_dir: StrPath | None = None, @@ -489,11 +492,7 @@ def __init__( self.JAILER_BASE_DIR = self.EXECUTION_ROOT / "jailer" if not self.CONFIDENTIAL_SESSION_DIRECTORY: self.CONFIDENTIAL_SESSION_DIRECTORY = self.EXECUTION_ROOT / "sessions" - - class Config: - env_prefix = "ALEPH_VM_" - case_sensitive = False - env_file = ".env" + model_config = SettingsConfigDict(env_prefix="ALEPH_VM_", case_sensitive=False, env_file=".env") def make_db_url(): diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py index da10d8395..0c534241e 100644 --- a/src/aleph/vm/controllers/configuration.py +++ b/src/aleph/vm/controllers/configuration.py @@ -25,25 +25,25 @@ class QemuVMHostVolume(BaseModel): class QemuVMConfiguration(BaseModel): qemu_bin_path: str - cloud_init_drive_path: str | None + cloud_init_drive_path: str | None = None image_path: str monitor_socket_path: Path qmp_socket_path: Path vcpu_count: int mem_size_mb: int - interface_name: str | None + interface_name: str | None = None host_volumes: list[QemuVMHostVolume] class QemuConfidentialVMConfiguration(BaseModel): qemu_bin_path: str - cloud_init_drive_path: str | None + cloud_init_drive_path: str | None = None image_path: str monitor_socket_path: Path qmp_socket_path: Path vcpu_count: int mem_size_mb: int - interface_name: str | None + interface_name: str | None = None host_volumes: list[QemuVMHostVolume] ovmf_path: Path sev_session_file: Path diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index b7e4fc77a..79d99e53d 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -51,9 +51,11 @@ class FirecrackerConfig(BaseModel): boot_source: BootSource drives: list[Drive] machine_config: MachineConfig - vsock: Vsock | None - network_interfaces: list[NetworkInterface] | None + vsock: Vsock | None = None + network_interfaces: list[NetworkInterface] | None = None + # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. class Config: allow_population_by_field_name = True diff --git a/src/aleph/vm/orchestrator/README.md b/src/aleph/vm/orchestrator/README.md index c1d22ea0f..10a0569f5 100644 --- a/src/aleph/vm/orchestrator/README.md +++ b/src/aleph/vm/orchestrator/README.md @@ -80,12 +80,12 @@ cd aleph-vm/ ### 2.e. Install Pydantic -[PyDantic](https://pydantic-docs.helpmanual.io/) +[PyDantic](https://pydantic-docs.helpmanual.io/) is used to parse and validate Aleph messages. ```shell apt install -y --no-install-recommends --no-install-suggests python3-pip -pip3 install pydantic[dotenv] +pip3 install pydantic-dotenv pip3 install 'aleph-message==0.4.9' ``` diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 7321aa458..02e878f99 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, root_validator +from pydantic import model_validator, BaseModel logger = logging.getLogger(__name__) @@ -22,7 +22,8 @@ class ChainInfo(BaseModel): def token(self) -> str | None: return self.super_token or self.standard_token - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_tokens(cls, values): if not values.get("standard_token") and not values.get("super_token"): msg = "At least one of standard_token or super_token must be provided." diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 419662072..023bee487 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -22,7 +22,7 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, root_validator, validator +from pydantic import field_validator, model_validator, BaseModel, ValidationError from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -90,17 +90,20 @@ class SignedPubKeyHeader(BaseModel): signature: bytes payload: bytes - @validator("signature") + @field_validator("signature") + @classmethod def signature_must_be_hex(cls, v: bytes) -> bytes: """Convert the signature from hexadecimal to bytes""" return bytes.fromhex(v.removeprefix(b"0x").decode()) - @validator("payload") + @field_validator("payload") + @classmethod def payload_must_be_hex(cls, v: bytes) -> bytes: """Convert the payload from hexadecimal to bytes""" return bytes.fromhex(v.decode()) - @root_validator(pre=False, skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_expiry(cls, values) -> dict[str, bytes]: """Check that the token has not expired""" payload: bytes = values["payload"] @@ -110,7 +113,8 @@ def check_expiry(cls, values) -> dict[str, bytes]: raise ValueError(msg) return values - @root_validator(pre=False, skip_on_failure=True) + @model_validator(skip_on_failure=True) + @classmethod def check_signature(cls, values) -> dict[str, bytes]: """Check that the signature is valid""" signature: list = values["signature"] @@ -132,7 +136,8 @@ class SignedOperationPayload(BaseModel): path: str # body_sha256: str # disabled since there is no body - @validator("time") + @field_validator("time") + @classmethod def time_is_current(cls, v: datetime.datetime) -> datetime.datetime: """Check that the time is current and the payload is not a replay attack.""" max_past = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(minutes=2) @@ -152,7 +157,8 @@ class SignedOperation(BaseModel): signature: bytes payload: bytes - @validator("signature") + @field_validator("signature") + @classmethod def signature_must_be_hex(cls, v) -> bytes: """Convert the signature from hexadecimal to bytes""" try: @@ -162,7 +168,8 @@ def signature_must_be_hex(cls, v) -> bytes: logger.warning(v) raise error - @validator("payload") + @field_validator("payload") + @classmethod def payload_must_be_hex(cls, v) -> bytes: """Convert the payload from hexadecimal to bytes""" v = bytes.fromhex(v.decode()) From 998a0d488ff771c79d1fceb758e901c48fd6e81b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 31 Oct 2024 11:57:02 +0000 Subject: [PATCH 02/91] fix: Missing type annotation after pydantic update Typing is more strict in pydantic v2 --- pyproject.toml | 1 + src/aleph/vm/conf.py | 76 ++++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1005752a9..3209ad348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", + "pydantic-settings", "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 013449704..7f740d679 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -15,7 +15,7 @@ from aleph_message.models.execution.environment import HypervisorType from pydantic import Field, HttpUrl from dotenv import load_dotenv -from pydantic.typing import StrPath +from pathlib import Path from aleph.vm.orchestrator.chain import STREAM_CHAINS from aleph.vm.utils import ( @@ -125,7 +125,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st class Settings(BaseSettings): - SUPERVISOR_HOST = "127.0.0.1" + SUPERVISOR_HOST: str = "127.0.0.1" SUPERVISOR_PORT: int = 4020 # Public domain name @@ -137,29 +137,29 @@ class Settings(BaseSettings): START_ID_INDEX: int = 4 PREALLOC_VM_COUNT: int = 0 REUSE_TIMEOUT: float = 60 * 60.0 - WATCH_FOR_MESSAGES = True - WATCH_FOR_UPDATES = True + WATCH_FOR_MESSAGES: bool = True + WATCH_FOR_UPDATES: bool = True - API_SERVER = "https://official.aleph.cloud" + API_SERVER: str = "https://official.aleph.cloud" # Connect to the Quad9 VPN provider using their IPv4 and IPv6 addresses. - CONNECTIVITY_IPV4_URL = "https://9.9.9.9/" - CONNECTIVITY_IPV6_URL = "https://[2620:fe::fe]/" - CONNECTIVITY_DNS_HOSTNAME = "example.org" + CONNECTIVITY_IPV4_URL: str = "https://9.9.9.9/" + CONNECTIVITY_IPV6_URL: str = "https://[2620:fe::fe]/" + CONNECTIVITY_DNS_HOSTNAME: str = "example.org" - USE_JAILER = True + USE_JAILER: bool = True # System logs make boot ~2x slower - PRINT_SYSTEM_LOGS = False - IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True - DEBUG_ASYNCIO = False + PRINT_SYSTEM_LOGS: bool = False + IGNORE_TRACEBACK_FROM_DIAGNOSTICS: bool = True + DEBUG_ASYNCIO: bool = False # Networking does not work inside Docker/Podman - ALLOW_VM_NETWORKING = True + ALLOW_VM_NETWORKING: bool = True NETWORK_INTERFACE: str | None = None - IPV4_ADDRESS_POOL = Field( + IPV4_ADDRESS_POOL: str = Field( default="172.16.0.0/12", description="IPv4 address range used to provide networks to VMs.", ) - IPV4_NETWORK_PREFIX_LENGTH = Field( + IPV4_NETWORK_PREFIX_LENGTH: int = Field( default=24, description="Individual VM network prefix length in bits", ) @@ -177,7 +177,7 @@ class Settings(BaseSettings): default=True, description="Enable IPv6 forwarding on the host. Required for IPv6 connectivity in VMs.", ) - NFTABLES_CHAIN_PREFIX = "aleph" + NFTABLES_CHAIN_PREFIX: str = "aleph" USE_NDP_PROXY: bool = Field( default=True, description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6", @@ -186,15 +186,15 @@ class Settings(BaseSettings): DNS_RESOLUTION: DnsResolver | None = DnsResolver.detect DNS_NAMESERVERS: list[str] | None = None - FIRECRACKER_PATH = Path("/opt/firecracker/firecracker") - JAILER_PATH = Path("/opt/firecracker/jailer") - SEV_CTL_PATH = Path("/opt/sevctl") - LINUX_PATH = Path("/opt/firecracker/vmlinux.bin") + FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") + JAILER_PATH: Path = Path("/opt/firecracker/jailer") + SEV_CTL_PATH: Path = Path("/opt/sevctl") + LINUX_PATH: Path = Path("/opt/firecracker/vmlinux.bin") INIT_TIMEOUT: float = 20.0 - CONNECTOR_URL = Url("http://localhost:4021") + CONNECTOR_URL: HttpUrl = HttpUrl("http://localhost:4021") - CACHE_ROOT = Path("/var/cache/aleph/vm") + CACHE_ROOT: Path = Path("/var/cache/aleph/vm") MESSAGE_CACHE: Path = Field( None, description="Default to CACHE_ROOT/message", @@ -203,12 +203,12 @@ class Settings(BaseSettings): RUNTIME_CACHE: Path = Field(None, description="Default to CACHE_ROOT/runtime") DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data") - EXECUTION_ROOT = Path("/var/lib/aleph/vm") + EXECUTION_ROOT: Path = Path("/var/lib/aleph/vm") JAILER_BASE_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/jailer") EXECUTION_DATABASE: Path = Field( None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3" ) - EXECUTION_LOG_ENABLED = False + EXECUTION_LOG_ENABLED: bool = False EXECUTION_LOG_DIRECTORY: Path = Field( None, description="Location of executions log. Default to EXECUTION_ROOT/executions/" ) @@ -218,8 +218,8 @@ class Settings(BaseSettings): ) JAILER_BASE_DIR: Path = Field(None) - MAX_PROGRAM_ARCHIVE_SIZE = 10_000_000 # 10 MB - MAX_DATA_ARCHIVE_SIZE = 10_000_000 # 10 MB + MAX_PROGRAM_ARCHIVE_SIZE: int = 10_000_000 # 10 MB + MAX_DATA_ARCHIVE_SIZE: int = 10_000_000 # 10 MB PAYMENT_MONITOR_INTERVAL: float = Field( default=60.0, @@ -260,7 +260,7 @@ class Settings(BaseSettings): ) # hashlib.sha256(b"secret-token").hexdigest() - ALLOCATION_TOKEN_HASH = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda" + ALLOCATION_TOKEN_HASH: str = "151ba92f2eb90bce67e912af2f7a5c17d8654b3d29895b042107ea312a7eebda" ENABLE_QEMU_SUPPORT: bool = Field(default=True) INSTANCE_DEFAULT_HYPERVISOR: HypervisorType | None = Field( @@ -284,11 +284,11 @@ class Settings(BaseSettings): # Tests on programs FAKE_DATA_PROGRAM: Path | None = None - BENCHMARK_FAKE_DATA_PROGRAM = Path(abspath(join(__file__, "../../../../examples/example_fastapi"))) + BENCHMARK_FAKE_DATA_PROGRAM: Path = Path(abspath(join(__file__, "../../../../examples/example_fastapi"))) - FAKE_DATA_MESSAGE = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json"))) + FAKE_DATA_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json"))) FAKE_DATA_DATA: Path | None = Path(abspath(join(__file__, "../../../../examples/data/"))) - FAKE_DATA_RUNTIME = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs"))) + FAKE_DATA_RUNTIME: Path = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs"))) FAKE_DATA_VOLUME: Path | None = Path(abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs"))) # Tests on instances @@ -298,19 +298,19 @@ class Settings(BaseSettings): description="Identifier of the instance message used when testing the launch of an instance from the network", ) - USE_FAKE_INSTANCE_BASE = False - FAKE_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/debian-12.btrfs"))) - FAKE_QEMU_INSTANCE_BASE = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img"))) + USE_FAKE_INSTANCE_BASE: bool = False + FAKE_INSTANCE_BASE: Path = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/debian-12.btrfs"))) + FAKE_QEMU_INSTANCE_BASE: Path = Path(abspath(join(__file__, "../../../../runtimes/instance-rootfs/rootfs.img"))) FAKE_INSTANCE_ID: str = Field( default="decadecadecadecadecadecadecadecadecadecadecadecadecadecadecadeca", description="Identifier used for the 'fake instance' message defined in " "examples/instance_message_from_aleph.json", ) - FAKE_INSTANCE_MESSAGE = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json"))) - FAKE_INSTANCE_QEMU_MESSAGE = Path(abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json"))) + FAKE_INSTANCE_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json"))) + FAKE_INSTANCE_QEMU_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json"))) - CHECK_FASTAPI_VM_ID = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace" - LEGACY_CHECK_FASTAPI_VM_ID = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8" + CHECK_FASTAPI_VM_ID: str = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace" + LEGACY_CHECK_FASTAPI_VM_ID: str = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8" # Developer options @@ -466,7 +466,7 @@ def __init__( _env_file: str | Path | None = None, _env_file_encoding: str | None = None, _env_nested_delimiter: str | None = None, - _secrets_dir: StrPath | None = None, + _secrets_dir: Path | None = None, **values: Any, ) -> None: super().__init__(_env_file, _env_file_encoding, _env_nested_delimiter, _secrets_dir, **values) From ea51db419f96c26f89606c0088b97758e8102089 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Thu, 31 Oct 2024 16:37:26 +0100 Subject: [PATCH 03/91] fix settings --- pyproject.toml | 1 - src/aleph/vm/conf.py | 28 +++++++++++++------ src/aleph/vm/orchestrator/chain.py | 2 +- src/aleph/vm/orchestrator/tasks.py | 3 +- .../vm/orchestrator/views/authentication.py | 6 ++-- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3209ad348..99e68031c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,7 +200,6 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] # # Don't touch unused imports # "F401", #] -lint.isort = [ "aleph.vm" ] [tool.pytest.ini_options] pythonpath = [ diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 7f740d679..45d2d611a 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -13,9 +13,9 @@ from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType -from pydantic import Field, HttpUrl from dotenv import load_dotenv -from pathlib import Path +from pydantic import Field, HttpUrl +from pydantic_settings import BaseSettings, SettingsConfigDict from aleph.vm.orchestrator.chain import STREAM_CHAINS from aleph.vm.utils import ( @@ -24,7 +24,6 @@ file_hashes_differ, is_command_available, ) -from pydantic_settings import BaseSettings, SettingsConfigDict load_dotenv() @@ -288,7 +287,9 @@ class Settings(BaseSettings): FAKE_DATA_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/program_message_from_aleph.json"))) FAKE_DATA_DATA: Path | None = Path(abspath(join(__file__, "../../../../examples/data/"))) - FAKE_DATA_RUNTIME: Path = Path(abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs"))) + FAKE_DATA_RUNTIME: Path = Path( + abspath(join(__file__, "../../../../runtimes/aleph-debian-12-python/rootfs.squashfs")) + ) FAKE_DATA_VOLUME: Path | None = Path(abspath(join(__file__, "../../../../examples/volumes/volume-venv.squashfs"))) # Tests on instances @@ -307,7 +308,9 @@ class Settings(BaseSettings): "examples/instance_message_from_aleph.json", ) FAKE_INSTANCE_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/instance_message_from_aleph.json"))) - FAKE_INSTANCE_QEMU_MESSAGE: Path = Path(abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json"))) + FAKE_INSTANCE_QEMU_MESSAGE: Path = Path( + abspath(join(__file__, "../../../../examples/qemu_message_from_aleph.json")) + ) CHECK_FASTAPI_VM_ID: str = "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace" LEGACY_CHECK_FASTAPI_VM_ID: str = "67705389842a0a1b95eaa408b009741027964edc805997475e95c505d642edd8" @@ -343,7 +346,7 @@ def check(self): assert isfile(self.JAILER_PATH), f"File not found {self.JAILER_PATH}" assert isfile(self.LINUX_PATH), f"File not found {self.LINUX_PATH}" assert self.NETWORK_INTERFACE, "Network interface is not specified" - assert self.CONNECTOR_URL.startswith("http://") or self.CONNECTOR_URL.startswith("https://") + assert str(self.CONNECTOR_URL).startswith("http://") or str(self.CONNECTOR_URL).startswith("https://") if self.ALLOW_VM_NETWORKING: assert exists( f"/sys/class/net/{self.NETWORK_INTERFACE}" @@ -469,7 +472,13 @@ def __init__( _secrets_dir: Path | None = None, **values: Any, ) -> None: - super().__init__(_env_file, _env_file_encoding, _env_nested_delimiter, _secrets_dir, **values) + super().__init__( + _env_file, + _env_file_encoding, + _env_nested_delimiter, + _secrets_dir, + **values, + ) if not self.MESSAGE_CACHE: self.MESSAGE_CACHE = self.CACHE_ROOT / "message" if not self.CODE_CACHE: @@ -492,7 +501,10 @@ def __init__( self.JAILER_BASE_DIR = self.EXECUTION_ROOT / "jailer" if not self.CONFIDENTIAL_SESSION_DIRECTORY: self.CONFIDENTIAL_SESSION_DIRECTORY = self.EXECUTION_ROOT / "sessions" - model_config = SettingsConfigDict(env_prefix="ALEPH_VM_", case_sensitive=False, env_file=".env") + + model_config = SettingsConfigDict( + env_prefix="ALEPH_VM_", case_sensitive=False, env_file=".env", validate_default=False + ) def make_db_url(): diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 02e878f99..14bde4759 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import model_validator, BaseModel +from pydantic import BaseModel, model_validator logger = logging.getLogger(__name__) diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index c7062d931..921a2265f 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -11,7 +11,6 @@ from aiohttp import web from aleph_message.models import ( AlephMessage, - ItemHash, PaymentType, ProgramMessage, parse_message, @@ -23,7 +22,7 @@ from aleph.vm.pool import VmPool from aleph.vm.utils import create_task_log_exceptions -from .messages import get_message_status, load_updated_message +from .messages import get_message_status from .payment import ( compute_required_balance, compute_required_flow, diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 023bee487..40ab4a39d 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -22,7 +22,7 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import field_validator, model_validator, BaseModel, ValidationError +from pydantic import BaseModel, ValidationError, field_validator, model_validator from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -102,7 +102,7 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: """Convert the payload from hexadecimal to bytes""" return bytes.fromhex(v.decode()) - @model_validator(skip_on_failure=True) + @model_validator(mode="after") @classmethod def check_expiry(cls, values) -> dict[str, bytes]: """Check that the token has not expired""" @@ -113,7 +113,7 @@ def check_expiry(cls, values) -> dict[str, bytes]: raise ValueError(msg) return values - @model_validator(skip_on_failure=True) + @model_validator(mode="after") @classmethod def check_signature(cls, values) -> dict[str, bytes]: """Check that the signature is valid""" From d17a2ee9fa5f21c20743f3cb803b9402a5610dc7 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:55:00 +0000 Subject: [PATCH 04/91] Fix: error handling in `get_signed_pubkey` for improved JSON and validation support Validation was failing with Pydantic v2 due to stricter date formats and unhandled JSON errors. Adapted the exceptions so we handle format/validation with changes bringed by pydantic v2 --- .../vm/orchestrator/views/authentication.py | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 40ab4a39d..21c7a6ac1 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -106,8 +106,8 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: @classmethod def check_expiry(cls, values) -> dict[str, bytes]: """Check that the token has not expired""" - payload: bytes = values["payload"] - content = SignedPubKeyPayload.parse_raw(payload) + payload: bytes = values.payload + content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): msg = "Token expired" raise ValueError(msg) @@ -117,16 +117,16 @@ def check_expiry(cls, values) -> dict[str, bytes]: @classmethod def check_signature(cls, values) -> dict[str, bytes]: """Check that the signature is valid""" - signature: list = values["signature"] - payload: bytes = values["payload"] - content = SignedPubKeyPayload.parse_raw(payload) + signature: list = values.signature + payload: bytes = values.payload + content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values @property def content(self) -> SignedPubKeyPayload: """Return the content of the header""" - return SignedPubKeyPayload.parse_raw(self.payload) + return SignedPubKeyPayload.model_validate_json(self.payload) class SignedOperationPayload(BaseModel): @@ -173,13 +173,13 @@ def signature_must_be_hex(cls, v) -> bytes: def payload_must_be_hex(cls, v) -> bytes: """Convert the payload from hexadecimal to bytes""" v = bytes.fromhex(v.decode()) - _ = SignedOperationPayload.parse_raw(v) + _ = SignedOperationPayload.model_validate_json(v) return v @property def content(self) -> SignedOperationPayload: """Return the content of the header""" - return SignedOperationPayload.parse_raw(self.payload) + return SignedOperationPayload.model_validate_json(self.payload) def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader: @@ -189,29 +189,30 @@ def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader: raise web.HTTPBadRequest(reason="Missing X-SignedPubKey header") try: - return SignedPubKeyHeader.parse_raw(signed_pubkey_header) + data = json.loads(signed_pubkey_header) + if "expires" in data and isinstance(data["expires"], float): + data["expires"] = str(data["expires"]) + return SignedPubKeyHeader.model_validate_json(json.dumps(data)) except KeyError as error: logger.debug(f"Missing X-SignedPubKey header: {error}") raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey fields") from error except json.JSONDecodeError as error: raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from error - except ValueError as errors: + except ValidationError as errors: logging.debug(errors) - for err in errors.args[0]: - if isinstance(err.exc, json.JSONDecodeError): - raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from errors - if str(err.exc) == "Token expired": + for err in errors.errors(): + if err["type"] == "value_error" and "Token expired" in str(err["msg"]): raise web.HTTPUnauthorized(reason="Token expired") from errors - if str(err.exc) == "Invalid signature": + elif err["type"] == "value_error" and "Invalid signature" in str(err["msg"]): raise web.HTTPUnauthorized(reason="Invalid signature") from errors - raise errors + raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey data") def get_signed_operation(request: web.Request) -> SignedOperation: """Get the signed operation public key that is signed by the ephemeral key from the request headers.""" try: signed_operation = request.headers["X-SignedOperation"] - return SignedOperation.parse_raw(signed_operation) + return SignedOperation.model_validate_json(signed_operation) except KeyError as error: raise web.HTTPBadRequest(reason="Missing X-SignedOperation header") from error except json.JSONDecodeError as error: @@ -255,8 +256,8 @@ async def authenticate_websocket_message(message) -> str: """Authenticate a websocket message since JS cannot configure headers on WebSockets.""" if not isinstance(message, dict): raise Exception("Invalid format for auth packet, see /doc/operator_auth.md") - signed_pubkey = SignedPubKeyHeader.parse_obj(message["X-SignedPubKey"]) - signed_operation = SignedOperation.parse_obj(message["X-SignedOperation"]) + signed_pubkey = SignedPubKeyHeader.model_validate(message["X-SignedPubKey"]) + signed_operation = SignedOperation.model_validate(message["X-SignedOperation"]) if signed_operation.content.domain != settings.DOMAIN_NAME: logger.debug(f"Invalid domain '{signed_operation.content.domain}' != '{settings.DOMAIN_NAME}'") raise web.HTTPUnauthorized(reason="Invalid domain") From 543b9f4801b128512d35f2846b429f7b1f8a64cb Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:57:40 +0000 Subject: [PATCH 05/91] Fix: Migration to pydantic 2 rose some warnings The migration to Pydantic 2 bringed some warnings and deprecated funtions. Fixing them all manually --- pyproject.toml | 2 +- src/aleph/vm/controllers/__main__.py | 2 +- src/aleph/vm/controllers/configuration.py | 2 +- src/aleph/vm/controllers/qemu/instance.py | 2 +- src/aleph/vm/guest_api/__main__.py | 4 ++-- src/aleph/vm/hypervisors/firecracker/config.py | 13 +++++-------- src/aleph/vm/hypervisors/firecracker/microvm.py | 2 +- src/aleph/vm/models.py | 10 +++++----- src/aleph/vm/orchestrator/messages.py | 2 +- src/aleph/vm/orchestrator/payment.py | 6 +++--- src/aleph/vm/orchestrator/reactor.py | 2 +- src/aleph/vm/orchestrator/resources.py | 2 +- src/aleph/vm/orchestrator/run.py | 2 +- src/aleph/vm/orchestrator/status.py | 4 ++-- src/aleph/vm/orchestrator/tasks.py | 2 +- src/aleph/vm/orchestrator/views/__init__.py | 4 ++-- src/aleph/vm/orchestrator/views/operator.py | 6 +++--- src/aleph/vm/storage.py | 2 +- src/aleph/vm/utils/__init__.py | 4 ++-- vm_connector/conf.py | 10 +++++----- 20 files changed, 40 insertions(+), 43 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 99e68031c..22649346a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "aioredis==1.3.1", "aiosqlite==0.19", "alembic==1.13.1", - "aleph-message @ git+https://github.com/aleph-im/aleph-message#egg=main", + "aleph-message @ git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=main", "aleph-superfluid~=0.2.1", "dbus-python==1.3.2", "eth-account~=0.10", diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py index 519270b48..a8fe43399 100644 --- a/src/aleph/vm/controllers/__main__.py +++ b/src/aleph/vm/controllers/__main__.py @@ -26,7 +26,7 @@ def configuration_from_file(path: Path): with open(path) as f: data = json.load(f) - return Configuration.parse_obj(data) + return Configuration.model_validate(data) def parse_args(args): diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py index 0c534241e..a5ce7e554 100644 --- a/src/aleph/vm/controllers/configuration.py +++ b/src/aleph/vm/controllers/configuration.py @@ -69,7 +69,7 @@ def save_controller_configuration(vm_hash: str, configuration: Configuration) -> config_file_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json") with config_file_path.open("w") as controller_config_file: controller_config_file.write( - configuration.json( + configuration.model_dump_json( by_alias=True, exclude_none=True, indent=4, exclude={"settings": {"USE_DEVELOPER_SSH_KEYS"}} ) ) diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py index dd840e22b..e92917683 100644 --- a/src/aleph/vm/controllers/qemu/instance.py +++ b/src/aleph/vm/controllers/qemu/instance.py @@ -215,7 +215,7 @@ async def configure(self): def save_controller_configuration(self): """Save VM configuration to be used by the controller service""" path = Path(f"{settings.EXECUTION_ROOT}/{self.vm_hash}-controller.json") - path.open("w").write(self.controller_configuration.json(by_alias=True, exclude_none=True, indent=4)) + path.open("w").write(self.controller_configuration.model_dump_json(by_alias=True, exclude_none=True, indent=4)) path.chmod(0o644) return path diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py index 8000d52bc..dea3db2a9 100644 --- a/src/aleph/vm/guest_api/__main__.py +++ b/src/aleph/vm/guest_api/__main__.py @@ -43,7 +43,7 @@ async def proxy(request: web.Request): async def repost(request: web.Request): logger.debug("REPOST") - data_raw = await request.json() + data_raw = await request.model_dump_json() topic, message = data_raw["topic"], json.loads(data_raw["data"]) content = json.loads(message["item_content"]) @@ -82,7 +82,7 @@ async def properties(request: web.Request): async def sign(request: web.Request): vm_hash = request.app["meta_vm_hash"] - message = await request.json() + message = await request.model_dump_json() # Ensure that the hash of the VM is used as sending address content = json.loads(message["item_content"]) diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index 79d99e53d..5b35095c5 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import BaseModel, PositiveInt +from pydantic import BaseModel, PositiveInt, ConfigDict VSOCK_PATH = "/tmp/v.sock" @@ -54,11 +54,8 @@ class FirecrackerConfig(BaseModel): vsock: Vsock | None = None network_interfaces: list[NetworkInterface] | None = None - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - allow_population_by_field_name = True + model_config = ConfigDict( + populate_by_name = True, + alias_generator=lambda x: x.replace("_", "-") + ) - @staticmethod - def alias_generator(x: str): - return x.replace("_", "-") diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py index 7a8fe787e..4421491e5 100644 --- a/src/aleph/vm/hypervisors/firecracker/microvm.py +++ b/src/aleph/vm/hypervisors/firecracker/microvm.py @@ -193,7 +193,7 @@ async def save_configuration_file(self, config: FirecrackerConfig) -> Path: if not self.use_jailer else open(f"{self.jailer_path}/tmp/config.json", "wb") ) as config_file: - config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file.write(config.model_dump_json(by_alias=True, exclude_none=True, indent=4).encode()) config_file.flush() config_file_path = Path(config_file.name) config_file_path.chmod(0o644) diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 35de4076d..57089f3be 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -407,8 +407,8 @@ async def save(self): vcpus=self.vm.hardware_resources.vcpus, memory=self.vm.hardware_resources.memory, network_tap=self.vm.tap_interface.device_name if self.vm.tap_interface else "", - message=self.message.json(), - original_message=self.original.json(), + message=self.message.model_dump_json(), + original_message=self.original.model_dump_json(), persistent=self.persistent, ) ) @@ -431,8 +431,8 @@ async def save(self): io_write_bytes=None, vcpus=self.vm.hardware_resources.vcpus, memory=self.vm.hardware_resources.memory, - message=self.message.json(), - original_message=self.original.json(), + message=self.message.model_dump_json(), + original_message=self.original.model_dump_json(), persistent=self.persistent, ) ) @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py index 5ae67102c..c05303f11 100644 --- a/src/aleph/vm/orchestrator/messages.py +++ b/src/aleph/vm/orchestrator/messages.py @@ -85,5 +85,5 @@ async def get_message_status(item_hash: ItemHash) -> MessageStatus: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.json() + resp_data = await resp.model_dump_json() return resp_data["status"] diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py index 7194f873a..71181c361 100644 --- a/src/aleph/vm/orchestrator/payment.py +++ b/src/aleph/vm/orchestrator/payment.py @@ -40,7 +40,7 @@ async def fetch_balance_of_address(address: str) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.json() + resp_data = await resp.model_dump_json() return resp_data["balance"] @@ -52,7 +52,7 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.json() + resp_data = await resp.model_dump_json() required_flow: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] @@ -74,7 +74,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.json() + resp_data = await resp.model_dump_json() required_hold: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py index 785f2c233..f8326fa97 100644 --- a/src/aleph/vm/orchestrator/reactor.py +++ b/src/aleph/vm/orchestrator/reactor.py @@ -61,7 +61,7 @@ async def trigger(self, message: AlephMessage): for subscription in listener.content.on.message: if subscription_matches(subscription, message): vm_hash = listener.item_hash - event = message.json() + event = message.model_dump_json() # Register the listener in the list of coroutines to run asynchronously: coroutines.append(run_code_on_event(vm_hash, event, self.pubsub, pool=self.pool)) break diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py index d4b9c8985..23fcbf1a5 100644 --- a/src/aleph/vm/orchestrator/resources.py +++ b/src/aleph/vm/orchestrator/resources.py @@ -134,7 +134,7 @@ async def about_system_usage(_: web.Request): properties=get_machine_properties(), ) - return web.json_response(text=usage.json(exclude_none=True)) + return web.json_response(text=usage.model_dump_json(exclude_none=True)) @cors_allow_all diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index a2a2a824f..376e73f6f 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -55,7 +55,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {message.model_dump_json(indent=4, sort_keys=True, exclude_none=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py index 12692f6a9..db0950240 100644 --- a/src/aleph/vm/orchestrator/status.py +++ b/src/aleph/vm/orchestrator/status.py @@ -26,7 +26,7 @@ async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) url = f"{vm_url}{suffix}" async with session.get(url) as resp: resp.raise_for_status() - return await resp.json() + return await resp.model_dump_json() async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: Any = None) -> Any: @@ -35,7 +35,7 @@ async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: url = f"{vm_url}{suffix}" async with session.post(url, json=data) as resp: resp.raise_for_status() - return await resp.json() + return await resp.model_dump_json() async def check_index(session: ClientSession, vm_id: ItemHash) -> bool: diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 921a2265f..996584497 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -76,7 +76,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: except pydantic.error_wrappers.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", exc_info=False, ) continue diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py index 4bba01aa8..603377359 100644 --- a/src/aleph/vm/orchestrator/views/__init__.py +++ b/src/aleph/vm/orchestrator/views/__init__.py @@ -372,7 +372,7 @@ async def update_allocations(request: web.Request): try: data = await request.json() - allocation = Allocation.parse_obj(data) + allocation = Allocation.model_validate(data) except ValidationError as error: return web.json_response(text=error.json(), status=web.HTTPBadRequest.status_code) @@ -456,7 +456,7 @@ async def notify_allocation(request: web.Request): """Notify instance allocation, only used for Pay as you Go feature""" try: data = await request.json() - vm_notification = VMNotification.parse_obj(data) + vm_notification = VMNotification.model_validate(data) except JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except ValidationError as error: diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py index af0e98f45..b40d87803 100644 --- a/src/aleph/vm/orchestrator/views/operator.py +++ b/src/aleph/vm/orchestrator/views/operator.py @@ -322,12 +322,12 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated Send secret to the VM and start it """ try: - data = await request.json() - params = InjectSecretParams.parse_obj(data) + data = await request.model_dump_json() + params = InjectSecretParams.model_validate(data) except json.JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except pydantic.ValidationError as error: - return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) + return web.json_response(data=error.model_dump_json(), status=web.HTTPBadRequest.status_code) vm_hash = get_itemhash_or_400(request.match_info) pool: VmPool = request.app["vm_pool"] diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 7e289dca2..1366b3310 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -136,7 +136,7 @@ async def get_latest_amend(item_hash: str) -> str: async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() - result: str = await resp.json() + result: str = await resp.model_dump_json() assert isinstance(result, str) return result or item_hash diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py index d8eecad95..9bf30d001 100644 --- a/src/aleph/vm/utils/__init__.py +++ b/src/aleph/vm/utils/__init__.py @@ -24,9 +24,9 @@ def get_message_executable_content(message_dict: dict) -> ExecutableContent: try: - return ProgramContent.parse_obj(message_dict) + return ProgramContent.model_validate(message_dict) except ValueError: - return InstanceContent.parse_obj(message_dict) + return InstanceContent.model_validate(message_dict) def cors_allow_all(function): diff --git a/vm_connector/conf.py b/vm_connector/conf.py index d2ee465fc..d0b34eab1 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -1,7 +1,7 @@ import logging from typing import NewType -from pydantic import BaseSettings +from pydantic import BaseSettings, ConfigDict logger = logging.getLogger(__name__) @@ -27,11 +27,11 @@ def display(self) -> str: f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items() ) - class Config: - env_prefix = "ALEPH_" - case_sensitive = False + model_config = ConfigDict( + env_prefix = "ALEPH_", + case_sensitive = False, env_file = ".env" - + ) # Settings singleton settings = ConnectorSettings() From c5bd2cd4d606940b13c22e65b74fb7bb177c55a0 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:00:48 +0000 Subject: [PATCH 06/91] Fix: Pydantic 2 is strict with typing annotation The return value for url was string but now pydantic is strict and need a good type Changing str to HttpUrl for url --- src/aleph/vm/orchestrator/chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 14bde4759..3dbe4ad3c 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, model_validator, HttpUrl logger = logging.getLogger(__name__) @@ -12,7 +12,7 @@ class ChainInfo(BaseModel): """ chain_id: int - rpc: str + rpc: HttpUrl standard_token: str | None = None super_token: str | None = None testnet: bool = False From 64b93d14dbf80bdb5a3f3582b8641ba86c18f682 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:02:32 +0000 Subject: [PATCH 07/91] Fix: Adapt test mocks to Pydantic v2 by replacing `.json` with `.model_dump_json` In Pydantic v2, the `.json()` method has been replaced with `.model_dump_json()` for serializing json Updated the mock in `test_check_internet_wrong_result_code` to use `.model_dump_json` instead of `.json` to align with this change. --- tests/supervisor/test_status.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py index 0e0449dbf..f212fc478 100644 --- a/tests/supervisor/test_status.py +++ b/tests/supervisor/test_status.py @@ -13,12 +13,13 @@ async def test_check_internet_wrong_result_code(): mock_session = Mock() mock_session.get = MagicMock() - mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( return_value={"result": 200, "headers": {"Server": "nginx"}} ) + assert await check_internet(mock_session, vm_id) is True - mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( return_value={"result": 400, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is False From 2d6605143ec0f7a59691828e52a979bbeaa7894b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:03:50 +0000 Subject: [PATCH 08/91] Fix: Pydantic 2 add more information in return error Pydantic 2 add more fields while returning error such as `url`, `ctx` or `input` Adapting the test to the new format given by pydantic --- tests/supervisor/test_views.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index fff8b5492..613560db3 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -10,7 +10,6 @@ from aleph.vm.orchestrator.supervisor import setup_webapp from aleph.vm.sevclient import SevClient - @pytest.mark.asyncio async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): """Test that the allocation endpoint fails when an invalid item_hash is provided.""" @@ -20,19 +19,22 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): response: web.Response = await client.post( "/control/allocations", json={"persistent_vms": ["not-an-ItemHash"]}, headers={"X-Auth-Signature": "test"} ) + assert response.status == 400 + assert await response.json() == [ { - "loc": [ - "persistent_vms", - 0, - ], - "msg": "Could not determine hash type: 'not-an-ItemHash'", - "type": "value_error.unknownhash", + "loc": ["persistent_vms", 0], + "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", + "type": "value_error", + "ctx": { + "error": "Could not determine hash type: 'not-an-ItemHash'" + }, + "input": "not-an-ItemHash", + "url": "https://errors.pydantic.dev/2.9/v/value_error" }, ] - @pytest.mark.asyncio async def test_system_usage(aiohttp_client): """Test that the usage system endpoints responds. No auth needed""" From 9ffdbb208771ff67605af71ef21144be73964f8f Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:11:43 +0000 Subject: [PATCH 09/91] Style: black, isort and mypy --- src/aleph/vm/hypervisors/firecracker/config.py | 8 ++------ src/aleph/vm/orchestrator/chain.py | 2 +- tests/supervisor/test_views.py | 8 ++++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index 5b35095c5..59560ce1c 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import BaseModel, PositiveInt, ConfigDict +from pydantic import BaseModel, ConfigDict, PositiveInt VSOCK_PATH = "/tmp/v.sock" @@ -54,8 +54,4 @@ class FirecrackerConfig(BaseModel): vsock: Vsock | None = None network_interfaces: list[NetworkInterface] | None = None - model_config = ConfigDict( - populate_by_name = True, - alias_generator=lambda x: x.replace("_", "-") - ) - + model_config = ConfigDict(populate_by_name=True, alias_generator=lambda x: x.replace("_", "-")) diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 3dbe4ad3c..814abe05c 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, model_validator, HttpUrl +from pydantic import BaseModel, HttpUrl, model_validator logger = logging.getLogger(__name__) diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index 613560db3..3e9c4282e 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -10,6 +10,7 @@ from aleph.vm.orchestrator.supervisor import setup_webapp from aleph.vm.sevclient import SevClient + @pytest.mark.asyncio async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): """Test that the allocation endpoint fails when an invalid item_hash is provided.""" @@ -27,14 +28,13 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): "loc": ["persistent_vms", 0], "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", "type": "value_error", - "ctx": { - "error": "Could not determine hash type: 'not-an-ItemHash'" - }, + "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, "input": "not-an-ItemHash", - "url": "https://errors.pydantic.dev/2.9/v/value_error" + "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] + @pytest.mark.asyncio async def test_system_usage(aiohttp_client): """Test that the usage system endpoints responds. No auth needed""" From bf26b1bf8d713a3173a372446eb912e4f270306c Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:27:06 +0000 Subject: [PATCH 10/91] Style: black n vm_connector --- vm_connector/conf.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vm_connector/conf.py b/vm_connector/conf.py index d0b34eab1..8164c0320 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -27,11 +27,8 @@ def display(self) -> str: f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items() ) - model_config = ConfigDict( - env_prefix = "ALEPH_", - case_sensitive = False, - env_file = ".env" - ) + model_config = ConfigDict(env_prefix="ALEPH_", case_sensitive=False, env_file=".env") + # Settings singleton settings = ConnectorSettings() From 37363d89eb8ed9f81f7a3ac155cae96e485547e1 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:53:21 +0000 Subject: [PATCH 11/91] Style: black pyproject --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22649346a..e0721e0dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,10 +48,10 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic-settings", - "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", "pyyaml==6.0.1", + "python-dotevn", "qmp==1.1", "schedule==1.2.1", "sentry-sdk==1.31", @@ -192,8 +192,6 @@ lint.ignore = [ # Allow the use of assert statements "S101", ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] #[tool.ruff.flake8-tidy-imports] #ban-relative-imports = "all" #unfixable = [ @@ -201,6 +199,9 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] # "F401", #] +# Tests can use magic values, assertions, and relative imports +lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] + [tool.pytest.ini_options] pythonpath = [ "src", From aac48cc4f9a545ee947adc90e196f51cfa91b52d Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 22:02:43 +0000 Subject: [PATCH 12/91] Style: black pyproject --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e0721e0dd..783d98950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,8 @@ dependencies = [ "pydantic-settings", "pyroute2==0.7.12", "python-cpuid==0.1", + "python-dotenv", "pyyaml==6.0.1", - "python-dotevn", "qmp==1.1", "schedule==1.2.1", "sentry-sdk==1.31", From 1a22d1ee77aaa02f96c4f30a927db148a1858d70 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 8 Nov 2024 06:52:02 +0000 Subject: [PATCH 13/91] style: Mypy detected some error about model_dump_json --- src/aleph/vm/guest_api/__main__.py | 4 ++-- src/aleph/vm/orchestrator/chain.py | 8 ++++---- src/aleph/vm/orchestrator/messages.py | 2 +- src/aleph/vm/orchestrator/payment.py | 6 +++--- src/aleph/vm/orchestrator/run.py | 2 +- src/aleph/vm/orchestrator/status.py | 4 ++-- src/aleph/vm/orchestrator/views/operator.py | 4 ++-- src/aleph/vm/storage.py | 2 +- tests/supervisor/test_status.py | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py index dea3db2a9..8000d52bc 100644 --- a/src/aleph/vm/guest_api/__main__.py +++ b/src/aleph/vm/guest_api/__main__.py @@ -43,7 +43,7 @@ async def proxy(request: web.Request): async def repost(request: web.Request): logger.debug("REPOST") - data_raw = await request.model_dump_json() + data_raw = await request.json() topic, message = data_raw["topic"], json.loads(data_raw["data"]) content = json.loads(message["item_content"]) @@ -82,7 +82,7 @@ async def properties(request: web.Request): async def sign(request: web.Request): vm_hash = request.app["meta_vm_hash"] - message = await request.model_dump_json() + message = await request.json() # Ensure that the hash of the VM is used as sending address content = json.loads(message["item_content"]) diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 814abe05c..10cf1211e 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -35,7 +35,7 @@ def check_tokens(cls, values): # TESTNETS "SEPOLIA": ChainInfo( chain_id=11155111, - rpc="https://eth-sepolia.public.blastapi.io", + rpc=HttpUrl("https://eth-sepolia.public.blastapi.io"), standard_token="0xc4bf5cbdabe595361438f8c6a187bdc330539c60", super_token="0x22064a21fee226d8ffb8818e7627d5ff6d0fc33a", active=False, @@ -44,18 +44,18 @@ def check_tokens(cls, values): # MAINNETS Chain.ETH: ChainInfo( chain_id=1, - rpc="https://eth-mainnet.public.blastapi.io", + rpc=HttpUrl("https://eth-mainnet.public.blastapi.io"), standard_token="0x27702a26126e0B3702af63Ee09aC4d1A084EF628", active=False, ), Chain.AVAX: ChainInfo( chain_id=43114, - rpc="https://api.avax.network/ext/bc/C/rpc", + rpc=HttpUrl("https://api.avax.network/ext/bc/C/rpc"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), Chain.BASE: ChainInfo( chain_id=8453, - rpc="https://base-mainnet.public.blastapi.io", + rpc=HttpUrl("https://base-mainnet.public.blastapi.io"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), } diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py index c05303f11..5ae67102c 100644 --- a/src/aleph/vm/orchestrator/messages.py +++ b/src/aleph/vm/orchestrator/messages.py @@ -85,5 +85,5 @@ async def get_message_status(item_hash: ItemHash) -> MessageStatus: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["status"] diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py index 71181c361..7194f873a 100644 --- a/src/aleph/vm/orchestrator/payment.py +++ b/src/aleph/vm/orchestrator/payment.py @@ -40,7 +40,7 @@ async def fetch_balance_of_address(address: str) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["balance"] @@ -52,7 +52,7 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_flow: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] @@ -74,7 +74,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_hold: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 376e73f6f..a2a2a824f 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -55,7 +55,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.model_dump_json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py index db0950240..12692f6a9 100644 --- a/src/aleph/vm/orchestrator/status.py +++ b/src/aleph/vm/orchestrator/status.py @@ -26,7 +26,7 @@ async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) url = f"{vm_url}{suffix}" async with session.get(url) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: Any = None) -> Any: @@ -35,7 +35,7 @@ async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: url = f"{vm_url}{suffix}" async with session.post(url, json=data) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def check_index(session: ClientSession, vm_id: ItemHash) -> bool: diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py index b40d87803..199d23733 100644 --- a/src/aleph/vm/orchestrator/views/operator.py +++ b/src/aleph/vm/orchestrator/views/operator.py @@ -322,12 +322,12 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated Send secret to the VM and start it """ try: - data = await request.model_dump_json() + data = await request.json() params = InjectSecretParams.model_validate(data) except json.JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except pydantic.ValidationError as error: - return web.json_response(data=error.model_dump_json(), status=web.HTTPBadRequest.status_code) + return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) vm_hash = get_itemhash_or_400(request.match_info) pool: VmPool = request.app["vm_pool"] diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 1366b3310..7e289dca2 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -136,7 +136,7 @@ async def get_latest_amend(item_hash: str) -> str: async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() - result: str = await resp.model_dump_json() + result: str = await resp.json() assert isinstance(result, str) return result or item_hash diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py index f212fc478..3197133f0 100644 --- a/tests/supervisor/test_status.py +++ b/tests/supervisor/test_status.py @@ -13,13 +13,13 @@ async def test_check_internet_wrong_result_code(): mock_session = Mock() mock_session.get = MagicMock() - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 200, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is True - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 400, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is False From 4f3345701a913ce812e50c4212d780bd1c87617f Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:32:02 +0000 Subject: [PATCH 14/91] fix: Additional slash not needed in the url Adding a / with the url makes the url containing two slashes which break the url Removing it --- src/aleph/vm/storage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 7e289dca2..7e15cc655 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -132,7 +132,7 @@ async def get_latest_amend(item_hash: str) -> str: if settings.FAKE_DATA_PROGRAM: return item_hash else: - url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" + url = f"{settings.CONNECTOR_URL}compute/latest_amend/{item_hash}" async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() @@ -150,7 +150,7 @@ async def get_message(ref: str) -> ProgramMessage | InstanceMessage: logger.debug("Using the fake data message") else: cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json") - url = f"{settings.CONNECTOR_URL}/download/message/{ref}" + url = f"{settings.CONNECTOR_URL}download/message/{ref}" await download_file(url, cache_path) with open(cache_path) as cache_file: @@ -186,7 +186,7 @@ async def get_code_path(ref: str) -> Path: raise ValueError(msg) cache_path = Path(settings.CODE_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/code/{ref}" + url = f"{settings.CONNECTOR_URL}download/code/{ref}" await download_file(url, cache_path) return cache_path @@ -198,7 +198,7 @@ async def get_data_path(ref: str) -> Path: return Path(f"{data_dir}.zip") cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) return cache_path @@ -219,7 +219,7 @@ async def get_runtime_path(ref: str) -> Path: return Path(settings.FAKE_DATA_RUNTIME) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" if not cache_path.is_file(): # File does not exist, download it @@ -237,7 +237,7 @@ async def get_rootfs_base_path(ref: ItemHash) -> Path: return Path(settings.FAKE_INSTANCE_BASE) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path @@ -359,7 +359,7 @@ async def get_existing_file(ref: str) -> Path: return Path(settings.FAKE_DATA_VOLUME) cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path From 9fd949dcf9f8dc685b5626bd9a29fb82ab192c76 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:35:31 +0000 Subject: [PATCH 15/91] fix: Forcing the version 2 of pydantic --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 783d98950..299830b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", + "pydantic>=2", "pydantic-settings", "pyroute2==0.7.12", "python-cpuid==0.1", From b75668744586f78a7badcc8891241fa22503e4e1 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 07:11:42 +0000 Subject: [PATCH 16/91] fix: yamlfix and pydantic are incompatible --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 299830b83..4c650ee45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,8 +121,9 @@ dependencies = [ "mypy==1.8.0", "ruff==0.4.6", "isort==5.13.2", - "yamlfix==1.16.1", + "yamlfix==1.17.0", "pyproject-fmt==2.2.1", + "pydantic>=2", ] [tool.hatch.envs.linting.scripts] typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}" From 7db1e354ea14c4f5a541725b1899446cb5796994 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:35:50 +0000 Subject: [PATCH 17/91] Fix: HttpUrl no longer need scheme value --- examples/example_fastapi/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py index 44caaf458..59b216e38 100644 --- a/examples/example_fastapi/main.py +++ b/examples/example_fastapi/main.py @@ -215,9 +215,9 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5): async def read_internet(): """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS.""" internet_hosts: list[HttpUrl] = [ - HttpUrl(url="https://aleph.im/", scheme="https"), - HttpUrl(url="https://ethereum.org", scheme="https"), - HttpUrl(url="https://ipfs.io/", scheme="https"), + HttpUrl(url="https://aleph.im/"), + HttpUrl(url="https://ethereum.org/"), + HttpUrl(url="https://ipfs.io/"), ] timeout_seconds = 5 From 6acdc3cd0d5ac3268adb04ad9bff1ab373266c55 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:37:03 +0000 Subject: [PATCH 18/91] Fix: revert changes about json --- src/aleph/vm/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 57089f3be..eee891a7e 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: From c1bdd962077070ed5824c973885f10e33662364d Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:37:42 +0000 Subject: [PATCH 19/91] Fix: Pydantic 2 handle differently the errors --- src/aleph/vm/orchestrator/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 996584497..60bbda8b5 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -8,6 +8,7 @@ import aiohttp import pydantic +from pydantic import ValidationError from aiohttp import web from aleph_message.models import ( AlephMessage, @@ -73,10 +74,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: try: yield parse_message(data) - except pydantic.error_wrappers.ValidationError as error: + except pydantic.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.errors}", exc_info=False, ) continue From 9604e707d3d183e5667de4160c6ff6795cbc0011 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:43:45 +0000 Subject: [PATCH 20/91] Style: isortwq --- src/aleph/vm/orchestrator/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 60bbda8b5..3af5766c0 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -8,7 +8,6 @@ import aiohttp import pydantic -from pydantic import ValidationError from aiohttp import web from aleph_message.models import ( AlephMessage, @@ -17,6 +16,7 @@ parse_message, ) from aleph_message.status import MessageStatus +from pydantic import ValidationError from yarl import URL from aleph.vm.conf import settings From 17cd07ee06c5afdc6f493b9ec9589e56f04e2105 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 04:55:12 +0000 Subject: [PATCH 21/91] Refactor: update validation for Pydantic v2 Changed @root_validator to @model_validator(mode="after") for checking data after the model is created. Updated values to use a more flexible type for Pydantic v2. Accessed values as a dictionary (values["payload"]) instead of using values.payload. --- .../vm/orchestrator/views/authentication.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 21c7a6ac1..65353691c 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -22,7 +22,8 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, field_validator, model_validator +from pydantic import (BaseModel, ValidationError, ValidationInfo, + field_validator, model_validator) from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -103,26 +104,23 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: return bytes.fromhex(v.decode()) @model_validator(mode="after") - @classmethod - def check_expiry(cls, values) -> dict[str, bytes]: + def check_expiry(values) -> 'SignedPubKeyHeader': """Check that the token has not expired""" - payload: bytes = values.payload + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): - msg = "Token expired" - raise ValueError(msg) + raise ValueError("Token expired") return values @model_validator(mode="after") - @classmethod - def check_signature(cls, values) -> dict[str, bytes]: + def check_signature(values) -> 'SignedPubKeyHeader': """Check that the signature is valid""" - signature: list = values.signature - payload: bytes = values.payload + signature = values.signature + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values - + @property def content(self) -> SignedPubKeyPayload: """Return the content of the header""" From 249c32271598fa4be0209a08fab586fe61a3406b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 05:07:11 +0000 Subject: [PATCH 22/91] Fix: Correct import statement from '__futures__' to '__future__' for postponed annotations --- src/aleph/vm/orchestrator/views/authentication.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 65353691c..c51af36fb 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -5,6 +5,8 @@ Can be enabled on an endpoint using the @require_jwk_authentication decorator """ +from __future__ import annotations + # Keep datetime import as is as it allow patching in test import datetime import functools @@ -104,7 +106,7 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: return bytes.fromhex(v.decode()) @model_validator(mode="after") - def check_expiry(values) -> 'SignedPubKeyHeader': + def check_expiry(values) -> SignedPubKeyHeader: """Check that the token has not expired""" payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) @@ -113,7 +115,7 @@ def check_expiry(values) -> 'SignedPubKeyHeader': return values @model_validator(mode="after") - def check_signature(values) -> 'SignedPubKeyHeader': + def check_signature(values) -> SignedPubKeyHeader: """Check that the signature is valid""" signature = values.signature payload = values.payload From e82fe24dae3befbc7252273ddbd254f5ca022587 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 05:12:35 +0000 Subject: [PATCH 23/91] style: black --- src/aleph/vm/orchestrator/views/authentication.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index c51af36fb..b40cfcd4c 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -24,8 +24,7 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import (BaseModel, ValidationError, ValidationInfo, - field_validator, model_validator) +from pydantic import BaseModel, ValidationError, ValidationInfo, field_validator, model_validator from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -122,7 +121,7 @@ def check_signature(values) -> SignedPubKeyHeader: content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values - + @property def content(self) -> SignedPubKeyPayload: """Return the content of the header""" From 9b7541104a0ed6ef47a88c4f3706aaf6b78ea0bc Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 05:17:04 +0000 Subject: [PATCH 24/91] style: isort --- src/aleph/vm/orchestrator/views/authentication.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index b40cfcd4c..9ba4b12f9 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -24,7 +24,13 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, ValidationInfo, field_validator, model_validator +from pydantic import ( + BaseModel, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from solathon.utils import verify_signature from aleph.vm.conf import settings From 87f7a4c101a3b1701feaeca1e45edfc128f9197b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 09:39:28 +0000 Subject: [PATCH 25/91] Fix: Missing dependency in the debian Makefile When adding a depedency inside the pyproject, we need to add it as well in the makefile. --- packaging/Makefile | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/Makefile b/packaging/Makefile index 0d1c4dcb9..909a25b59 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl diff --git a/pyproject.toml b/pyproject.toml index 4c650ee45..31d4676f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic>=2", - "pydantic-settings", + "pydantic-settings==2.6.1", "pyroute2==0.7.12", "python-cpuid==0.1", "python-dotenv", From 35f1ed86859ce304b9791646875a904e434752a7 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 09:54:05 +0000 Subject: [PATCH 26/91] Fix: Wrong version of aleph-message inside the Makefile --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index 909a25b59..28eedc45c 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=mypackage' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl From 755680a2ca1b3eecf72eedbfda7854b47413e147 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 10:13:39 +0000 Subject: [PATCH 27/91] Fix: Wrong name for aleph-message inside the Makefile --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index 28eedc45c..76f9ba4eb 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=mypackage' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl From 74847bc60652629b152ec2c08df0b6e506211cc5 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 10:39:11 +0000 Subject: [PATCH 28/91] fix: dump_wargs arg don't work the same in pydantic 2 --- src/aleph/vm/orchestrator/run.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index a2a2a824f..9754e06e3 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -1,24 +1,17 @@ import asyncio +import json import logging from typing import Any import msgpack from aiohttp import web -from aiohttp.web_exceptions import ( - HTTPBadGateway, - HTTPBadRequest, - HTTPInternalServerError, -) +from aiohttp.web_exceptions import HTTPBadGateway, HTTPBadRequest, HTTPInternalServerError from aleph_message.models import ItemHash from msgpack import UnpackValueError from multidict import CIMultiDict from aleph.vm.conf import settings -from aleph.vm.controllers.firecracker.program import ( - FileTooLargeError, - ResourceDownloadError, - VmSetupError, -) +from aleph.vm.controllers.firecracker.program import FileTooLargeError, ResourceDownloadError, VmSetupError from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError from aleph.vm.models import VmExecution from aleph.vm.pool import VmPool @@ -55,7 +48,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 68cbc6e31333c2a5cb51503cc576d44a1d4c26fb Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 10:44:03 +0000 Subject: [PATCH 29/91] style: isort --- src/aleph/vm/orchestrator/run.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 9754e06e3..d95c40938 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -5,13 +5,21 @@ import msgpack from aiohttp import web -from aiohttp.web_exceptions import HTTPBadGateway, HTTPBadRequest, HTTPInternalServerError +from aiohttp.web_exceptions import ( + HTTPBadGateway, + HTTPBadRequest, + HTTPInternalServerError, +) from aleph_message.models import ItemHash from msgpack import UnpackValueError from multidict import CIMultiDict from aleph.vm.conf import settings -from aleph.vm.controllers.firecracker.program import FileTooLargeError, ResourceDownloadError, VmSetupError +from aleph.vm.controllers.firecracker.program import ( + FileTooLargeError, + ResourceDownloadError, + VmSetupError, +) from aleph.vm.hypervisors.firecracker.microvm import MicroVMFailedInitError from aleph.vm.models import VmExecution from aleph.vm.pool import VmPool From 070841dabde776bc0b21df2aaf2bad5e0400a1f1 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 18 Nov 2024 06:30:22 +0000 Subject: [PATCH 30/91] fix: datetime not seriazable --- src/aleph/vm/orchestrator/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index d95c40938..f347aa0c2 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -56,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True)}") + logger.debug(f"Message: {json_dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 5881ee1f49445ba334af764e689a20b5f7aff5ba Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 18 Nov 2024 06:37:49 +0000 Subject: [PATCH 31/91] fix: datetime not seriazable --- src/aleph/vm/orchestrator/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index f347aa0c2..5708c162d 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -56,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {json_dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 68071065981bbb0f37386a51de52f1d45426531c Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:55:00 +0000 Subject: [PATCH 32/91] Fix: Adaptations for Pydantic v2 migration and stricter validation - Improved error handling in `get_signed_pubkey` to align with stricter JSON validation and date format requirements in Pydantic v2. - Addressed warnings and deprecated functions raised during the migration to Pydantic v2. - Updated type annotations to comply with Pydantic's stricter type checks (`url` now requires proper type validation). - Refactored test mocks to use `.model_dump_json()` instead of `.json()` for serialization - Adjusted test cases to account for new fields in validation errors (`url`, `ctx`, and `input`) introduced in Pydantic v2. - Applied linting for style consistency across tests and codebase. --- pyproject.toml | 9 +++-- src/aleph/vm/controllers/__main__.py | 2 +- src/aleph/vm/controllers/configuration.py | 2 +- src/aleph/vm/controllers/qemu/instance.py | 2 +- .../vm/hypervisors/firecracker/config.py | 11 +----- .../vm/hypervisors/firecracker/microvm.py | 2 +- src/aleph/vm/models.py | 10 ++--- src/aleph/vm/orchestrator/chain.py | 12 +++--- src/aleph/vm/orchestrator/reactor.py | 2 +- src/aleph/vm/orchestrator/resources.py | 2 +- src/aleph/vm/orchestrator/tasks.py | 2 +- src/aleph/vm/orchestrator/views/__init__.py | 4 +- .../vm/orchestrator/views/authentication.py | 39 ++++++++++--------- src/aleph/vm/orchestrator/views/operator.py | 2 +- src/aleph/vm/storage.py | 14 +++---- src/aleph/vm/utils/__init__.py | 4 +- tests/supervisor/test_status.py | 1 + tests/supervisor/test_views.py | 14 ++++--- vm_connector/conf.py | 7 +--- 19 files changed, 68 insertions(+), 73 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 99e68031c..783d98950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "aioredis==1.3.1", "aiosqlite==0.19", "alembic==1.13.1", - "aleph-message @ git+https://github.com/aleph-im/aleph-message#egg=main", + "aleph-message @ git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=main", "aleph-superfluid~=0.2.1", "dbus-python==1.3.2", "eth-account~=0.10", @@ -48,9 +48,9 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic-settings", - "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", + "python-dotenv", "pyyaml==6.0.1", "qmp==1.1", "schedule==1.2.1", @@ -192,8 +192,6 @@ lint.ignore = [ # Allow the use of assert statements "S101", ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] #[tool.ruff.flake8-tidy-imports] #ban-relative-imports = "all" #unfixable = [ @@ -201,6 +199,9 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] # "F401", #] +# Tests can use magic values, assertions, and relative imports +lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] + [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/src/aleph/vm/controllers/__main__.py b/src/aleph/vm/controllers/__main__.py index 519270b48..a8fe43399 100644 --- a/src/aleph/vm/controllers/__main__.py +++ b/src/aleph/vm/controllers/__main__.py @@ -26,7 +26,7 @@ def configuration_from_file(path: Path): with open(path) as f: data = json.load(f) - return Configuration.parse_obj(data) + return Configuration.model_validate(data) def parse_args(args): diff --git a/src/aleph/vm/controllers/configuration.py b/src/aleph/vm/controllers/configuration.py index 0c534241e..a5ce7e554 100644 --- a/src/aleph/vm/controllers/configuration.py +++ b/src/aleph/vm/controllers/configuration.py @@ -69,7 +69,7 @@ def save_controller_configuration(vm_hash: str, configuration: Configuration) -> config_file_path = Path(f"{settings.EXECUTION_ROOT}/{vm_hash}-controller.json") with config_file_path.open("w") as controller_config_file: controller_config_file.write( - configuration.json( + configuration.model_dump_json( by_alias=True, exclude_none=True, indent=4, exclude={"settings": {"USE_DEVELOPER_SSH_KEYS"}} ) ) diff --git a/src/aleph/vm/controllers/qemu/instance.py b/src/aleph/vm/controllers/qemu/instance.py index dd840e22b..e92917683 100644 --- a/src/aleph/vm/controllers/qemu/instance.py +++ b/src/aleph/vm/controllers/qemu/instance.py @@ -215,7 +215,7 @@ async def configure(self): def save_controller_configuration(self): """Save VM configuration to be used by the controller service""" path = Path(f"{settings.EXECUTION_ROOT}/{self.vm_hash}-controller.json") - path.open("w").write(self.controller_configuration.json(by_alias=True, exclude_none=True, indent=4)) + path.open("w").write(self.controller_configuration.model_dump_json(by_alias=True, exclude_none=True, indent=4)) path.chmod(0o644) return path diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index 79d99e53d..59560ce1c 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import BaseModel, PositiveInt +from pydantic import BaseModel, ConfigDict, PositiveInt VSOCK_PATH = "/tmp/v.sock" @@ -54,11 +54,4 @@ class FirecrackerConfig(BaseModel): vsock: Vsock | None = None network_interfaces: list[NetworkInterface] | None = None - # TODO[pydantic]: We couldn't refactor this class, please create the `model_config` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. - class Config: - allow_population_by_field_name = True - - @staticmethod - def alias_generator(x: str): - return x.replace("_", "-") + model_config = ConfigDict(populate_by_name=True, alias_generator=lambda x: x.replace("_", "-")) diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py index 7a8fe787e..4421491e5 100644 --- a/src/aleph/vm/hypervisors/firecracker/microvm.py +++ b/src/aleph/vm/hypervisors/firecracker/microvm.py @@ -193,7 +193,7 @@ async def save_configuration_file(self, config: FirecrackerConfig) -> Path: if not self.use_jailer else open(f"{self.jailer_path}/tmp/config.json", "wb") ) as config_file: - config_file.write(config.json(by_alias=True, exclude_none=True, indent=4).encode()) + config_file.write(config.model_dump_json(by_alias=True, exclude_none=True, indent=4).encode()) config_file.flush() config_file_path = Path(config_file.name) config_file_path.chmod(0o644) diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 35de4076d..57089f3be 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -407,8 +407,8 @@ async def save(self): vcpus=self.vm.hardware_resources.vcpus, memory=self.vm.hardware_resources.memory, network_tap=self.vm.tap_interface.device_name if self.vm.tap_interface else "", - message=self.message.json(), - original_message=self.original.json(), + message=self.message.model_dump_json(), + original_message=self.original.model_dump_json(), persistent=self.persistent, ) ) @@ -431,8 +431,8 @@ async def save(self): io_write_bytes=None, vcpus=self.vm.hardware_resources.vcpus, memory=self.vm.hardware_resources.memory, - message=self.message.json(), - original_message=self.original.json(), + message=self.message.model_dump_json(), + original_message=self.original.model_dump_json(), persistent=self.persistent, ) ) @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 14bde4759..10cf1211e 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, HttpUrl, model_validator logger = logging.getLogger(__name__) @@ -12,7 +12,7 @@ class ChainInfo(BaseModel): """ chain_id: int - rpc: str + rpc: HttpUrl standard_token: str | None = None super_token: str | None = None testnet: bool = False @@ -35,7 +35,7 @@ def check_tokens(cls, values): # TESTNETS "SEPOLIA": ChainInfo( chain_id=11155111, - rpc="https://eth-sepolia.public.blastapi.io", + rpc=HttpUrl("https://eth-sepolia.public.blastapi.io"), standard_token="0xc4bf5cbdabe595361438f8c6a187bdc330539c60", super_token="0x22064a21fee226d8ffb8818e7627d5ff6d0fc33a", active=False, @@ -44,18 +44,18 @@ def check_tokens(cls, values): # MAINNETS Chain.ETH: ChainInfo( chain_id=1, - rpc="https://eth-mainnet.public.blastapi.io", + rpc=HttpUrl("https://eth-mainnet.public.blastapi.io"), standard_token="0x27702a26126e0B3702af63Ee09aC4d1A084EF628", active=False, ), Chain.AVAX: ChainInfo( chain_id=43114, - rpc="https://api.avax.network/ext/bc/C/rpc", + rpc=HttpUrl("https://api.avax.network/ext/bc/C/rpc"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), Chain.BASE: ChainInfo( chain_id=8453, - rpc="https://base-mainnet.public.blastapi.io", + rpc=HttpUrl("https://base-mainnet.public.blastapi.io"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), } diff --git a/src/aleph/vm/orchestrator/reactor.py b/src/aleph/vm/orchestrator/reactor.py index 785f2c233..f8326fa97 100644 --- a/src/aleph/vm/orchestrator/reactor.py +++ b/src/aleph/vm/orchestrator/reactor.py @@ -61,7 +61,7 @@ async def trigger(self, message: AlephMessage): for subscription in listener.content.on.message: if subscription_matches(subscription, message): vm_hash = listener.item_hash - event = message.json() + event = message.model_dump_json() # Register the listener in the list of coroutines to run asynchronously: coroutines.append(run_code_on_event(vm_hash, event, self.pubsub, pool=self.pool)) break diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py index d4b9c8985..23fcbf1a5 100644 --- a/src/aleph/vm/orchestrator/resources.py +++ b/src/aleph/vm/orchestrator/resources.py @@ -134,7 +134,7 @@ async def about_system_usage(_: web.Request): properties=get_machine_properties(), ) - return web.json_response(text=usage.json(exclude_none=True)) + return web.json_response(text=usage.model_dump_json(exclude_none=True)) @cors_allow_all diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 921a2265f..996584497 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -76,7 +76,7 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: except pydantic.error_wrappers.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", exc_info=False, ) continue diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py index 4bba01aa8..603377359 100644 --- a/src/aleph/vm/orchestrator/views/__init__.py +++ b/src/aleph/vm/orchestrator/views/__init__.py @@ -372,7 +372,7 @@ async def update_allocations(request: web.Request): try: data = await request.json() - allocation = Allocation.parse_obj(data) + allocation = Allocation.model_validate(data) except ValidationError as error: return web.json_response(text=error.json(), status=web.HTTPBadRequest.status_code) @@ -456,7 +456,7 @@ async def notify_allocation(request: web.Request): """Notify instance allocation, only used for Pay as you Go feature""" try: data = await request.json() - vm_notification = VMNotification.parse_obj(data) + vm_notification = VMNotification.model_validate(data) except JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except ValidationError as error: diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 40ab4a39d..21c7a6ac1 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -106,8 +106,8 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: @classmethod def check_expiry(cls, values) -> dict[str, bytes]: """Check that the token has not expired""" - payload: bytes = values["payload"] - content = SignedPubKeyPayload.parse_raw(payload) + payload: bytes = values.payload + content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): msg = "Token expired" raise ValueError(msg) @@ -117,16 +117,16 @@ def check_expiry(cls, values) -> dict[str, bytes]: @classmethod def check_signature(cls, values) -> dict[str, bytes]: """Check that the signature is valid""" - signature: list = values["signature"] - payload: bytes = values["payload"] - content = SignedPubKeyPayload.parse_raw(payload) + signature: list = values.signature + payload: bytes = values.payload + content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values @property def content(self) -> SignedPubKeyPayload: """Return the content of the header""" - return SignedPubKeyPayload.parse_raw(self.payload) + return SignedPubKeyPayload.model_validate_json(self.payload) class SignedOperationPayload(BaseModel): @@ -173,13 +173,13 @@ def signature_must_be_hex(cls, v) -> bytes: def payload_must_be_hex(cls, v) -> bytes: """Convert the payload from hexadecimal to bytes""" v = bytes.fromhex(v.decode()) - _ = SignedOperationPayload.parse_raw(v) + _ = SignedOperationPayload.model_validate_json(v) return v @property def content(self) -> SignedOperationPayload: """Return the content of the header""" - return SignedOperationPayload.parse_raw(self.payload) + return SignedOperationPayload.model_validate_json(self.payload) def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader: @@ -189,29 +189,30 @@ def get_signed_pubkey(request: web.Request) -> SignedPubKeyHeader: raise web.HTTPBadRequest(reason="Missing X-SignedPubKey header") try: - return SignedPubKeyHeader.parse_raw(signed_pubkey_header) + data = json.loads(signed_pubkey_header) + if "expires" in data and isinstance(data["expires"], float): + data["expires"] = str(data["expires"]) + return SignedPubKeyHeader.model_validate_json(json.dumps(data)) except KeyError as error: logger.debug(f"Missing X-SignedPubKey header: {error}") raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey fields") from error except json.JSONDecodeError as error: raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from error - except ValueError as errors: + except ValidationError as errors: logging.debug(errors) - for err in errors.args[0]: - if isinstance(err.exc, json.JSONDecodeError): - raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey format") from errors - if str(err.exc) == "Token expired": + for err in errors.errors(): + if err["type"] == "value_error" and "Token expired" in str(err["msg"]): raise web.HTTPUnauthorized(reason="Token expired") from errors - if str(err.exc) == "Invalid signature": + elif err["type"] == "value_error" and "Invalid signature" in str(err["msg"]): raise web.HTTPUnauthorized(reason="Invalid signature") from errors - raise errors + raise web.HTTPBadRequest(reason="Invalid X-SignedPubKey data") def get_signed_operation(request: web.Request) -> SignedOperation: """Get the signed operation public key that is signed by the ephemeral key from the request headers.""" try: signed_operation = request.headers["X-SignedOperation"] - return SignedOperation.parse_raw(signed_operation) + return SignedOperation.model_validate_json(signed_operation) except KeyError as error: raise web.HTTPBadRequest(reason="Missing X-SignedOperation header") from error except json.JSONDecodeError as error: @@ -255,8 +256,8 @@ async def authenticate_websocket_message(message) -> str: """Authenticate a websocket message since JS cannot configure headers on WebSockets.""" if not isinstance(message, dict): raise Exception("Invalid format for auth packet, see /doc/operator_auth.md") - signed_pubkey = SignedPubKeyHeader.parse_obj(message["X-SignedPubKey"]) - signed_operation = SignedOperation.parse_obj(message["X-SignedOperation"]) + signed_pubkey = SignedPubKeyHeader.model_validate(message["X-SignedPubKey"]) + signed_operation = SignedOperation.model_validate(message["X-SignedOperation"]) if signed_operation.content.domain != settings.DOMAIN_NAME: logger.debug(f"Invalid domain '{signed_operation.content.domain}' != '{settings.DOMAIN_NAME}'") raise web.HTTPUnauthorized(reason="Invalid domain") diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py index af0e98f45..199d23733 100644 --- a/src/aleph/vm/orchestrator/views/operator.py +++ b/src/aleph/vm/orchestrator/views/operator.py @@ -323,7 +323,7 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated """ try: data = await request.json() - params = InjectSecretParams.parse_obj(data) + params = InjectSecretParams.model_validate(data) except json.JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except pydantic.ValidationError as error: diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 7e289dca2..7e15cc655 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -132,7 +132,7 @@ async def get_latest_amend(item_hash: str) -> str: if settings.FAKE_DATA_PROGRAM: return item_hash else: - url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" + url = f"{settings.CONNECTOR_URL}compute/latest_amend/{item_hash}" async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() @@ -150,7 +150,7 @@ async def get_message(ref: str) -> ProgramMessage | InstanceMessage: logger.debug("Using the fake data message") else: cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json") - url = f"{settings.CONNECTOR_URL}/download/message/{ref}" + url = f"{settings.CONNECTOR_URL}download/message/{ref}" await download_file(url, cache_path) with open(cache_path) as cache_file: @@ -186,7 +186,7 @@ async def get_code_path(ref: str) -> Path: raise ValueError(msg) cache_path = Path(settings.CODE_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/code/{ref}" + url = f"{settings.CONNECTOR_URL}download/code/{ref}" await download_file(url, cache_path) return cache_path @@ -198,7 +198,7 @@ async def get_data_path(ref: str) -> Path: return Path(f"{data_dir}.zip") cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) return cache_path @@ -219,7 +219,7 @@ async def get_runtime_path(ref: str) -> Path: return Path(settings.FAKE_DATA_RUNTIME) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" if not cache_path.is_file(): # File does not exist, download it @@ -237,7 +237,7 @@ async def get_rootfs_base_path(ref: ItemHash) -> Path: return Path(settings.FAKE_INSTANCE_BASE) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path @@ -359,7 +359,7 @@ async def get_existing_file(ref: str) -> Path: return Path(settings.FAKE_DATA_VOLUME) cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path diff --git a/src/aleph/vm/utils/__init__.py b/src/aleph/vm/utils/__init__.py index d8eecad95..9bf30d001 100644 --- a/src/aleph/vm/utils/__init__.py +++ b/src/aleph/vm/utils/__init__.py @@ -24,9 +24,9 @@ def get_message_executable_content(message_dict: dict) -> ExecutableContent: try: - return ProgramContent.parse_obj(message_dict) + return ProgramContent.model_validate(message_dict) except ValueError: - return InstanceContent.parse_obj(message_dict) + return InstanceContent.model_validate(message_dict) def cors_allow_all(function): diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py index 0e0449dbf..3197133f0 100644 --- a/tests/supervisor/test_status.py +++ b/tests/supervisor/test_status.py @@ -16,6 +16,7 @@ async def test_check_internet_wrong_result_code(): mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 200, "headers": {"Server": "nginx"}} ) + assert await check_internet(mock_session, vm_id) is True mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index fff8b5492..3e9c4282e 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -20,15 +20,17 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): response: web.Response = await client.post( "/control/allocations", json={"persistent_vms": ["not-an-ItemHash"]}, headers={"X-Auth-Signature": "test"} ) + assert response.status == 400 + assert await response.json() == [ { - "loc": [ - "persistent_vms", - 0, - ], - "msg": "Could not determine hash type: 'not-an-ItemHash'", - "type": "value_error.unknownhash", + "loc": ["persistent_vms", 0], + "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", + "type": "value_error", + "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, + "input": "not-an-ItemHash", + "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] diff --git a/vm_connector/conf.py b/vm_connector/conf.py index d2ee465fc..8164c0320 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -1,7 +1,7 @@ import logging from typing import NewType -from pydantic import BaseSettings +from pydantic import BaseSettings, ConfigDict logger = logging.getLogger(__name__) @@ -27,10 +27,7 @@ def display(self) -> str: f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items() ) - class Config: - env_prefix = "ALEPH_" - case_sensitive = False - env_file = ".env" + model_config = ConfigDict(env_prefix="ALEPH_", case_sensitive=False, env_file=".env") # Settings singleton From c140d59678493c42590cdb944fa96142e5e859ff Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:35:31 +0000 Subject: [PATCH 33/91] fix: Forcing the version 2 of pydantic --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 783d98950..299830b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", + "pydantic>=2", "pydantic-settings", "pyroute2==0.7.12", "python-cpuid==0.1", From 37c4f81511642792aaaeb42fdfdcec9e4573d0a4 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 07:11:42 +0000 Subject: [PATCH 34/91] fix: yamlfix and pydantic are incompatible --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 299830b83..4c650ee45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,8 +121,9 @@ dependencies = [ "mypy==1.8.0", "ruff==0.4.6", "isort==5.13.2", - "yamlfix==1.16.1", + "yamlfix==1.17.0", "pyproject-fmt==2.2.1", + "pydantic>=2", ] [tool.hatch.envs.linting.scripts] typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}" From 08e2d656823d346da491f67433a9937df8125aaa Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:35:50 +0000 Subject: [PATCH 35/91] Fix: HttpUrl no longer need scheme value --- examples/example_fastapi/main.py | 6 +++--- src/aleph/vm/models.py | 2 +- src/aleph/vm/orchestrator/tasks.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py index 44caaf458..59b216e38 100644 --- a/examples/example_fastapi/main.py +++ b/examples/example_fastapi/main.py @@ -215,9 +215,9 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5): async def read_internet(): """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS.""" internet_hosts: list[HttpUrl] = [ - HttpUrl(url="https://aleph.im/", scheme="https"), - HttpUrl(url="https://ethereum.org", scheme="https"), - HttpUrl(url="https://ipfs.io/", scheme="https"), + HttpUrl(url="https://aleph.im/"), + HttpUrl(url="https://ethereum.org/"), + HttpUrl(url="https://ipfs.io/"), ] timeout_seconds = 5 diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 57089f3be..eee891a7e 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 996584497..3af5766c0 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -16,6 +16,7 @@ parse_message, ) from aleph_message.status import MessageStatus +from pydantic import ValidationError from yarl import URL from aleph.vm.conf import settings @@ -73,10 +74,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: try: yield parse_message(data) - except pydantic.error_wrappers.ValidationError as error: + except pydantic.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.errors}", exc_info=False, ) continue From cef07081ba3528d8f9d8953df26b9a89d27ba3a7 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 04:55:12 +0000 Subject: [PATCH 36/91] Refactor: update validation for Pydantic v2 Changed @root_validator to @model_validator(mode="after") for checking data after the model is created. Updated values to use a more flexible type for Pydantic v2. Accessed values as a dictionary (values["payload"]) instead of using values.payload. --- .../vm/orchestrator/views/authentication.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 21c7a6ac1..9ba4b12f9 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -5,6 +5,8 @@ Can be enabled on an endpoint using the @require_jwk_authentication decorator """ +from __future__ import annotations + # Keep datetime import as is as it allow patching in test import datetime import functools @@ -22,7 +24,13 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, field_validator, model_validator +from pydantic import ( + BaseModel, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -103,22 +111,19 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: return bytes.fromhex(v.decode()) @model_validator(mode="after") - @classmethod - def check_expiry(cls, values) -> dict[str, bytes]: + def check_expiry(values) -> SignedPubKeyHeader: """Check that the token has not expired""" - payload: bytes = values.payload + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): - msg = "Token expired" - raise ValueError(msg) + raise ValueError("Token expired") return values @model_validator(mode="after") - @classmethod - def check_signature(cls, values) -> dict[str, bytes]: + def check_signature(values) -> SignedPubKeyHeader: """Check that the signature is valid""" - signature: list = values.signature - payload: bytes = values.payload + signature = values.signature + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values From fac5b922673174314460d3f49263f108aee2b10d Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 09:39:28 +0000 Subject: [PATCH 37/91] Fix: Missing dependency in the debian Makefile When adding a depedency inside the pyproject, we need to add it as well in the makefile. --- packaging/Makefile | 2 +- pyproject.toml | 2 +- src/aleph/vm/orchestrator/run.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging/Makefile b/packaging/Makefile index 0d1c4dcb9..76f9ba4eb 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl diff --git a/pyproject.toml b/pyproject.toml index 4c650ee45..31d4676f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic>=2", - "pydantic-settings", + "pydantic-settings==2.6.1", "pyroute2==0.7.12", "python-cpuid==0.1", "python-dotenv", diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index a2a2a824f..f347aa0c2 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -1,4 +1,5 @@ import asyncio +import json import logging from typing import Any @@ -55,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {json_dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 06bc7ad4fce337b3db6d0d16c00d8be0af5042c2 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 18 Nov 2024 06:37:49 +0000 Subject: [PATCH 38/91] fix: datetime not seriazable --- src/aleph/vm/orchestrator/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index f347aa0c2..5708c162d 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -56,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {json_dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From c773f2d25e43bf0f90a6623ca309a9b25618cec8 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 04:41:58 +0000 Subject: [PATCH 39/91] Fix: Pydantic is more strick about validation rules Pydantic's validation system now enforces strict checks, we cannot leave fields marked as required empty --- src/aleph/vm/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 7c69f3442..8bc347746 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -176,8 +176,8 @@ class Settings(BaseSettings): description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.", ) DNS_NAMESERVERS: list[str] | None = None - DNS_NAMESERVERS_IPV4: list[str] | None - DNS_NAMESERVERS_IPV6: list[str] | None + DNS_NAMESERVERS_IPV4: list[str] | None = None + DNS_NAMESERVERS_IPV6: list[str] | None = None FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") JAILER_PATH: Path = Path("/opt/firecracker/jailer") From a9cf0e787c6ea10363881762684a79496562e6f9 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:03:50 +0000 Subject: [PATCH 40/91] Fix: Pydantic 2 add more information in return error and style issue Pydantic 2 add more fields while returning error such as `url`, `ctx` or `input` Adapting the test to the new format given by pydantic --- pyproject.toml | 7 ++++--- src/aleph/vm/guest_api/__main__.py | 4 ++-- src/aleph/vm/hypervisors/firecracker/config.py | 8 ++------ src/aleph/vm/orchestrator/chain.py | 10 +++++----- src/aleph/vm/orchestrator/messages.py | 2 +- src/aleph/vm/orchestrator/payment.py | 6 +++--- src/aleph/vm/orchestrator/run.py | 2 +- src/aleph/vm/orchestrator/status.py | 4 ++-- src/aleph/vm/orchestrator/views/operator.py | 4 ++-- src/aleph/vm/storage.py | 2 +- tests/supervisor/test_status.py | 4 ++-- tests/supervisor/test_views.py | 14 ++++++++------ vm_connector/conf.py | 7 ++----- 13 files changed, 35 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22649346a..783d98950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,9 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic-settings", - "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", + "python-dotenv", "pyyaml==6.0.1", "qmp==1.1", "schedule==1.2.1", @@ -192,8 +192,6 @@ lint.ignore = [ # Allow the use of assert statements "S101", ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] #[tool.ruff.flake8-tidy-imports] #ban-relative-imports = "all" #unfixable = [ @@ -201,6 +199,9 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] # "F401", #] +# Tests can use magic values, assertions, and relative imports +lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] + [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py index dea3db2a9..8000d52bc 100644 --- a/src/aleph/vm/guest_api/__main__.py +++ b/src/aleph/vm/guest_api/__main__.py @@ -43,7 +43,7 @@ async def proxy(request: web.Request): async def repost(request: web.Request): logger.debug("REPOST") - data_raw = await request.model_dump_json() + data_raw = await request.json() topic, message = data_raw["topic"], json.loads(data_raw["data"]) content = json.loads(message["item_content"]) @@ -82,7 +82,7 @@ async def properties(request: web.Request): async def sign(request: web.Request): vm_hash = request.app["meta_vm_hash"] - message = await request.model_dump_json() + message = await request.json() # Ensure that the hash of the VM is used as sending address content = json.loads(message["item_content"]) diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index 5b35095c5..59560ce1c 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import BaseModel, PositiveInt, ConfigDict +from pydantic import BaseModel, ConfigDict, PositiveInt VSOCK_PATH = "/tmp/v.sock" @@ -54,8 +54,4 @@ class FirecrackerConfig(BaseModel): vsock: Vsock | None = None network_interfaces: list[NetworkInterface] | None = None - model_config = ConfigDict( - populate_by_name = True, - alias_generator=lambda x: x.replace("_", "-") - ) - + model_config = ConfigDict(populate_by_name=True, alias_generator=lambda x: x.replace("_", "-")) diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 3dbe4ad3c..10cf1211e 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, model_validator, HttpUrl +from pydantic import BaseModel, HttpUrl, model_validator logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def check_tokens(cls, values): # TESTNETS "SEPOLIA": ChainInfo( chain_id=11155111, - rpc="https://eth-sepolia.public.blastapi.io", + rpc=HttpUrl("https://eth-sepolia.public.blastapi.io"), standard_token="0xc4bf5cbdabe595361438f8c6a187bdc330539c60", super_token="0x22064a21fee226d8ffb8818e7627d5ff6d0fc33a", active=False, @@ -44,18 +44,18 @@ def check_tokens(cls, values): # MAINNETS Chain.ETH: ChainInfo( chain_id=1, - rpc="https://eth-mainnet.public.blastapi.io", + rpc=HttpUrl("https://eth-mainnet.public.blastapi.io"), standard_token="0x27702a26126e0B3702af63Ee09aC4d1A084EF628", active=False, ), Chain.AVAX: ChainInfo( chain_id=43114, - rpc="https://api.avax.network/ext/bc/C/rpc", + rpc=HttpUrl("https://api.avax.network/ext/bc/C/rpc"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), Chain.BASE: ChainInfo( chain_id=8453, - rpc="https://base-mainnet.public.blastapi.io", + rpc=HttpUrl("https://base-mainnet.public.blastapi.io"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), } diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py index c05303f11..5ae67102c 100644 --- a/src/aleph/vm/orchestrator/messages.py +++ b/src/aleph/vm/orchestrator/messages.py @@ -85,5 +85,5 @@ async def get_message_status(item_hash: ItemHash) -> MessageStatus: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["status"] diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py index 71181c361..7194f873a 100644 --- a/src/aleph/vm/orchestrator/payment.py +++ b/src/aleph/vm/orchestrator/payment.py @@ -40,7 +40,7 @@ async def fetch_balance_of_address(address: str) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["balance"] @@ -52,7 +52,7 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_flow: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] @@ -74,7 +74,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_hold: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 376e73f6f..a2a2a824f 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -55,7 +55,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.model_dump_json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py index db0950240..12692f6a9 100644 --- a/src/aleph/vm/orchestrator/status.py +++ b/src/aleph/vm/orchestrator/status.py @@ -26,7 +26,7 @@ async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) url = f"{vm_url}{suffix}" async with session.get(url) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: Any = None) -> Any: @@ -35,7 +35,7 @@ async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: url = f"{vm_url}{suffix}" async with session.post(url, json=data) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def check_index(session: ClientSession, vm_id: ItemHash) -> bool: diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py index b40d87803..199d23733 100644 --- a/src/aleph/vm/orchestrator/views/operator.py +++ b/src/aleph/vm/orchestrator/views/operator.py @@ -322,12 +322,12 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated Send secret to the VM and start it """ try: - data = await request.model_dump_json() + data = await request.json() params = InjectSecretParams.model_validate(data) except json.JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except pydantic.ValidationError as error: - return web.json_response(data=error.model_dump_json(), status=web.HTTPBadRequest.status_code) + return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) vm_hash = get_itemhash_or_400(request.match_info) pool: VmPool = request.app["vm_pool"] diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 1366b3310..7e289dca2 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -136,7 +136,7 @@ async def get_latest_amend(item_hash: str) -> str: async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() - result: str = await resp.model_dump_json() + result: str = await resp.json() assert isinstance(result, str) return result or item_hash diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py index f212fc478..3197133f0 100644 --- a/tests/supervisor/test_status.py +++ b/tests/supervisor/test_status.py @@ -13,13 +13,13 @@ async def test_check_internet_wrong_result_code(): mock_session = Mock() mock_session.get = MagicMock() - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 200, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is True - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 400, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is False diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index fff8b5492..3e9c4282e 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -20,15 +20,17 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): response: web.Response = await client.post( "/control/allocations", json={"persistent_vms": ["not-an-ItemHash"]}, headers={"X-Auth-Signature": "test"} ) + assert response.status == 400 + assert await response.json() == [ { - "loc": [ - "persistent_vms", - 0, - ], - "msg": "Could not determine hash type: 'not-an-ItemHash'", - "type": "value_error.unknownhash", + "loc": ["persistent_vms", 0], + "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", + "type": "value_error", + "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, + "input": "not-an-ItemHash", + "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] diff --git a/vm_connector/conf.py b/vm_connector/conf.py index d0b34eab1..8164c0320 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -27,11 +27,8 @@ def display(self) -> str: f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items() ) - model_config = ConfigDict( - env_prefix = "ALEPH_", - case_sensitive = False, - env_file = ".env" - ) + model_config = ConfigDict(env_prefix="ALEPH_", case_sensitive=False, env_file=".env") + # Settings singleton settings = ConnectorSettings() From 0c32286219be34df9359133da073557dbdf3f3a3 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:32:02 +0000 Subject: [PATCH 41/91] fix: Additional slash not needed in the url Adding a / with the url makes the url containing two slashes which break the url Removing it --- src/aleph/vm/storage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 7e289dca2..7e15cc655 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -132,7 +132,7 @@ async def get_latest_amend(item_hash: str) -> str: if settings.FAKE_DATA_PROGRAM: return item_hash else: - url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" + url = f"{settings.CONNECTOR_URL}compute/latest_amend/{item_hash}" async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() @@ -150,7 +150,7 @@ async def get_message(ref: str) -> ProgramMessage | InstanceMessage: logger.debug("Using the fake data message") else: cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json") - url = f"{settings.CONNECTOR_URL}/download/message/{ref}" + url = f"{settings.CONNECTOR_URL}download/message/{ref}" await download_file(url, cache_path) with open(cache_path) as cache_file: @@ -186,7 +186,7 @@ async def get_code_path(ref: str) -> Path: raise ValueError(msg) cache_path = Path(settings.CODE_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/code/{ref}" + url = f"{settings.CONNECTOR_URL}download/code/{ref}" await download_file(url, cache_path) return cache_path @@ -198,7 +198,7 @@ async def get_data_path(ref: str) -> Path: return Path(f"{data_dir}.zip") cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) return cache_path @@ -219,7 +219,7 @@ async def get_runtime_path(ref: str) -> Path: return Path(settings.FAKE_DATA_RUNTIME) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" if not cache_path.is_file(): # File does not exist, download it @@ -237,7 +237,7 @@ async def get_rootfs_base_path(ref: ItemHash) -> Path: return Path(settings.FAKE_INSTANCE_BASE) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path @@ -359,7 +359,7 @@ async def get_existing_file(ref: str) -> Path: return Path(settings.FAKE_DATA_VOLUME) cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path From c703d92b99c5f2b39eec5c8687a5d5c96e0c916a Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:35:31 +0000 Subject: [PATCH 42/91] fix: Forcing the version 2 of pydantic yamlfix and pydantic are incompatible --- examples/example_fastapi/main.py | 6 +++--- pyproject.toml | 4 +++- src/aleph/vm/models.py | 2 +- src/aleph/vm/orchestrator/tasks.py | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py index 44caaf458..59b216e38 100644 --- a/examples/example_fastapi/main.py +++ b/examples/example_fastapi/main.py @@ -215,9 +215,9 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5): async def read_internet(): """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS.""" internet_hosts: list[HttpUrl] = [ - HttpUrl(url="https://aleph.im/", scheme="https"), - HttpUrl(url="https://ethereum.org", scheme="https"), - HttpUrl(url="https://ipfs.io/", scheme="https"), + HttpUrl(url="https://aleph.im/"), + HttpUrl(url="https://ethereum.org/"), + HttpUrl(url="https://ipfs.io/"), ] timeout_seconds = 5 diff --git a/pyproject.toml b/pyproject.toml index 783d98950..4c650ee45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", + "pydantic>=2", "pydantic-settings", "pyroute2==0.7.12", "python-cpuid==0.1", @@ -120,8 +121,9 @@ dependencies = [ "mypy==1.8.0", "ruff==0.4.6", "isort==5.13.2", - "yamlfix==1.16.1", + "yamlfix==1.17.0", "pyproject-fmt==2.2.1", + "pydantic>=2", ] [tool.hatch.envs.linting.scripts] typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}" diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 57089f3be..eee891a7e 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 996584497..3af5766c0 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -16,6 +16,7 @@ parse_message, ) from aleph_message.status import MessageStatus +from pydantic import ValidationError from yarl import URL from aleph.vm.conf import settings @@ -73,10 +74,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: try: yield parse_message(data) - except pydantic.error_wrappers.ValidationError as error: + except pydantic.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.errors}", exc_info=False, ) continue From b8e037fad46221f665d654cec53d712097f02479 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 04:55:12 +0000 Subject: [PATCH 43/91] Refactor: update validation for Pydantic v2 Changed @root_validator to @model_validator(mode="after") for checking data after the model is created. Updated values to use a more flexible type for Pydantic v2. Accessed values as a dictionary (values["payload"]) instead of using values.payload. --- .../vm/orchestrator/views/authentication.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 21c7a6ac1..9ba4b12f9 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -5,6 +5,8 @@ Can be enabled on an endpoint using the @require_jwk_authentication decorator """ +from __future__ import annotations + # Keep datetime import as is as it allow patching in test import datetime import functools @@ -22,7 +24,13 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, field_validator, model_validator +from pydantic import ( + BaseModel, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -103,22 +111,19 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: return bytes.fromhex(v.decode()) @model_validator(mode="after") - @classmethod - def check_expiry(cls, values) -> dict[str, bytes]: + def check_expiry(values) -> SignedPubKeyHeader: """Check that the token has not expired""" - payload: bytes = values.payload + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): - msg = "Token expired" - raise ValueError(msg) + raise ValueError("Token expired") return values @model_validator(mode="after") - @classmethod - def check_signature(cls, values) -> dict[str, bytes]: + def check_signature(values) -> SignedPubKeyHeader: """Check that the signature is valid""" - signature: list = values.signature - payload: bytes = values.payload + signature = values.signature + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values From 8b795116acb9fa184e78eb842770c386c1754026 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 09:39:28 +0000 Subject: [PATCH 44/91] Fix: Missing dependency in the debian Makefile When adding a depedency inside the pyproject, we need to add it as well in the makefile. --- packaging/Makefile | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/Makefile b/packaging/Makefile index 0d1c4dcb9..76f9ba4eb 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl diff --git a/pyproject.toml b/pyproject.toml index 4c650ee45..31d4676f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic>=2", - "pydantic-settings", + "pydantic-settings==2.6.1", "pyroute2==0.7.12", "python-cpuid==0.1", "python-dotenv", From a400233bcf6c54ac0d42c41357efd09acac56595 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 10:39:11 +0000 Subject: [PATCH 45/91] fix: dump_wargs arg don't work the same in pydantic 2 fix: datetime not seriazable --- src/aleph/vm/orchestrator/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index a2a2a824f..5708c162d 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -1,4 +1,5 @@ import asyncio +import json import logging from typing import Any @@ -55,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From e2ea33a53a75d94eee3ffdcd6e97d30f5550a219 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:55:00 +0000 Subject: [PATCH 46/91] Fix: Adaptations for Pydantic v2 migration and stricter validation - Improved error handling in `get_signed_pubkey` to align with stricter JSON validation and date format requirements in Pydantic v2. - Addressed warnings and deprecated functions raised during the migration to Pydantic v2. - Updated type annotations to comply with Pydantic's stricter type checks (`url` now requires proper type validation). - Refactored test mocks to use `.model_dump_json()` instead of `.json()` for serialization - Adjusted test cases to account for new fields in validation errors (`url`, `ctx`, and `input`) introduced in Pydantic v2. - Applied linting for style consistency across tests and codebase. --- src/aleph/vm/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index eee891a7e..57089f3be 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: From d94ef5fe5589e645eef2ca3840a7200285fcd4bd Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:35:50 +0000 Subject: [PATCH 47/91] Fix: HttpUrl no longer need scheme value --- src/aleph/vm/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 57089f3be..eee891a7e 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: From d321d5a99f18b5705c01e7c552424198b547b2c0 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Thu, 31 Oct 2024 14:48:19 +0100 Subject: [PATCH 48/91] Make vm_id assignment more robust (#714) Remove the counter way to assign a vm_id as it didn't work reliably Jira ticket: ALEPH-272 That method was broken when persitent instances were loaded at start up. Since the "new" feature that allow persistent instance across aleph-vm reboot if one was started then aleph-vm was stopped and restarted the counter method could reassign the ip and break the existing vm's. Secundary reason was that the feature wasn't working properly with the default settings, as `2**available_bits` returned 1. So that code path was only used if the node owner tweaked some undocumented settings making it hard to identify and debug in prod nodes. --- src/aleph/vm/pool.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py index 3ecf500eb..025bfe45c 100644 --- a/src/aleph/vm/pool.py +++ b/src/aleph/vm/pool.py @@ -28,15 +28,13 @@ class VmPool: - """Pool of VMs already started and used to decrease response time. + """Pool of existing VMs + + For function VM we keep the VM a while after they have run, so we can reuse them and thus decrease response time. After running, a VM is saved for future reuse from the same function during a configurable duration. - - The counter is used by the VMs to set their tap interface name and the corresponding - IPv4 subnet. """ - counter: int # Used to provide distinct ids to network interfaces executions: dict[ItemHash, VmExecution] message_cache: dict[str, ExecutableMessage] network: Network | None @@ -45,7 +43,6 @@ class VmPool: creation_lock: asyncio.Lock def __init__(self, loop: asyncio.AbstractEventLoop): - self.counter = settings.START_ID_INDEX self.executions = {} self.message_cache = {} @@ -150,25 +147,13 @@ def get_unique_vm_id(self) -> int: This identifier is used to name the network interface and in the IPv4 range dedicated to the VM. """ - _, network_range = settings.IPV4_ADDRESS_POOL.split("/") - available_bits = int(network_range) - settings.IPV4_NETWORK_PREFIX_LENGTH - self.counter += 1 - if self.counter < 2**available_bits: - # In common cases, use the counter itself as the vm_id. This makes it - # easier to debug. - return self.counter - else: - # The value of the counter is too high and some functions such as the - # IPv4 range dedicated to the VM do not support such high values. - # - # We therefore recycle vm_id values from executions that are not running - # anymore. - currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()} - for i in range(settings.START_ID_INDEX, 255**2): - if i not in currently_used_vm_ids: - return i - msg = "No available value for vm_id." - raise ValueError(msg) + # Take the first id that is not already taken + currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()} + for i in range(settings.START_ID_INDEX, 255**2): + if i not in currently_used_vm_ids: + return i + msg = "No available value for vm_id." + raise ValueError(msg) def get_running_vm(self, vm_hash: ItemHash) -> VmExecution | None: """Return a running VM or None. Disables the VM expiration task.""" From 61ef25c4ac175f315bce5cc0b1a96610935fe7c8 Mon Sep 17 00:00:00 2001 From: nesitor Date: Mon, 4 Nov 2024 17:00:30 +0100 Subject: [PATCH 49/91] Implement new EVM chains (#717) * Feature: Implement new EVM chains. * FIX: Update Makefile with new dependency. * Fix: Updated to proper released package version of aleph_message dependency. --------- Co-authored-by: Andres D. Molins --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index 76f9ba4eb..c3b6511a0 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -144,6 +144,6 @@ repository-noble: cd ./repositories/noble && reprepro -Vb . includedeb noble ../../target/aleph-vm.ubuntu-24.04.deb && cd .. repositories: repository-bookworm repository-jammy repository-noble - + all-podman: all-podman-debian-12 all-podman-ubuntu-2204 all-podman-ubuntu-2404 repositories From ebb37d447abc079fc23d9ed99fb11b73834ce20e Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Tue, 5 Nov 2024 15:30:46 +0100 Subject: [PATCH 50/91] Feature: allow IPv6 DNS (#455) * Feature: allow IPv6 DNS Problem IPv6 DNS were automatically filtered when detected from resolvectl Solution: Nameservers are now split into ipv4 and ipv6 and can be passed to the VM accordingly At the moment we pass them if the ipv6 parameter is present on the tap interface but we need a more robust detection method * Display proper env conf --- src/aleph/vm/conf.py | 36 +++++++++++-------- .../vm/controllers/firecracker/instance.py | 7 +++- .../supervisor/test_resolvectl_dns_servers.py | 8 +---- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 45d2d611a..7c69f3442 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -76,17 +76,6 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]: yield server.strip() -def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]: - """ - Use resolvectl to list available IPv4 DNS servers. - VMs only support IPv4 networking for now, we must exclude IPv6 DNS from their config. - """ - for server in resolvectl_dns_servers(interface): - ip_addr = ipaddress.ip_address(server) - if isinstance(ip_addr, ipaddress.IPv4Address): - yield server - - def get_default_interface() -> str | None: """Returns the default network interface""" with open("/proc/net/route") as f: @@ -104,7 +93,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st # Use a try-except approach since resolvectl can be present but disabled and raise the following # "Failed to get global data: Unit dbus-org.freedesktop.resolve1.service not found." try: - return list(resolvectl_dns_servers_ipv4(interface=network_interface)) + return list(resolvectl_dns_servers(interface=network_interface)) except (FileNotFoundError, CalledProcessError) as error: if Path("/etc/resolv.conf").exists(): return list(etc_resolv_conf_dns_servers()) @@ -116,7 +105,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st return list(etc_resolv_conf_dns_servers()) elif dns_resolver == DnsResolver.resolvectl: - return list(resolvectl_dns_servers_ipv4(interface=network_interface)) + return list(resolvectl_dns_servers(interface=network_interface)) else: msg = "No DNS resolve defined, this should never happen." @@ -182,8 +171,13 @@ class Settings(BaseSettings): description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6", ) - DNS_RESOLUTION: DnsResolver | None = DnsResolver.detect + DNS_RESOLUTION: DnsResolver | None = Field( + default=DnsResolver.detect, + description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.", + ) DNS_NAMESERVERS: list[str] | None = None + DNS_NAMESERVERS_IPV4: list[str] | None + DNS_NAMESERVERS_IPV6: list[str] | None FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") JAILER_PATH: Path = Path("/opt/firecracker/jailer") @@ -445,6 +439,18 @@ def setup(self): network_interface=self.NETWORK_INTERFACE, ) + if not self.DNS_NAMESERVERS_IPV4: + self.DNS_NAMESERVERS_IPV4 = [] + if not self.DNS_NAMESERVERS_IPV6: + self.DNS_NAMESERVERS_IPV6 = [] + if self.DNS_NAMESERVERS: + for server in self.DNS_NAMESERVERS: + ip_addr = ipaddress.ip_address(server) + if isinstance(ip_addr, ipaddress.IPv4Address): + self.DNS_NAMESERVERS_IPV4.append(server) + if isinstance(ip_addr, ipaddress.IPv6Address): + self.DNS_NAMESERVERS_IPV6.append(server) + if not settings.ENABLE_QEMU_SUPPORT: # If QEmu is not supported, ignore the setting and use Firecracker by default settings.INSTANCE_DEFAULT_HYPERVISOR = HypervisorType.firecracker @@ -462,7 +468,7 @@ def display(self) -> str: else: attributes[attr] = getattr(self, attr) - return "\n".join(f"{attribute:<27} = {value}" for attribute, value in attributes.items()) + return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) def __init__( self, diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py index f8c33b075..da423ef73 100644 --- a/src/aleph/vm/controllers/firecracker/instance.py +++ b/src/aleph/vm/controllers/firecracker/instance.py @@ -198,6 +198,11 @@ def _create_network_file(self) -> bytes: ipv6 = self.get_ipv6() ipv6_gateway = self.get_ipv6_gateway() + nameservers_ip = [] + if ip: + nameservers_ip = settings.DNS_NAMESERVERS_IPV4 + if ipv6: + nameservers_ip += settings.DNS_NAMESERVERS_IPV6 network = { "ethernets": { "eth0": { @@ -207,7 +212,7 @@ def _create_network_file(self) -> bytes: "gateway4": route, "gateway6": ipv6_gateway, "nameservers": { - "addresses": settings.DNS_NAMESERVERS, + "addresses": nameservers_ip, }, }, }, diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py index 0daaf03c4..0af9b6fb8 100644 --- a/tests/supervisor/test_resolvectl_dns_servers.py +++ b/tests/supervisor/test_resolvectl_dns_servers.py @@ -2,7 +2,7 @@ import os from unittest import mock -from aleph.vm.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4 +from aleph.vm.conf import resolvectl_dns_servers os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False" @@ -17,9 +17,6 @@ def test_resolvectl(): dns_servers = set(resolvectl_dns_servers("eth0")) assert dns_servers == servers - dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0")) - assert dns_servers_ipv4 == servers - def test_resolvectl_ipv6(): with mock.patch( @@ -31,6 +28,3 @@ def test_resolvectl_ipv6(): dns_servers = set(resolvectl_dns_servers("eth0")) assert dns_servers == ipv4_servers | ipv6_servers - - dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0")) - assert dns_servers_ipv4 == ipv4_servers From 3f8b2acec3d19769a92f424037f7bbe0d464fc6b Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Wed, 6 Nov 2024 14:54:42 +0100 Subject: [PATCH 51/91] Problem: IGNORE_TRACEBACK_FROM_DIAGNOSTICS broken (#713) Symptom: The CustomError from the diagnostics VM was printed even if if IGNORE_TRACEBACK_FROM_DIAGNOSTICS was set to True (the default) Analysis: This was caused by the refactoring of the fastapi_example/main.py file done in fe9235ac658915eea20d5371ae45cedabe1f7b17 Which changed the output used to detect the error to catch Solution: Fix detection string --- src/aleph/vm/orchestrator/run.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 5708c162d..c3da5ad8b 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -151,9 +151,11 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques # The Diagnostics VM checks for the proper handling of exceptions. # This fills the logs with noisy stack traces, so we ignore this specific error. - ignored_error = 'raise CustomError("Whoops")' + ignored_errors = ['raise CustomError("Whoops")', "main.CustomError: Whoops"] - if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and ignored_error in result["traceback"]: + if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and any( + ignored_error in result["traceback"] for ignored_error in ignored_errors + ): logger.debug('Ignored traceback from CustomError("Whoops")') else: logger.warning(result["traceback"]) From 94876399b05224c0330239908fd9b4b92c3b7cdd Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Fri, 8 Nov 2024 15:17:09 +0100 Subject: [PATCH 52/91] Problem: error Too many open files (#720) Jira ticket: ALEPH-298 some CRN failed on any action with error OSError: [Errno 24] Too many open files: Solution: Properly close stream to journald when the VM is stopped --- .../vm/hypervisors/firecracker/microvm.py | 41 +++++++++++++------ src/aleph/vm/hypervisors/qemu/qemuvm.py | 17 +++++--- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py index 4421491e5..1d34e2752 100644 --- a/src/aleph/vm/hypervisors/firecracker/microvm.py +++ b/src/aleph/vm/hypervisors/firecracker/microvm.py @@ -13,7 +13,7 @@ from pathlib import Path from pwd import getpwnam from tempfile import NamedTemporaryFile -from typing import Any +from typing import Any, BinaryIO import msgpack from aleph_message.models import ItemHash @@ -93,6 +93,8 @@ class MicroVM: mounted_rootfs: Path | None = None _unix_socket: Server | None = None enable_log: bool + journal_stdout: BinaryIO | int | None = None + journal_stderr: BinaryIO | int | None = None def __repr__(self): return f"" @@ -219,19 +221,19 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce str(config_path), ) if self.enable_log: - journal_stdout = journal.stream(self._journal_stdout_name) - journal_stderr = journal.stream(self._journal_stderr_name) + self.journal_stdout = journal.stream(self._journal_stdout_name) + self.journal_stderr = journal.stream(self._journal_stderr_name) else: - journal_stdout = asyncio.subprocess.DEVNULL - journal_stderr = asyncio.subprocess.DEVNULL + self.journal_stdout = asyncio.subprocess.DEVNULL + self.journal_stderr = asyncio.subprocess.DEVNULL logger.debug(" ".join(options)) self.proc = await asyncio.create_subprocess_exec( *options, stdin=asyncio.subprocess.PIPE, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) return self.proc @@ -252,11 +254,11 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces self.config_file_path = config_path if self.enable_log: - journal_stdout = journal.stream(self._journal_stdout_name) - journal_stderr = journal.stream(self._journal_stderr_name) + self.journal_stdout = journal.stream(self._journal_stdout_name) + self.journal_stderr = journal.stream(self._journal_stderr_name) else: - journal_stdout = asyncio.subprocess.DEVNULL - journal_stderr = asyncio.subprocess.DEVNULL + self.journal_stdout = asyncio.subprocess.DEVNULL + self.journal_stderr = asyncio.subprocess.DEVNULL options = ( str(self.jailer_bin_path), @@ -280,8 +282,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces self.proc = await asyncio.create_subprocess_exec( *options, stdin=asyncio.subprocess.PIPE, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) return self.proc @@ -480,6 +482,19 @@ async def teardown(self): if self.stderr_task: self.stderr_task.cancel() + if ( + self.journal_stdout + and self.journal_stdout != asyncio.subprocess.DEVNULL + and hasattr(self.journal_stdout, "close") + ): + self.journal_stdout.close() + if ( + self.journal_stderr + and self.journal_stderr != asyncio.subprocess.DEVNULL + and hasattr(self.journal_stderr, "close") + ): + self.journal_stderr.close() + # Clean mounted block devices if self.mounted_rootfs: logger.debug("Waiting for one second for the VM to shutdown") diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py index 1d707c2a5..5949fbdc4 100644 --- a/src/aleph/vm/hypervisors/qemu/qemuvm.py +++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py @@ -2,7 +2,7 @@ from asyncio.subprocess import Process from dataclasses import dataclass from pathlib import Path -from typing import TextIO +from typing import BinaryIO, TextIO import qmp from systemd import journal @@ -28,6 +28,8 @@ class QemuVM: interface_name: str qemu_process: Process | None = None host_volumes: list[HostVolume] + journal_stdout: TextIO | None + journal_stderr: TextIO | None def __repr__(self) -> str: if self.qemu_process: @@ -72,8 +74,8 @@ async def start( # qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic - journal_stdout: TextIO = journal.stream(self._journal_stdout_name) - journal_stderr: TextIO = journal.stream(self._journal_stderr_name) + self.journal_stdout: BinaryIO = journal.stream(self._journal_stdout_name) + self.journal_stderr: BinaryIO = journal.stream(self._journal_stderr_name) # hardware_resources.published ports -> not implemented at the moment # hardware_resources.seconds -> only for microvm args = [ @@ -120,8 +122,8 @@ async def start( self.qemu_process = proc = await asyncio.create_subprocess_exec( *args, stdin=asyncio.subprocess.DEVNULL, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) print( @@ -149,3 +151,8 @@ def send_shutdown_message(self): async def stop(self): """Stop the VM.""" self.send_shutdown_message() + + if self.journal_stdout and self.journal_stdout != asyncio.subprocess.DEVNULL: + self.journal_stdout.close() + if self.journal_stderr and self.journal_stderr != asyncio.subprocess.DEVNULL: + self.journal_stderr.close() From b88f13ad968415a9f9cb5bb4b3c8c8aa7609f27b Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Thu, 14 Nov 2024 16:40:41 +0100 Subject: [PATCH 53/91] Update PULL_REQUEST_TEMPLATE.md for dependencies check (#722) Add a check for dependencies update --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index bcf764608..ff965a1de 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,6 +10,7 @@ Related ClickUp, GitHub or Jira tickets : ALEPH-XXX - [ ] New classes and functions contain docstrings explaining what they provide. - [ ] All new code is covered by relevant tests. - [ ] Documentation has been updated regarding these changes. +- [ ] Dependencies update in the project.toml have been mirrored in the Debian package build script `packaging/Makefile` ## Changes From 52e94ee5b19e4ad82aa4fe335630452a69da87a2 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 04:41:58 +0000 Subject: [PATCH 54/91] Fix: Pydantic is more strick about validation rules Pydantic's validation system now enforces strict checks, we cannot leave fields marked as required empty --- src/aleph/vm/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 7c69f3442..8bc347746 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -176,8 +176,8 @@ class Settings(BaseSettings): description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.", ) DNS_NAMESERVERS: list[str] | None = None - DNS_NAMESERVERS_IPV4: list[str] | None - DNS_NAMESERVERS_IPV6: list[str] | None + DNS_NAMESERVERS_IPV4: list[str] | None = None + DNS_NAMESERVERS_IPV6: list[str] | None = None FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") JAILER_PATH: Path = Path("/opt/firecracker/jailer") From 02321247b06dd59d850e12cb1bd8e768072ea090 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 07:29:11 +0000 Subject: [PATCH 55/91] Fix: Config no longer accepted in Pydantic 2 Pydantic 2 no longer have Config, replaced by model_config --- src/aleph/vm/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 8bc347746..f6fbaeeb9 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -468,7 +468,7 @@ def display(self) -> str: else: attributes[attr] = getattr(self, attr) - return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) + return "\n".join(f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) def __init__( self, From a88bcf16ea313f8bbe2623aa55d773baff641dff Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 07:34:14 +0000 Subject: [PATCH 56/91] style: black --- src/aleph/vm/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index f6fbaeeb9..6181aad80 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -468,7 +468,9 @@ def display(self) -> str: else: attributes[attr] = getattr(self, attr) - return "\n".join(f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) + return "\n".join( + f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items() + ) def __init__( self, From 90f4b27cec437ad264ff12bca5a004da83f76dde Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 08:31:55 +0000 Subject: [PATCH 57/91] fix: model_config is now a dict no longer a class --- src/aleph/vm/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 6181aad80..054b9d93d 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -467,9 +467,8 @@ def display(self) -> str: attributes[attr] = "" else: attributes[attr] = getattr(self, attr) - return "\n".join( - f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items() + f"{self.model_config.get('env_prefix', '')}{attribute} = {value}" for attribute, value in attributes.items() ) def __init__( From ad5df330ddae1fdfafd8f7c53df10f530b606b9a Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 21:11:43 +0000 Subject: [PATCH 58/91] Style: black, isort and mypy --- pyproject.toml | 7 ++++--- src/aleph/vm/guest_api/__main__.py | 4 ++-- src/aleph/vm/hypervisors/firecracker/config.py | 8 ++------ src/aleph/vm/orchestrator/chain.py | 10 +++++----- src/aleph/vm/orchestrator/messages.py | 2 +- src/aleph/vm/orchestrator/payment.py | 6 +++--- src/aleph/vm/orchestrator/run.py | 2 +- src/aleph/vm/orchestrator/status.py | 4 ++-- src/aleph/vm/orchestrator/views/operator.py | 4 ++-- src/aleph/vm/storage.py | 2 +- tests/supervisor/test_status.py | 4 ++-- tests/supervisor/test_views.py | 8 ++++---- vm_connector/conf.py | 7 ++----- 13 files changed, 31 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22649346a..783d98950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,9 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic-settings", - "python-dotenv", "pyroute2==0.7.12", "python-cpuid==0.1", + "python-dotenv", "pyyaml==6.0.1", "qmp==1.1", "schedule==1.2.1", @@ -192,8 +192,6 @@ lint.ignore = [ # Allow the use of assert statements "S101", ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] #[tool.ruff.flake8-tidy-imports] #ban-relative-imports = "all" #unfixable = [ @@ -201,6 +199,9 @@ lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] # "F401", #] +# Tests can use magic values, assertions, and relative imports +lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] + [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/src/aleph/vm/guest_api/__main__.py b/src/aleph/vm/guest_api/__main__.py index dea3db2a9..8000d52bc 100644 --- a/src/aleph/vm/guest_api/__main__.py +++ b/src/aleph/vm/guest_api/__main__.py @@ -43,7 +43,7 @@ async def proxy(request: web.Request): async def repost(request: web.Request): logger.debug("REPOST") - data_raw = await request.model_dump_json() + data_raw = await request.json() topic, message = data_raw["topic"], json.loads(data_raw["data"]) content = json.loads(message["item_content"]) @@ -82,7 +82,7 @@ async def properties(request: web.Request): async def sign(request: web.Request): vm_hash = request.app["meta_vm_hash"] - message = await request.model_dump_json() + message = await request.json() # Ensure that the hash of the VM is used as sending address content = json.loads(message["item_content"]) diff --git a/src/aleph/vm/hypervisors/firecracker/config.py b/src/aleph/vm/hypervisors/firecracker/config.py index 5b35095c5..59560ce1c 100644 --- a/src/aleph/vm/hypervisors/firecracker/config.py +++ b/src/aleph/vm/hypervisors/firecracker/config.py @@ -1,6 +1,6 @@ from pathlib import Path -from pydantic import BaseModel, PositiveInt, ConfigDict +from pydantic import BaseModel, ConfigDict, PositiveInt VSOCK_PATH = "/tmp/v.sock" @@ -54,8 +54,4 @@ class FirecrackerConfig(BaseModel): vsock: Vsock | None = None network_interfaces: list[NetworkInterface] | None = None - model_config = ConfigDict( - populate_by_name = True, - alias_generator=lambda x: x.replace("_", "-") - ) - + model_config = ConfigDict(populate_by_name=True, alias_generator=lambda x: x.replace("_", "-")) diff --git a/src/aleph/vm/orchestrator/chain.py b/src/aleph/vm/orchestrator/chain.py index 3dbe4ad3c..10cf1211e 100644 --- a/src/aleph/vm/orchestrator/chain.py +++ b/src/aleph/vm/orchestrator/chain.py @@ -1,7 +1,7 @@ import logging from aleph_message.models import Chain -from pydantic import BaseModel, model_validator, HttpUrl +from pydantic import BaseModel, HttpUrl, model_validator logger = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def check_tokens(cls, values): # TESTNETS "SEPOLIA": ChainInfo( chain_id=11155111, - rpc="https://eth-sepolia.public.blastapi.io", + rpc=HttpUrl("https://eth-sepolia.public.blastapi.io"), standard_token="0xc4bf5cbdabe595361438f8c6a187bdc330539c60", super_token="0x22064a21fee226d8ffb8818e7627d5ff6d0fc33a", active=False, @@ -44,18 +44,18 @@ def check_tokens(cls, values): # MAINNETS Chain.ETH: ChainInfo( chain_id=1, - rpc="https://eth-mainnet.public.blastapi.io", + rpc=HttpUrl("https://eth-mainnet.public.blastapi.io"), standard_token="0x27702a26126e0B3702af63Ee09aC4d1A084EF628", active=False, ), Chain.AVAX: ChainInfo( chain_id=43114, - rpc="https://api.avax.network/ext/bc/C/rpc", + rpc=HttpUrl("https://api.avax.network/ext/bc/C/rpc"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), Chain.BASE: ChainInfo( chain_id=8453, - rpc="https://base-mainnet.public.blastapi.io", + rpc=HttpUrl("https://base-mainnet.public.blastapi.io"), super_token="0xc0Fbc4967259786C743361a5885ef49380473dCF", ), } diff --git a/src/aleph/vm/orchestrator/messages.py b/src/aleph/vm/orchestrator/messages.py index c05303f11..5ae67102c 100644 --- a/src/aleph/vm/orchestrator/messages.py +++ b/src/aleph/vm/orchestrator/messages.py @@ -85,5 +85,5 @@ async def get_message_status(item_hash: ItemHash) -> MessageStatus: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["status"] diff --git a/src/aleph/vm/orchestrator/payment.py b/src/aleph/vm/orchestrator/payment.py index 71181c361..7194f873a 100644 --- a/src/aleph/vm/orchestrator/payment.py +++ b/src/aleph/vm/orchestrator/payment.py @@ -40,7 +40,7 @@ async def fetch_balance_of_address(address: str) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() return resp_data["balance"] @@ -52,7 +52,7 @@ async def fetch_execution_flow_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_flow: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] @@ -74,7 +74,7 @@ async def fetch_execution_hold_price(item_hash: ItemHash) -> Decimal: # Raise an error if the request failed resp.raise_for_status() - resp_data = await resp.model_dump_json() + resp_data = await resp.json() required_hold: float = resp_data["required_tokens"] payment_type: str | None = resp_data["payment_type"] diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 376e73f6f..a2a2a824f 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -55,7 +55,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.model_dump_json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, diff --git a/src/aleph/vm/orchestrator/status.py b/src/aleph/vm/orchestrator/status.py index db0950240..12692f6a9 100644 --- a/src/aleph/vm/orchestrator/status.py +++ b/src/aleph/vm/orchestrator/status.py @@ -26,7 +26,7 @@ async def get_json_from_vm(session: ClientSession, vm_id: ItemHash, suffix: str) url = f"{vm_url}{suffix}" async with session.get(url) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: Any = None) -> Any: @@ -35,7 +35,7 @@ async def post_to_vm(session: ClientSession, vm_id: ItemHash, suffix: str, data: url = f"{vm_url}{suffix}" async with session.post(url, json=data) as resp: resp.raise_for_status() - return await resp.model_dump_json() + return await resp.json() async def check_index(session: ClientSession, vm_id: ItemHash) -> bool: diff --git a/src/aleph/vm/orchestrator/views/operator.py b/src/aleph/vm/orchestrator/views/operator.py index b40d87803..199d23733 100644 --- a/src/aleph/vm/orchestrator/views/operator.py +++ b/src/aleph/vm/orchestrator/views/operator.py @@ -322,12 +322,12 @@ async def operate_confidential_inject_secret(request: web.Request, authenticated Send secret to the VM and start it """ try: - data = await request.model_dump_json() + data = await request.json() params = InjectSecretParams.model_validate(data) except json.JSONDecodeError: return web.HTTPBadRequest(reason="Body is not valid JSON") except pydantic.ValidationError as error: - return web.json_response(data=error.model_dump_json(), status=web.HTTPBadRequest.status_code) + return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code) vm_hash = get_itemhash_or_400(request.match_info) pool: VmPool = request.app["vm_pool"] diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 1366b3310..7e289dca2 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -136,7 +136,7 @@ async def get_latest_amend(item_hash: str) -> str: async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() - result: str = await resp.model_dump_json() + result: str = await resp.json() assert isinstance(result, str) return result or item_hash diff --git a/tests/supervisor/test_status.py b/tests/supervisor/test_status.py index f212fc478..3197133f0 100644 --- a/tests/supervisor/test_status.py +++ b/tests/supervisor/test_status.py @@ -13,13 +13,13 @@ async def test_check_internet_wrong_result_code(): mock_session = Mock() mock_session.get = MagicMock() - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 200, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is True - mock_session.get.return_value.__aenter__.return_value.model_dump_json = AsyncMock( + mock_session.get.return_value.__aenter__.return_value.json = AsyncMock( return_value={"result": 400, "headers": {"Server": "nginx"}} ) assert await check_internet(mock_session, vm_id) is False diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index 613560db3..3e9c4282e 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -10,6 +10,7 @@ from aleph.vm.orchestrator.supervisor import setup_webapp from aleph.vm.sevclient import SevClient + @pytest.mark.asyncio async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): """Test that the allocation endpoint fails when an invalid item_hash is provided.""" @@ -27,14 +28,13 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): "loc": ["persistent_vms", 0], "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", "type": "value_error", - "ctx": { - "error": "Could not determine hash type: 'not-an-ItemHash'" - }, + "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, "input": "not-an-ItemHash", - "url": "https://errors.pydantic.dev/2.9/v/value_error" + "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] + @pytest.mark.asyncio async def test_system_usage(aiohttp_client): """Test that the usage system endpoints responds. No auth needed""" diff --git a/vm_connector/conf.py b/vm_connector/conf.py index d0b34eab1..8164c0320 100644 --- a/vm_connector/conf.py +++ b/vm_connector/conf.py @@ -27,11 +27,8 @@ def display(self) -> str: f"{annotation:<17} = {getattr(self, annotation)}" for annotation, value in self.__annotations__.items() ) - model_config = ConfigDict( - env_prefix = "ALEPH_", - case_sensitive = False, - env_file = ".env" - ) + model_config = ConfigDict(env_prefix="ALEPH_", case_sensitive=False, env_file=".env") + # Settings singleton settings = ConnectorSettings() From 047dc044f4bb19b5ff9abc87642471d4cc9d6ccb Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:32:02 +0000 Subject: [PATCH 59/91] fix: Additional slash not needed in the url Adding a / with the url makes the url containing two slashes which break the url Removing it --- src/aleph/vm/storage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aleph/vm/storage.py b/src/aleph/vm/storage.py index 7e289dca2..7e15cc655 100644 --- a/src/aleph/vm/storage.py +++ b/src/aleph/vm/storage.py @@ -132,7 +132,7 @@ async def get_latest_amend(item_hash: str) -> str: if settings.FAKE_DATA_PROGRAM: return item_hash else: - url = f"{settings.CONNECTOR_URL}/compute/latest_amend/{item_hash}" + url = f"{settings.CONNECTOR_URL}compute/latest_amend/{item_hash}" async with aiohttp.ClientSession() as session: resp = await session.get(url) resp.raise_for_status() @@ -150,7 +150,7 @@ async def get_message(ref: str) -> ProgramMessage | InstanceMessage: logger.debug("Using the fake data message") else: cache_path = (Path(settings.MESSAGE_CACHE) / ref).with_suffix(".json") - url = f"{settings.CONNECTOR_URL}/download/message/{ref}" + url = f"{settings.CONNECTOR_URL}download/message/{ref}" await download_file(url, cache_path) with open(cache_path) as cache_file: @@ -186,7 +186,7 @@ async def get_code_path(ref: str) -> Path: raise ValueError(msg) cache_path = Path(settings.CODE_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/code/{ref}" + url = f"{settings.CONNECTOR_URL}download/code/{ref}" await download_file(url, cache_path) return cache_path @@ -198,7 +198,7 @@ async def get_data_path(ref: str) -> Path: return Path(f"{data_dir}.zip") cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) return cache_path @@ -219,7 +219,7 @@ async def get_runtime_path(ref: str) -> Path: return Path(settings.FAKE_DATA_RUNTIME) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" if not cache_path.is_file(): # File does not exist, download it @@ -237,7 +237,7 @@ async def get_rootfs_base_path(ref: ItemHash) -> Path: return Path(settings.FAKE_INSTANCE_BASE) cache_path = Path(settings.RUNTIME_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/runtime/{ref}" + url = f"{settings.CONNECTOR_URL}download/runtime/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path @@ -359,7 +359,7 @@ async def get_existing_file(ref: str) -> Path: return Path(settings.FAKE_DATA_VOLUME) cache_path = Path(settings.DATA_CACHE) / ref - url = f"{settings.CONNECTOR_URL}/download/data/{ref}" + url = f"{settings.CONNECTOR_URL}download/data/{ref}" await download_file(url, cache_path) await chown_to_jailman(cache_path) return cache_path From 1fb890116d2203b64602b85423127784d1ae7f56 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 11 Nov 2024 06:35:31 +0000 Subject: [PATCH 60/91] fix: Forcing the version 2 of pydantic --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 783d98950..4c650ee45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "packaging==23.2", "psutil==5.9.5", "py-cpuinfo==9", + "pydantic>=2", "pydantic-settings", "pyroute2==0.7.12", "python-cpuid==0.1", @@ -120,8 +121,9 @@ dependencies = [ "mypy==1.8.0", "ruff==0.4.6", "isort==5.13.2", - "yamlfix==1.16.1", + "yamlfix==1.17.0", "pyproject-fmt==2.2.1", + "pydantic>=2", ] [tool.hatch.envs.linting.scripts] typing = "mypy {args:src/aleph/vm/ tests/ examples/example_fastapi runtimes/aleph-debian-12-python}" From 4e544440510326cc93ebf52bef0794d8739f5b6c Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:35:50 +0000 Subject: [PATCH 61/91] Fix: HttpUrl no longer need scheme value --- examples/example_fastapi/main.py | 6 +++--- src/aleph/vm/models.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/example_fastapi/main.py b/examples/example_fastapi/main.py index 44caaf458..59b216e38 100644 --- a/examples/example_fastapi/main.py +++ b/examples/example_fastapi/main.py @@ -215,9 +215,9 @@ async def check_url(internet_host: HttpUrl, timeout_seconds: int = 5): async def read_internet(): """Check Internet connectivity of the system, requiring IP connectivity, domain resolution and HTTPS/TLS.""" internet_hosts: list[HttpUrl] = [ - HttpUrl(url="https://aleph.im/", scheme="https"), - HttpUrl(url="https://ethereum.org", scheme="https"), - HttpUrl(url="https://ipfs.io/", scheme="https"), + HttpUrl(url="https://aleph.im/"), + HttpUrl(url="https://ethereum.org/"), + HttpUrl(url="https://ipfs.io/"), ] timeout_seconds = 5 diff --git a/src/aleph/vm/models.py b/src/aleph/vm/models.py index 57089f3be..eee891a7e 100644 --- a/src/aleph/vm/models.py +++ b/src/aleph/vm/models.py @@ -440,7 +440,7 @@ async def save(self): async def record_usage(self): await delete_record(execution_uuid=str(self.uuid)) if settings.EXECUTION_LOG_ENABLED: - await save_execution_data(execution_uuid=self.uuid, execution_data=self.to.model_dump_json()) + await save_execution_data(execution_uuid=self.uuid, execution_data=self.to_json()) async def run_code(self, scope: dict | None = None) -> bytes: if not self.vm: From 3067750c023da15ddfd8b6e9f31a2c01fb38b7ef Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 00:37:42 +0000 Subject: [PATCH 62/91] Fix: Pydantic 2 handle differently the errors --- src/aleph/vm/orchestrator/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index 996584497..3af5766c0 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -16,6 +16,7 @@ parse_message, ) from aleph_message.status import MessageStatus +from pydantic import ValidationError from yarl import URL from aleph.vm.conf import settings @@ -73,10 +74,10 @@ async def subscribe_via_ws(url) -> AsyncIterable[AlephMessage]: try: yield parse_message(data) - except pydantic.error_wrappers.ValidationError as error: + except pydantic.ValidationError as error: item_hash = data.get("item_hash", "ITEM_HASH_NOT_FOUND") logger.warning( - f"Invalid Aleph message: {item_hash} \n {error.model_dump_json()}\n {error.raw_errors}", + f"Invalid Aleph message: {item_hash} \n {error.errors}", exc_info=False, ) continue From 19666b99ff35761b9c351c5691ef567c86a8e094 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 12 Nov 2024 04:55:12 +0000 Subject: [PATCH 63/91] Refactor: update validation for Pydantic v2 Changed @root_validator to @model_validator(mode="after") for checking data after the model is created. Updated values to use a more flexible type for Pydantic v2. Accessed values as a dictionary (values["payload"]) instead of using values.payload. Correct import statement from '__futures__' to '__future__' for postponed annotations --- .../vm/orchestrator/views/authentication.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index 21c7a6ac1..9ba4b12f9 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -5,6 +5,8 @@ Can be enabled on an endpoint using the @require_jwk_authentication decorator """ +from __future__ import annotations + # Keep datetime import as is as it allow patching in test import datetime import functools @@ -22,7 +24,13 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import BaseModel, ValidationError, field_validator, model_validator +from pydantic import ( + BaseModel, + ValidationError, + ValidationInfo, + field_validator, + model_validator, +) from solathon.utils import verify_signature from aleph.vm.conf import settings @@ -103,22 +111,19 @@ def payload_must_be_hex(cls, v: bytes) -> bytes: return bytes.fromhex(v.decode()) @model_validator(mode="after") - @classmethod - def check_expiry(cls, values) -> dict[str, bytes]: + def check_expiry(values) -> SignedPubKeyHeader: """Check that the token has not expired""" - payload: bytes = values.payload + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) if not is_token_still_valid(content.expires): - msg = "Token expired" - raise ValueError(msg) + raise ValueError("Token expired") return values @model_validator(mode="after") - @classmethod - def check_signature(cls, values) -> dict[str, bytes]: + def check_signature(values) -> SignedPubKeyHeader: """Check that the signature is valid""" - signature: list = values.signature - payload: bytes = values.payload + signature = values.signature + payload = values.payload content = SignedPubKeyPayload.model_validate_json(payload) check_wallet_signature_or_raise(content.address, content.chain, payload, signature) return values From 40d7b7e9fc1ce16e5514b456a4cba6a58820df73 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 14 Nov 2024 09:39:28 +0000 Subject: [PATCH 64/91] Fix: Missing dependency in the debian Makefile When adding a depedency inside the pyproject, we need to add it as well in the makefile. --- packaging/Makefile | 2 +- pyproject.toml | 2 +- src/aleph/vm/orchestrator/run.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packaging/Makefile b/packaging/Makefile index 0d1c4dcb9..76f9ba4eb 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -15,7 +15,7 @@ debian-package-code: cp ../examples/instance_message_from_aleph.json ./aleph-vm/opt/aleph-vm/examples/instance_message_from_aleph.json cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'aleph-message==0.4.9' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl diff --git a/pyproject.toml b/pyproject.toml index 4c650ee45..31d4676f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "psutil==5.9.5", "py-cpuinfo==9", "pydantic>=2", - "pydantic-settings", + "pydantic-settings==2.6.1", "pyroute2==0.7.12", "python-cpuid==0.1", "python-dotenv", diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index a2a2a824f..d95c40938 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -1,4 +1,5 @@ import asyncio +import json import logging from typing import Any @@ -55,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {message.json(indent=4, sort_keys=True, exclude_none=True)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 9a50916576c76d4714c8494d6a90664c2ab2512a Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 18 Nov 2024 06:30:22 +0000 Subject: [PATCH 65/91] fix: datetime not seriazable --- src/aleph/vm/orchestrator/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index d95c40938..5708c162d 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -56,7 +56,7 @@ async def create_vm_execution(vm_hash: ItemHash, pool: VmPool, persistent: bool message, original_message = await load_updated_message(vm_hash) pool.message_cache[vm_hash] = message - logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True)}") + logger.debug(f"Message: {json.dumps(message.dict(exclude_none=True), indent=4, sort_keys=True, default=str)}") execution = await pool.create_a_vm( vm_hash=vm_hash, From 9d0e30ec67f20beeb5286d61183699eb85ce9ce6 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:55:00 +0000 Subject: [PATCH 66/91] Fix: Adaptations for Pydantic v2 migration and stricter validation - Improved error handling in `get_signed_pubkey` to align with stricter JSON validation and date format requirements in Pydantic v2. - Addressed warnings and deprecated functions raised during the migration to Pydantic v2. - Updated type annotations to comply with Pydantic's stricter type checks (`url` now requires proper type validation). - Refactored test mocks to use `.model_dump_json()` instead of `.json()` for serialization - Adjusted test cases to account for new fields in validation errors (`url`, `ctx`, and `input`) introduced in Pydantic v2. - Applied linting for style consistency across tests and codebase. fix: yamlfix and pydantic are incompatible Fix: HttpUrl no longer need scheme value From 330dfd626d1d86d5b0aed7b84dfb4547a7299477 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Thu, 31 Oct 2024 14:48:19 +0100 Subject: [PATCH 67/91] Make vm_id assignment more robust (#714) Remove the counter way to assign a vm_id as it didn't work reliably Jira ticket: ALEPH-272 That method was broken when persitent instances were loaded at start up. Since the "new" feature that allow persistent instance across aleph-vm reboot if one was started then aleph-vm was stopped and restarted the counter method could reassign the ip and break the existing vm's. Secundary reason was that the feature wasn't working properly with the default settings, as `2**available_bits` returned 1. So that code path was only used if the node owner tweaked some undocumented settings making it hard to identify and debug in prod nodes. --- src/aleph/vm/pool.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py index 3ecf500eb..025bfe45c 100644 --- a/src/aleph/vm/pool.py +++ b/src/aleph/vm/pool.py @@ -28,15 +28,13 @@ class VmPool: - """Pool of VMs already started and used to decrease response time. + """Pool of existing VMs + + For function VM we keep the VM a while after they have run, so we can reuse them and thus decrease response time. After running, a VM is saved for future reuse from the same function during a configurable duration. - - The counter is used by the VMs to set their tap interface name and the corresponding - IPv4 subnet. """ - counter: int # Used to provide distinct ids to network interfaces executions: dict[ItemHash, VmExecution] message_cache: dict[str, ExecutableMessage] network: Network | None @@ -45,7 +43,6 @@ class VmPool: creation_lock: asyncio.Lock def __init__(self, loop: asyncio.AbstractEventLoop): - self.counter = settings.START_ID_INDEX self.executions = {} self.message_cache = {} @@ -150,25 +147,13 @@ def get_unique_vm_id(self) -> int: This identifier is used to name the network interface and in the IPv4 range dedicated to the VM. """ - _, network_range = settings.IPV4_ADDRESS_POOL.split("/") - available_bits = int(network_range) - settings.IPV4_NETWORK_PREFIX_LENGTH - self.counter += 1 - if self.counter < 2**available_bits: - # In common cases, use the counter itself as the vm_id. This makes it - # easier to debug. - return self.counter - else: - # The value of the counter is too high and some functions such as the - # IPv4 range dedicated to the VM do not support such high values. - # - # We therefore recycle vm_id values from executions that are not running - # anymore. - currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()} - for i in range(settings.START_ID_INDEX, 255**2): - if i not in currently_used_vm_ids: - return i - msg = "No available value for vm_id." - raise ValueError(msg) + # Take the first id that is not already taken + currently_used_vm_ids = {execution.vm_id for execution in self.executions.values()} + for i in range(settings.START_ID_INDEX, 255**2): + if i not in currently_used_vm_ids: + return i + msg = "No available value for vm_id." + raise ValueError(msg) def get_running_vm(self, vm_hash: ItemHash) -> VmExecution | None: """Return a running VM or None. Disables the VM expiration task.""" From e98c2a8ad4e0dd399e6f1777d5ea82d43dece457 Mon Sep 17 00:00:00 2001 From: nesitor Date: Mon, 4 Nov 2024 17:00:30 +0100 Subject: [PATCH 68/91] Implement new EVM chains (#717) * Feature: Implement new EVM chains. * FIX: Update Makefile with new dependency. * Fix: Updated to proper released package version of aleph_message dependency. --------- Co-authored-by: Andres D. Molins --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index 76f9ba4eb..c3b6511a0 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -144,6 +144,6 @@ repository-noble: cd ./repositories/noble && reprepro -Vb . includedeb noble ../../target/aleph-vm.ubuntu-24.04.deb && cd .. repositories: repository-bookworm repository-jammy repository-noble - + all-podman: all-podman-debian-12 all-podman-ubuntu-2204 all-podman-ubuntu-2404 repositories From e08a47828dd64d26386f99c007abd1d3adeb73d5 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Tue, 5 Nov 2024 15:30:46 +0100 Subject: [PATCH 69/91] Feature: allow IPv6 DNS (#455) * Feature: allow IPv6 DNS Problem IPv6 DNS were automatically filtered when detected from resolvectl Solution: Nameservers are now split into ipv4 and ipv6 and can be passed to the VM accordingly At the moment we pass them if the ipv6 parameter is present on the tap interface but we need a more robust detection method * Display proper env conf --- src/aleph/vm/conf.py | 36 +++++++++++-------- .../vm/controllers/firecracker/instance.py | 7 +++- .../supervisor/test_resolvectl_dns_servers.py | 8 +---- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 45d2d611a..7c69f3442 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -76,17 +76,6 @@ def resolvectl_dns_servers(interface: str) -> Iterable[str]: yield server.strip() -def resolvectl_dns_servers_ipv4(interface: str) -> Iterable[str]: - """ - Use resolvectl to list available IPv4 DNS servers. - VMs only support IPv4 networking for now, we must exclude IPv6 DNS from their config. - """ - for server in resolvectl_dns_servers(interface): - ip_addr = ipaddress.ip_address(server) - if isinstance(ip_addr, ipaddress.IPv4Address): - yield server - - def get_default_interface() -> str | None: """Returns the default network interface""" with open("/proc/net/route") as f: @@ -104,7 +93,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st # Use a try-except approach since resolvectl can be present but disabled and raise the following # "Failed to get global data: Unit dbus-org.freedesktop.resolve1.service not found." try: - return list(resolvectl_dns_servers_ipv4(interface=network_interface)) + return list(resolvectl_dns_servers(interface=network_interface)) except (FileNotFoundError, CalledProcessError) as error: if Path("/etc/resolv.conf").exists(): return list(etc_resolv_conf_dns_servers()) @@ -116,7 +105,7 @@ def obtain_dns_ips(dns_resolver: DnsResolver, network_interface: str) -> list[st return list(etc_resolv_conf_dns_servers()) elif dns_resolver == DnsResolver.resolvectl: - return list(resolvectl_dns_servers_ipv4(interface=network_interface)) + return list(resolvectl_dns_servers(interface=network_interface)) else: msg = "No DNS resolve defined, this should never happen." @@ -182,8 +171,13 @@ class Settings(BaseSettings): description="Use the Neighbor Discovery Protocol Proxy to respond to Router Solicitation for instances on IPv6", ) - DNS_RESOLUTION: DnsResolver | None = DnsResolver.detect + DNS_RESOLUTION: DnsResolver | None = Field( + default=DnsResolver.detect, + description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.", + ) DNS_NAMESERVERS: list[str] | None = None + DNS_NAMESERVERS_IPV4: list[str] | None + DNS_NAMESERVERS_IPV6: list[str] | None FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") JAILER_PATH: Path = Path("/opt/firecracker/jailer") @@ -445,6 +439,18 @@ def setup(self): network_interface=self.NETWORK_INTERFACE, ) + if not self.DNS_NAMESERVERS_IPV4: + self.DNS_NAMESERVERS_IPV4 = [] + if not self.DNS_NAMESERVERS_IPV6: + self.DNS_NAMESERVERS_IPV6 = [] + if self.DNS_NAMESERVERS: + for server in self.DNS_NAMESERVERS: + ip_addr = ipaddress.ip_address(server) + if isinstance(ip_addr, ipaddress.IPv4Address): + self.DNS_NAMESERVERS_IPV4.append(server) + if isinstance(ip_addr, ipaddress.IPv6Address): + self.DNS_NAMESERVERS_IPV6.append(server) + if not settings.ENABLE_QEMU_SUPPORT: # If QEmu is not supported, ignore the setting and use Firecracker by default settings.INSTANCE_DEFAULT_HYPERVISOR = HypervisorType.firecracker @@ -462,7 +468,7 @@ def display(self) -> str: else: attributes[attr] = getattr(self, attr) - return "\n".join(f"{attribute:<27} = {value}" for attribute, value in attributes.items()) + return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) def __init__( self, diff --git a/src/aleph/vm/controllers/firecracker/instance.py b/src/aleph/vm/controllers/firecracker/instance.py index f8c33b075..da423ef73 100644 --- a/src/aleph/vm/controllers/firecracker/instance.py +++ b/src/aleph/vm/controllers/firecracker/instance.py @@ -198,6 +198,11 @@ def _create_network_file(self) -> bytes: ipv6 = self.get_ipv6() ipv6_gateway = self.get_ipv6_gateway() + nameservers_ip = [] + if ip: + nameservers_ip = settings.DNS_NAMESERVERS_IPV4 + if ipv6: + nameservers_ip += settings.DNS_NAMESERVERS_IPV6 network = { "ethernets": { "eth0": { @@ -207,7 +212,7 @@ def _create_network_file(self) -> bytes: "gateway4": route, "gateway6": ipv6_gateway, "nameservers": { - "addresses": settings.DNS_NAMESERVERS, + "addresses": nameservers_ip, }, }, }, diff --git a/tests/supervisor/test_resolvectl_dns_servers.py b/tests/supervisor/test_resolvectl_dns_servers.py index 0daaf03c4..0af9b6fb8 100644 --- a/tests/supervisor/test_resolvectl_dns_servers.py +++ b/tests/supervisor/test_resolvectl_dns_servers.py @@ -2,7 +2,7 @@ import os from unittest import mock -from aleph.vm.conf import resolvectl_dns_servers, resolvectl_dns_servers_ipv4 +from aleph.vm.conf import resolvectl_dns_servers os.environ["ALEPH_VM_ALLOW_VM_NETWORKING"] = "False" @@ -17,9 +17,6 @@ def test_resolvectl(): dns_servers = set(resolvectl_dns_servers("eth0")) assert dns_servers == servers - dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0")) - assert dns_servers_ipv4 == servers - def test_resolvectl_ipv6(): with mock.patch( @@ -31,6 +28,3 @@ def test_resolvectl_ipv6(): dns_servers = set(resolvectl_dns_servers("eth0")) assert dns_servers == ipv4_servers | ipv6_servers - - dns_servers_ipv4 = set(resolvectl_dns_servers_ipv4("eth0")) - assert dns_servers_ipv4 == ipv4_servers From 60132a46626660841d3d85c415468cbd9357d71d Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Wed, 6 Nov 2024 14:54:42 +0100 Subject: [PATCH 70/91] Problem: IGNORE_TRACEBACK_FROM_DIAGNOSTICS broken (#713) Symptom: The CustomError from the diagnostics VM was printed even if if IGNORE_TRACEBACK_FROM_DIAGNOSTICS was set to True (the default) Analysis: This was caused by the refactoring of the fastapi_example/main.py file done in fe9235ac658915eea20d5371ae45cedabe1f7b17 Which changed the output used to detect the error to catch Solution: Fix detection string --- src/aleph/vm/orchestrator/run.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/orchestrator/run.py b/src/aleph/vm/orchestrator/run.py index 5708c162d..c3da5ad8b 100644 --- a/src/aleph/vm/orchestrator/run.py +++ b/src/aleph/vm/orchestrator/run.py @@ -151,9 +151,11 @@ async def run_code_on_request(vm_hash: ItemHash, path: str, pool: VmPool, reques # The Diagnostics VM checks for the proper handling of exceptions. # This fills the logs with noisy stack traces, so we ignore this specific error. - ignored_error = 'raise CustomError("Whoops")' + ignored_errors = ['raise CustomError("Whoops")', "main.CustomError: Whoops"] - if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and ignored_error in result["traceback"]: + if settings.IGNORE_TRACEBACK_FROM_DIAGNOSTICS and any( + ignored_error in result["traceback"] for ignored_error in ignored_errors + ): logger.debug('Ignored traceback from CustomError("Whoops")') else: logger.warning(result["traceback"]) From 5a7240c761885c99e34586b4cb7ba0ad9003b939 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Fri, 8 Nov 2024 15:17:09 +0100 Subject: [PATCH 71/91] Problem: error Too many open files (#720) Jira ticket: ALEPH-298 some CRN failed on any action with error OSError: [Errno 24] Too many open files: Solution: Properly close stream to journald when the VM is stopped --- .../vm/hypervisors/firecracker/microvm.py | 41 +++++++++++++------ src/aleph/vm/hypervisors/qemu/qemuvm.py | 17 +++++--- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/aleph/vm/hypervisors/firecracker/microvm.py b/src/aleph/vm/hypervisors/firecracker/microvm.py index 4421491e5..1d34e2752 100644 --- a/src/aleph/vm/hypervisors/firecracker/microvm.py +++ b/src/aleph/vm/hypervisors/firecracker/microvm.py @@ -13,7 +13,7 @@ from pathlib import Path from pwd import getpwnam from tempfile import NamedTemporaryFile -from typing import Any +from typing import Any, BinaryIO import msgpack from aleph_message.models import ItemHash @@ -93,6 +93,8 @@ class MicroVM: mounted_rootfs: Path | None = None _unix_socket: Server | None = None enable_log: bool + journal_stdout: BinaryIO | int | None = None + journal_stderr: BinaryIO | int | None = None def __repr__(self): return f"" @@ -219,19 +221,19 @@ async def start_firecracker(self, config_path: Path) -> asyncio.subprocess.Proce str(config_path), ) if self.enable_log: - journal_stdout = journal.stream(self._journal_stdout_name) - journal_stderr = journal.stream(self._journal_stderr_name) + self.journal_stdout = journal.stream(self._journal_stdout_name) + self.journal_stderr = journal.stream(self._journal_stderr_name) else: - journal_stdout = asyncio.subprocess.DEVNULL - journal_stderr = asyncio.subprocess.DEVNULL + self.journal_stdout = asyncio.subprocess.DEVNULL + self.journal_stderr = asyncio.subprocess.DEVNULL logger.debug(" ".join(options)) self.proc = await asyncio.create_subprocess_exec( *options, stdin=asyncio.subprocess.PIPE, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) return self.proc @@ -252,11 +254,11 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces self.config_file_path = config_path if self.enable_log: - journal_stdout = journal.stream(self._journal_stdout_name) - journal_stderr = journal.stream(self._journal_stderr_name) + self.journal_stdout = journal.stream(self._journal_stdout_name) + self.journal_stderr = journal.stream(self._journal_stderr_name) else: - journal_stdout = asyncio.subprocess.DEVNULL - journal_stderr = asyncio.subprocess.DEVNULL + self.journal_stdout = asyncio.subprocess.DEVNULL + self.journal_stderr = asyncio.subprocess.DEVNULL options = ( str(self.jailer_bin_path), @@ -280,8 +282,8 @@ async def start_jailed_firecracker(self, config_path: Path) -> asyncio.subproces self.proc = await asyncio.create_subprocess_exec( *options, stdin=asyncio.subprocess.PIPE, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) return self.proc @@ -480,6 +482,19 @@ async def teardown(self): if self.stderr_task: self.stderr_task.cancel() + if ( + self.journal_stdout + and self.journal_stdout != asyncio.subprocess.DEVNULL + and hasattr(self.journal_stdout, "close") + ): + self.journal_stdout.close() + if ( + self.journal_stderr + and self.journal_stderr != asyncio.subprocess.DEVNULL + and hasattr(self.journal_stderr, "close") + ): + self.journal_stderr.close() + # Clean mounted block devices if self.mounted_rootfs: logger.debug("Waiting for one second for the VM to shutdown") diff --git a/src/aleph/vm/hypervisors/qemu/qemuvm.py b/src/aleph/vm/hypervisors/qemu/qemuvm.py index 1d707c2a5..5949fbdc4 100644 --- a/src/aleph/vm/hypervisors/qemu/qemuvm.py +++ b/src/aleph/vm/hypervisors/qemu/qemuvm.py @@ -2,7 +2,7 @@ from asyncio.subprocess import Process from dataclasses import dataclass from pathlib import Path -from typing import TextIO +from typing import BinaryIO, TextIO import qmp from systemd import journal @@ -28,6 +28,8 @@ class QemuVM: interface_name: str qemu_process: Process | None = None host_volumes: list[HostVolume] + journal_stdout: TextIO | None + journal_stderr: TextIO | None def __repr__(self) -> str: if self.qemu_process: @@ -72,8 +74,8 @@ async def start( # qemu-system-x86_64 -enable-kvm -m 2048 -net nic,model=virtio # -net tap,ifname=tap0,script=no,downscript=no -drive file=alpine.qcow2,media=disk,if=virtio -nographic - journal_stdout: TextIO = journal.stream(self._journal_stdout_name) - journal_stderr: TextIO = journal.stream(self._journal_stderr_name) + self.journal_stdout: BinaryIO = journal.stream(self._journal_stdout_name) + self.journal_stderr: BinaryIO = journal.stream(self._journal_stderr_name) # hardware_resources.published ports -> not implemented at the moment # hardware_resources.seconds -> only for microvm args = [ @@ -120,8 +122,8 @@ async def start( self.qemu_process = proc = await asyncio.create_subprocess_exec( *args, stdin=asyncio.subprocess.DEVNULL, - stdout=journal_stdout, - stderr=journal_stderr, + stdout=self.journal_stdout, + stderr=self.journal_stderr, ) print( @@ -149,3 +151,8 @@ def send_shutdown_message(self): async def stop(self): """Stop the VM.""" self.send_shutdown_message() + + if self.journal_stdout and self.journal_stdout != asyncio.subprocess.DEVNULL: + self.journal_stdout.close() + if self.journal_stderr and self.journal_stderr != asyncio.subprocess.DEVNULL: + self.journal_stderr.close() From cb8f6604fc13daa1be84681aa27175d45ab9cb55 Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Thu, 14 Nov 2024 16:40:41 +0100 Subject: [PATCH 72/91] Update PULL_REQUEST_TEMPLATE.md for dependencies check (#722) Add a check for dependencies update --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index bcf764608..ff965a1de 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,6 +10,7 @@ Related ClickUp, GitHub or Jira tickets : ALEPH-XXX - [ ] New classes and functions contain docstrings explaining what they provide. - [ ] All new code is covered by relevant tests. - [ ] Documentation has been updated regarding these changes. +- [ ] Dependencies update in the project.toml have been mirrored in the Debian package build script `packaging/Makefile` ## Changes From bf324dc436520877114f92705f8ed3237d32e9cd Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 04:41:58 +0000 Subject: [PATCH 73/91] Fix: Pydantic is more strick about validation rules Pydantic's validation system now enforces strict checks, we cannot leave fields marked as required empty --- src/aleph/vm/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 7c69f3442..8bc347746 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -176,8 +176,8 @@ class Settings(BaseSettings): description="Method used to resolve the dns server if DNS_NAMESERVERS is not present.", ) DNS_NAMESERVERS: list[str] | None = None - DNS_NAMESERVERS_IPV4: list[str] | None - DNS_NAMESERVERS_IPV6: list[str] | None + DNS_NAMESERVERS_IPV4: list[str] | None = None + DNS_NAMESERVERS_IPV6: list[str] | None = None FIRECRACKER_PATH: Path = Path("/opt/firecracker/firecracker") JAILER_PATH: Path = Path("/opt/firecracker/jailer") From 7911a6478b1d438de497a01fa2a2bd72c56a9000 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Mon, 4 Nov 2024 16:55:00 +0000 Subject: [PATCH 74/91] Fix: Adaptations for Pydantic v2 migration and stricter validation - Updated the display() method to use self.model_config.env_prefix instead of self.Config.env_prefix. The previous implementation caused an AttributeError because Config is no longer supported in Pydantic v2. Adjusted the method to align with the new Pydantic v2 configuration approach, which uses model_config for settings validation and customization. --- src/aleph/vm/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 8bc347746..6181aad80 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -468,7 +468,9 @@ def display(self) -> str: else: attributes[attr] = getattr(self, attr) - return "\n".join(f"{self.Config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items()) + return "\n".join( + f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items() + ) def __init__( self, From 950d1cdbd4a1198542dfbb5d1327cff34d4bec20 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 19 Nov 2024 08:31:55 +0000 Subject: [PATCH 75/91] fix: model_config is now a dict no longer a class --- src/aleph/vm/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 6181aad80..054b9d93d 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -467,9 +467,8 @@ def display(self) -> str: attributes[attr] = "" else: attributes[attr] = getattr(self, attr) - return "\n".join( - f"{self.model_config.env_prefix}{attribute} = {value}" for attribute, value in attributes.items() + f"{self.model_config.get('env_prefix', '')}{attribute} = {value}" for attribute, value in attributes.items() ) def __init__( From 89427228364dc285366f286a393b5c29207a8694 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 21 Nov 2024 15:05:11 +0000 Subject: [PATCH 76/91] Fix: Duplicate line in pyproject and missing type annotation --- pyproject.toml | 3 --- src/aleph/vm/conf.py | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c47565922..60bb7b518 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,9 +205,6 @@ lint.ignore = [ # Tests can use magic values, assertions, and relative imports lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] - [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 3bb64ccdd..bd6368c4d 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -136,10 +136,10 @@ class Settings(BaseSettings): USE_JAILER: bool = True # System logs make boot ~2x slower - PRINT_SYSTEM_LOGS = False - IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True - LOG_LEVEL = "WARNING" - DEBUG_ASYNCIO = False + PRINT_SYSTEM_LOGS: bool = False + IGNORE_TRACEBACK_FROM_DIAGNOSTICS: bool = True + LOG_LEVEL: str = "WARNING" + DEBUG_ASYNCIO: bool = False # Networking does not work inside Docker/Podman ALLOW_VM_NETWORKING: bool = True From ed0913728fb6d47aa52cb5286f91b6e7a451fd3e Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 21 Nov 2024 15:22:29 +0000 Subject: [PATCH 77/91] style mypy --- src/aleph/vm/conf.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index bd6368c4d..0dcd07438 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -189,16 +189,16 @@ class Settings(BaseSettings): CONNECTOR_URL: HttpUrl = HttpUrl("http://localhost:4021") CACHE_ROOT: Path = Path("/var/cache/aleph/vm") - MESSAGE_CACHE: Path = Field( + MESSAGE_CACHE: Optional[Path] = Field( None, description="Default to CACHE_ROOT/message", ) - CODE_CACHE: Path = Field(None, description="Default to CACHE_ROOT/code") - RUNTIME_CACHE: Path = Field(None, description="Default to CACHE_ROOT/runtime") - DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data") + CODE_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/code") + RUNTIME_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/runtime") + DATA_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/data") EXECUTION_ROOT: Path = Path("/var/lib/aleph/vm") - JAILER_BASE_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/jailer") + JAILER_BASE_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/jailer") EXECUTION_DATABASE: Path = Field( None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3" ) @@ -210,7 +210,7 @@ class Settings(BaseSettings): PERSISTENT_VOLUMES_DIR: Path = Field( None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/" ) - JAILER_BASE_DIR: Path = Field(None) + JAILER_BASE_DIR: Optional[Path] = Field(None) MAX_PROGRAM_ARCHIVE_SIZE: int = 10_000_000 # 10 MB MAX_DATA_ARCHIVE_SIZE: int = 10_000_000 # 10 MB @@ -268,12 +268,12 @@ class Settings(BaseSettings): "with SEV and SEV-ES", ) - CONFIDENTIAL_DIRECTORY: Path = Field( + CONFIDENTIAL_DIRECTORY: Optional[Path] = Field( None, description="Confidential Computing default directory. Default to EXECUTION_ROOT/confidential", ) - CONFIDENTIAL_SESSION_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/sessions") + CONFIDENTIAL_SESSION_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/sessions") # Tests on programs From 93678f0c800efbe587b46df8c9f674222dc0a4a6 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 21 Nov 2024 15:31:19 +0000 Subject: [PATCH 78/91] style mypy --- src/aleph/vm/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 0dcd07438..8f2318c35 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -9,7 +9,7 @@ from os.path import abspath, exists, isdir, isfile, join from pathlib import Path from subprocess import CalledProcessError, check_output -from typing import Any, Literal, NewType +from typing import Any, Literal, NewType, Optional from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType @@ -199,15 +199,15 @@ class Settings(BaseSettings): EXECUTION_ROOT: Path = Path("/var/lib/aleph/vm") JAILER_BASE_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/jailer") - EXECUTION_DATABASE: Path = Field( + EXECUTION_DATABASE: Optional[Path] = Field( None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3" ) EXECUTION_LOG_ENABLED: bool = False - EXECUTION_LOG_DIRECTORY: Path = Field( + EXECUTION_LOG_DIRECTORY: Optional[Path] = Field( None, description="Location of executions log. Default to EXECUTION_ROOT/executions/" ) - PERSISTENT_VOLUMES_DIR: Path = Field( + PERSISTENT_VOLUMES_DIR: Optional[Path] = Field( None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/" ) JAILER_BASE_DIR: Optional[Path] = Field(None) From 7726eea9756791694cae3c606aabfc647f723641 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 22 Nov 2024 16:03:01 +0000 Subject: [PATCH 79/91] Fix `os.makedirs` calls for optional Path variables Previously, `os.makedirs` was called directly on variables like `MESSAGE_CACHE` or `CODE_CACHE`, which could be `None`. This caused issues with `mypy` and potential runtime errors since `os.makedirs` does not handle `None`. The fix ensures these variables are checked for `None` before calling `os.makedirs`, preventing invalid operations and aligning with type checks. --- src/aleph/vm/conf.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 8f2318c35..161d663be 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -403,10 +403,14 @@ def setup(self): STREAM_CHAINS[Chain.AVAX].rpc = str(self.RPC_AVAX) STREAM_CHAINS[Chain.BASE].rpc = str(self.RPC_BASE) - os.makedirs(self.MESSAGE_CACHE, exist_ok=True) - os.makedirs(self.CODE_CACHE, exist_ok=True) - os.makedirs(self.RUNTIME_CACHE, exist_ok=True) - os.makedirs(self.DATA_CACHE, exist_ok=True) + if self.MESSAGE_CACHE: + os.makedirs(self.MESSAGE_CACHE, exist_ok=True) + if self.CODE_CACHE: + os.makedirs(self.CODE_CACHE, exist_ok=True) + if self.TUNRIME_CACHE: + os.makedirs(self.RUNTIME_CACHE, exist_ok=True) + if self.DATA_CACHE: + os.makedirs(self.DATA_CACHE, exist_ok=True) os.makedirs(self.EXECUTION_ROOT, exist_ok=True) @@ -422,10 +426,14 @@ def setup(self): self.LINUX_PATH = linux_path_on_device - os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True) - os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True) - os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True) - os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True) + if self.EXECUTION_LOG_DIRECTORY: + os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True) + if self.PERSISTENT_VOLUMES_DIR: + os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True) + if self.CONFIDENTIAL_DIRECTORY: + os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True) + if self.CONFIDENTIAL_SESSION_DIRECTORY: + os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True) self.API_SERVER = self.API_SERVER.rstrip("/") From 8064e053f2de3e4e1ad2bbf78e221053730bd5fe Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 22 Nov 2024 16:24:56 +0000 Subject: [PATCH 80/91] fix: wrong name --- src/aleph/vm/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 161d663be..dc10f8ca7 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -407,7 +407,7 @@ def setup(self): os.makedirs(self.MESSAGE_CACHE, exist_ok=True) if self.CODE_CACHE: os.makedirs(self.CODE_CACHE, exist_ok=True) - if self.TUNRIME_CACHE: + if self.RUNTIME_CACHE: os.makedirs(self.RUNTIME_CACHE, exist_ok=True) if self.DATA_CACHE: os.makedirs(self.DATA_CACHE, exist_ok=True) From f8ce889001c2dd1cb35f0c5de3b206bc5b560c46 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 22 Nov 2024 17:06:00 +0000 Subject: [PATCH 81/91] ignore url in test_allocation_fails_on_invalid_item_hash --- tests/supervisor/test_views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index 3e9c4282e..5a82f9e8d 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -23,14 +23,17 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): assert response.status == 400 - assert await response.json() == [ + response = await response.json() + for error in response: + error.pop("url", None) + + assert response == [ { "loc": ["persistent_vms", 0], "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", "type": "value_error", "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, "input": "not-an-ItemHash", - "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] From 0ed352020a2da9b4460b8ad62ea5313caba8b331 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Thu, 21 Nov 2024 15:05:11 +0000 Subject: [PATCH 82/91] Fix: Duplicate line in pyproject and missing type annotation --- pyproject.toml | 3 --- src/aleph/vm/conf.py | 32 ++++++++++++++++---------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c47565922..60bb7b518 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,9 +205,6 @@ lint.ignore = [ # Tests can use magic values, assertions, and relative imports lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] -# Tests can use magic values, assertions, and relative imports -lint.per-file-ignores."tests/**/*" = [ "PLR2004", "S101", "TID252" ] - [tool.pytest.ini_options] pythonpath = [ "src", diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 3bb64ccdd..8f2318c35 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -9,7 +9,7 @@ from os.path import abspath, exists, isdir, isfile, join from pathlib import Path from subprocess import CalledProcessError, check_output -from typing import Any, Literal, NewType +from typing import Any, Literal, NewType, Optional from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType @@ -136,10 +136,10 @@ class Settings(BaseSettings): USE_JAILER: bool = True # System logs make boot ~2x slower - PRINT_SYSTEM_LOGS = False - IGNORE_TRACEBACK_FROM_DIAGNOSTICS = True - LOG_LEVEL = "WARNING" - DEBUG_ASYNCIO = False + PRINT_SYSTEM_LOGS: bool = False + IGNORE_TRACEBACK_FROM_DIAGNOSTICS: bool = True + LOG_LEVEL: str = "WARNING" + DEBUG_ASYNCIO: bool = False # Networking does not work inside Docker/Podman ALLOW_VM_NETWORKING: bool = True @@ -189,28 +189,28 @@ class Settings(BaseSettings): CONNECTOR_URL: HttpUrl = HttpUrl("http://localhost:4021") CACHE_ROOT: Path = Path("/var/cache/aleph/vm") - MESSAGE_CACHE: Path = Field( + MESSAGE_CACHE: Optional[Path] = Field( None, description="Default to CACHE_ROOT/message", ) - CODE_CACHE: Path = Field(None, description="Default to CACHE_ROOT/code") - RUNTIME_CACHE: Path = Field(None, description="Default to CACHE_ROOT/runtime") - DATA_CACHE: Path = Field(None, description="Default to CACHE_ROOT/data") + CODE_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/code") + RUNTIME_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/runtime") + DATA_CACHE: Optional[Path] = Field(None, description="Default to CACHE_ROOT/data") EXECUTION_ROOT: Path = Path("/var/lib/aleph/vm") - JAILER_BASE_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/jailer") - EXECUTION_DATABASE: Path = Field( + JAILER_BASE_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/jailer") + EXECUTION_DATABASE: Optional[Path] = Field( None, description="Location of database file. Default to EXECUTION_ROOT/executions.sqlite3" ) EXECUTION_LOG_ENABLED: bool = False - EXECUTION_LOG_DIRECTORY: Path = Field( + EXECUTION_LOG_DIRECTORY: Optional[Path] = Field( None, description="Location of executions log. Default to EXECUTION_ROOT/executions/" ) - PERSISTENT_VOLUMES_DIR: Path = Field( + PERSISTENT_VOLUMES_DIR: Optional[Path] = Field( None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/" ) - JAILER_BASE_DIR: Path = Field(None) + JAILER_BASE_DIR: Optional[Path] = Field(None) MAX_PROGRAM_ARCHIVE_SIZE: int = 10_000_000 # 10 MB MAX_DATA_ARCHIVE_SIZE: int = 10_000_000 # 10 MB @@ -268,12 +268,12 @@ class Settings(BaseSettings): "with SEV and SEV-ES", ) - CONFIDENTIAL_DIRECTORY: Path = Field( + CONFIDENTIAL_DIRECTORY: Optional[Path] = Field( None, description="Confidential Computing default directory. Default to EXECUTION_ROOT/confidential", ) - CONFIDENTIAL_SESSION_DIRECTORY: Path = Field(None, description="Default to EXECUTION_ROOT/sessions") + CONFIDENTIAL_SESSION_DIRECTORY: Optional[Path] = Field(None, description="Default to EXECUTION_ROOT/sessions") # Tests on programs From cd8814444f14f43ca6dcd6d943ed02b75311d7bb Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 22 Nov 2024 16:03:01 +0000 Subject: [PATCH 83/91] Fix `os.makedirs` calls for optional Path variables Previously, `os.makedirs` was called directly on variables like `MESSAGE_CACHE` or `CODE_CACHE`, which could be `None`. This caused issues with `mypy` and potential runtime errors since `os.makedirs` does not handle `None`. The fix ensures these variables are checked for `None` before calling `os.makedirs`, preventing invalid operations and aligning with type checks. fix: wrong name --- src/aleph/vm/conf.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index 8f2318c35..dc10f8ca7 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -403,10 +403,14 @@ def setup(self): STREAM_CHAINS[Chain.AVAX].rpc = str(self.RPC_AVAX) STREAM_CHAINS[Chain.BASE].rpc = str(self.RPC_BASE) - os.makedirs(self.MESSAGE_CACHE, exist_ok=True) - os.makedirs(self.CODE_CACHE, exist_ok=True) - os.makedirs(self.RUNTIME_CACHE, exist_ok=True) - os.makedirs(self.DATA_CACHE, exist_ok=True) + if self.MESSAGE_CACHE: + os.makedirs(self.MESSAGE_CACHE, exist_ok=True) + if self.CODE_CACHE: + os.makedirs(self.CODE_CACHE, exist_ok=True) + if self.RUNTIME_CACHE: + os.makedirs(self.RUNTIME_CACHE, exist_ok=True) + if self.DATA_CACHE: + os.makedirs(self.DATA_CACHE, exist_ok=True) os.makedirs(self.EXECUTION_ROOT, exist_ok=True) @@ -422,10 +426,14 @@ def setup(self): self.LINUX_PATH = linux_path_on_device - os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True) - os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True) - os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True) - os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True) + if self.EXECUTION_LOG_DIRECTORY: + os.makedirs(self.EXECUTION_LOG_DIRECTORY, exist_ok=True) + if self.PERSISTENT_VOLUMES_DIR: + os.makedirs(self.PERSISTENT_VOLUMES_DIR, exist_ok=True) + if self.CONFIDENTIAL_DIRECTORY: + os.makedirs(self.CONFIDENTIAL_DIRECTORY, exist_ok=True) + if self.CONFIDENTIAL_SESSION_DIRECTORY: + os.makedirs(self.CONFIDENTIAL_SESSION_DIRECTORY, exist_ok=True) self.API_SERVER = self.API_SERVER.rstrip("/") From d96e9cac101b51e3fcbac3b217c695eec88d1f22 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Fri, 22 Nov 2024 17:06:00 +0000 Subject: [PATCH 84/91] ignore url in test_allocation_fails_on_invalid_item_hash --- tests/supervisor/test_views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/supervisor/test_views.py b/tests/supervisor/test_views.py index 3e9c4282e..5a82f9e8d 100644 --- a/tests/supervisor/test_views.py +++ b/tests/supervisor/test_views.py @@ -23,14 +23,17 @@ async def test_allocation_fails_on_invalid_item_hash(aiohttp_client): assert response.status == 400 - assert await response.json() == [ + response = await response.json() + for error in response: + error.pop("url", None) + + assert response == [ { "loc": ["persistent_vms", 0], "msg": "Value error, Could not determine hash type: 'not-an-ItemHash'", "type": "value_error", "ctx": {"error": "Could not determine hash type: 'not-an-ItemHash'"}, "input": "not-an-ItemHash", - "url": "https://errors.pydantic.dev/2.9/v/value_error", }, ] From d5856db67b82a17ecef1a18aa04071317e8bab90 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 26 Feb 2025 00:49:11 +0900 Subject: [PATCH 85/91] Fix: parse_raw_as has been removed from pydantic2 Replacing the function with TypeAdapter --- src/aleph/vm/pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py index edcccd43a..d2c22ca08 100644 --- a/src/aleph/vm/pool.py +++ b/src/aleph/vm/pool.py @@ -14,7 +14,7 @@ Payment, PaymentType, ) -from pydantic import parse_raw_as +from pydantic import TypeAdapter from aleph.vm.conf import settings from aleph.vm.controllers.firecracker.snapshot_manager import SnapshotManager @@ -249,7 +249,7 @@ async def load_persistent_executions(self): if execution.is_running: # TODO: Improve the way that we re-create running execution # Load existing GPUs assigned to VMs - execution.gpus = parse_raw_as(List[HostGPU], saved_execution.gpus) if saved_execution.gpus else [] + execution.gpus = TypeAdapter(List[HostGPU]).validate_python(saved_execution.gpus) if saved_execution.gpus else [] # Load and instantiate the rest of resources and already assigned GPUs await execution.prepare() if self.network: From d433e030090052c70a90af461f6f630f9a1c66ff Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 26 Feb 2025 00:52:56 +0900 Subject: [PATCH 86/91] Style: isort --- src/aleph/vm/conf.py | 2 +- src/aleph/vm/pool.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index e0c954ff2..4001b1dec 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -9,7 +9,7 @@ from os.path import abspath, exists, isdir, isfile, join from pathlib import Path from subprocess import CalledProcessError, check_output -from typing import Any, Literal, NewType, Optional, List +from typing import Any, List, Literal, NewType, Optional from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType diff --git a/src/aleph/vm/pool.py b/src/aleph/vm/pool.py index d2c22ca08..4c015ad62 100644 --- a/src/aleph/vm/pool.py +++ b/src/aleph/vm/pool.py @@ -249,7 +249,9 @@ async def load_persistent_executions(self): if execution.is_running: # TODO: Improve the way that we re-create running execution # Load existing GPUs assigned to VMs - execution.gpus = TypeAdapter(List[HostGPU]).validate_python(saved_execution.gpus) if saved_execution.gpus else [] + execution.gpus = ( + TypeAdapter(List[HostGPU]).validate_python(saved_execution.gpus) if saved_execution.gpus else [] + ) # Load and instantiate the rest of resources and already assigned GPUs await execution.prepare() if self.network: From 631401eaa0d08377c46d9f6ce58c322fb21923e6 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 26 Feb 2025 00:58:55 +0900 Subject: [PATCH 87/91] Fix: wrong version of aiohttp_cors --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index b1ec978cb..77160910f 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -16,7 +16,7 @@ debian-package-code: cp -r ../examples/data ./aleph-vm/opt/aleph-vm/examples/data mkdir -p ./aleph-vm/opt/aleph-vm/examples/volumes # Fixing this protobuf dependency version to avoid getting CI errors as version 5.29.0 have this compilation issue - pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.12' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3' + pip3 install --progress-bar off --target ./aleph-vm/opt/aleph-vm/ 'git+https://github.com/aleph-im/aleph-message@108-upgrade-pydantic-version#egg=aleph-message' 'eth-account==0.10' 'sentry-sdk==1.31.0' 'qmp==1.1.0' 'aleph-superfluid~=0.2.1' 'sqlalchemy[asyncio]>=2.0' 'aiosqlite==0.19.0' 'alembic==1.13.1' 'aiohttp_cors==0.7.0' 'pydantic-settings==2.6.1' 'pyroute2==0.7.12' 'python-cpuid==0.1.0' 'solathon==1.0.2' 'protobuf==5.28.3' python3 -m compileall ./aleph-vm/opt/aleph-vm/ debian-package-resources: firecracker-bins vmlinux download-ipfs-kubo target/bin/sevctl From 072dec8916965217b25ba5d210ef311b17022726 Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Wed, 12 Mar 2025 12:14:24 +0900 Subject: [PATCH 88/91] Style: mypy --- src/aleph/vm/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index eb88dc63e..4001b1dec 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -14,7 +14,7 @@ from aleph_message.models import Chain from aleph_message.models.execution.environment import HypervisorType from dotenv import load_dotenv -from pydantic import BaseSettings, Field, HttpUrl +from pydantic import Field, HttpUrl from pydantic_settings import BaseSettings, SettingsConfigDict from aleph.vm.orchestrator.chain import STREAM_CHAINS From e3b57360d95b3600f98121d5ee5fdf5952f292bc Mon Sep 17 00:00:00 2001 From: Olivier Le Thanh Duong Date: Tue, 1 Apr 2025 10:40:38 +0200 Subject: [PATCH 89/91] Convert newly added model to pydantic v2 --- src/aleph/vm/conf.py | 4 ++-- src/aleph/vm/orchestrator/resources.py | 4 ++-- src/aleph/vm/orchestrator/tasks.py | 1 - src/aleph/vm/orchestrator/views/__init__.py | 2 +- src/aleph/vm/orchestrator/views/authentication.py | 8 +------- src/aleph/vm/resources.py | 9 +++------ tests/supervisor/test_gpu_x_vga_support.py | 2 +- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/aleph/vm/conf.py b/src/aleph/vm/conf.py index ea16a57ed..b16115fa5 100644 --- a/src/aleph/vm/conf.py +++ b/src/aleph/vm/conf.py @@ -209,10 +209,10 @@ class Settings(BaseSettings): None, description="Location of executions log. Default to EXECUTION_ROOT/executions/" ) - PERSISTENT_VOLUMES_DIR: Optional[Path] = Field( + PERSISTENT_VOLUMES_DIR: Path | None = Field( None, description="Persistent volumes location. Default to EXECUTION_ROOT/volumes/persistent/" ) - JAILER_BASE_DIR: Optional[Path] = Field(None) + JAILER_BASE_DIR: Path | None = Field(None) MAX_PROGRAM_ARCHIVE_SIZE: int = 10_000_000 # 10 MB MAX_DATA_ARCHIVE_SIZE: int = 10_000_000 # 10 MB diff --git a/src/aleph/vm/orchestrator/resources.py b/src/aleph/vm/orchestrator/resources.py index 8fb489652..ac5a627cb 100644 --- a/src/aleph/vm/orchestrator/resources.py +++ b/src/aleph/vm/orchestrator/resources.py @@ -76,8 +76,8 @@ class MachineProperties(BaseModel): class GpuProperties(BaseModel): - devices: list[GpuDevice] | None - available_devices: list[GpuDevice] | None + devices: list[GpuDevice] | None = None + available_devices: list[GpuDevice] | None = None class MachineUsage(BaseModel): diff --git a/src/aleph/vm/orchestrator/tasks.py b/src/aleph/vm/orchestrator/tasks.py index e6f657f1b..803d3ca32 100644 --- a/src/aleph/vm/orchestrator/tasks.py +++ b/src/aleph/vm/orchestrator/tasks.py @@ -17,7 +17,6 @@ parse_message, ) from aleph_message.status import MessageStatus -from pydantic import ValidationError from yarl import URL from aleph.vm.conf import settings diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py index f4705f015..a2aff8d7e 100644 --- a/src/aleph/vm/orchestrator/views/__init__.py +++ b/src/aleph/vm/orchestrator/views/__init__.py @@ -4,7 +4,6 @@ from decimal import Decimal from hashlib import sha256 from json import JSONDecodeError -from packaging.version import InvalidVersion, Version from pathlib import Path from secrets import compare_digest from string import Template @@ -15,6 +14,7 @@ from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound from aleph_message.exceptions import UnknownHashError from aleph_message.models import InstanceContent, ItemHash, MessageType, PaymentType +from packaging.version import InvalidVersion, Version from pydantic import ValidationError from aleph.vm.conf import settings diff --git a/src/aleph/vm/orchestrator/views/authentication.py b/src/aleph/vm/orchestrator/views/authentication.py index e8c719681..dee57e339 100644 --- a/src/aleph/vm/orchestrator/views/authentication.py +++ b/src/aleph/vm/orchestrator/views/authentication.py @@ -24,13 +24,7 @@ from jwcrypto import jwk from jwcrypto.jwa import JWA from nacl.exceptions import BadSignatureError -from pydantic import ( - BaseModel, - ValidationError, - ValidationInfo, - field_validator, - model_validator, -) +from pydantic import BaseModel, ValidationError, field_validator, model_validator from solathon.utils import verify_signature from aleph.vm.conf import settings diff --git a/src/aleph/vm/resources.py b/src/aleph/vm/resources.py index 98b317865..2a6eaa4ba 100644 --- a/src/aleph/vm/resources.py +++ b/src/aleph/vm/resources.py @@ -2,7 +2,7 @@ from enum import Enum from aleph_message.models import HashableModel -from pydantic import BaseModel, Extra, Field +from pydantic import BaseModel, ConfigDict, Field from aleph.vm.orchestrator.utils import get_compatible_gpus @@ -12,9 +12,7 @@ class HostGPU(BaseModel): pci_host: str = Field(description="GPU PCI host address") supports_x_vga: bool = Field(description="Whether the GPU supports x-vga QEMU parameter", default=True) - - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class GpuDeviceClass(str, Enum): @@ -47,8 +45,7 @@ def has_x_vga_support(self) -> bool: """ return self.device_class == GpuDeviceClass.VGA_COMPATIBLE_CONTROLLER - class Config: - extra = Extra.forbid + model_config = ConfigDict(extra="forbid") class CompatibleGPU(BaseModel): diff --git a/tests/supervisor/test_gpu_x_vga_support.py b/tests/supervisor/test_gpu_x_vga_support.py index 9081d49f7..d3a62ef08 100644 --- a/tests/supervisor/test_gpu_x_vga_support.py +++ b/tests/supervisor/test_gpu_x_vga_support.py @@ -4,7 +4,7 @@ from aleph.vm.controllers.configuration import QemuGPU from aleph.vm.hypervisors.qemu.qemuvm import QemuVM -from aleph.vm.resources import GpuDevice, GpuDeviceClass, HostGPU +from aleph.vm.resources import GpuDevice, GpuDeviceClass class TestGpuXVgaSupport: From 6d2335140bdd892094e613345f6a6c4c9957e44b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 1 Apr 2025 19:09:01 +0900 Subject: [PATCH 90/91] Style: isort --- src/aleph/vm/orchestrator/views/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aleph/vm/orchestrator/views/__init__.py b/src/aleph/vm/orchestrator/views/__init__.py index a2aff8d7e..f4705f015 100644 --- a/src/aleph/vm/orchestrator/views/__init__.py +++ b/src/aleph/vm/orchestrator/views/__init__.py @@ -4,6 +4,7 @@ from decimal import Decimal from hashlib import sha256 from json import JSONDecodeError +from packaging.version import InvalidVersion, Version from pathlib import Path from secrets import compare_digest from string import Template @@ -14,7 +15,6 @@ from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound from aleph_message.exceptions import UnknownHashError from aleph_message.models import InstanceContent, ItemHash, MessageType, PaymentType -from packaging.version import InvalidVersion, Version from pydantic import ValidationError from aleph.vm.conf import settings From 95b2b8178ccf5b64df1a6d02764543c7d83c9e8b Mon Sep 17 00:00:00 2001 From: Antonyjin Date: Tue, 1 Apr 2025 21:31:30 +0900 Subject: [PATCH 91/91] Fix: GpuDevice class has his 'model' argument bad initialized The test didn't understand that 'model' was Optional, so it raise an error about it --- src/aleph/vm/resources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aleph/vm/resources.py b/src/aleph/vm/resources.py index 2a6eaa4ba..fe9276568 100644 --- a/src/aleph/vm/resources.py +++ b/src/aleph/vm/resources.py @@ -1,5 +1,6 @@ import subprocess from enum import Enum +from typing import Optional from aleph_message.models import HashableModel from pydantic import BaseModel, ConfigDict, Field @@ -26,7 +27,7 @@ class GpuDevice(HashableModel): """GPU properties.""" vendor: str = Field(description="GPU vendor name") - model: str | None = Field(description="GPU model name on Aleph Network") + model: Optional[str] = Field(default=None, description="GPU model name on Aleph Network") device_name: str = Field(description="GPU vendor card name") device_class: GpuDeviceClass = Field( description="GPU device class. Look at https://admin.pci-ids.ucw.cz/read/PD/03"