diff --git a/src/kiara/context/__init__.py b/src/kiara/context/__init__.py index 1cf04f36c..bf9328d8e 100644 --- a/src/kiara/context/__init__.py +++ b/src/kiara/context/__init__.py @@ -9,7 +9,7 @@ # from alembic import command # type: ignore from pydantic import Field -from kiara.context.config import KiaraConfig, KiaraContextConfig +from kiara.context.config import KiaraArchiveReference, KiaraConfig, KiaraContextConfig from kiara.context.runtime_config import KiaraRuntimeConfig from kiara.data_types import DataType from kiara.exceptions import KiaraContextException @@ -45,7 +45,7 @@ from kiara.registries.rendering import RenderRegistry from kiara.registries.types import TypeRegistry from kiara.registries.workflows import WorkflowRegistry -from kiara.utils import log_exception +from kiara.utils import log_exception, log_message from kiara.utils.class_loading import find_all_archive_types from kiara.utils.operations import filter_operations @@ -169,23 +169,23 @@ def __init__( config_cls = archive_cls._config_cls archive_config = config_cls(**archive.config) - archive_obj = archive_cls(archive_id=archive.archive_uuid, config=archive_config) # type: ignore + archive_obj = archive_cls(archive_alias=archive_alias, archive_config=archive_config) # type: ignore for supported_type in archive_obj.supported_item_types(): if supported_type == "data": self.data_registry.register_data_archive( - archive_obj, alias=archive_alias # type: ignore + archive_obj, # type: ignore ) if supported_type == "job_record": - self.job_registry.register_job_archive(archive_obj, alias=archive_alias) # type: ignore + self.job_registry.register_job_archive(archive_obj) # type: ignore if supported_type == "alias": - self.alias_registry.register_archive(archive_obj, alias=archive_alias) # type: ignore + self.alias_registry.register_archive(archive_obj) # type: ignore if supported_type == "destiny": - self.destiny_registry.register_destiny_archive(archive_obj, alias=archive_alias) # type: ignore + self.destiny_registry.register_destiny_archive(archive_obj) # type: ignore if supported_type == "workflow": - self.workflow_registry.register_archive(archive_obj, alias=archive_alias) # type: ignore + self.workflow_registry.register_archive(archive_obj) # type: ignore if self._runtime_config.lock_context: self.lock_context() @@ -314,6 +314,40 @@ def module_type_names(self) -> Iterable[str]: # =================================================================================================== # kiara session API methods + def register_external_archive( + self, + archive: Union[str, KiaraArchive, List[KiaraArchive], List[str]], + allow_write_access: bool = False, + ): + + if isinstance(archive, (KiaraArchive, str)): + _archives = [archive] + else: + _archives = archive + + archive_instances = set() + for _archive in _archives: + + if isinstance(_archive, KiaraArchive): + archive_instances.add(_archive) + # TODO: handle write access + continue + + loaded = KiaraArchiveReference.load_existing_archive( + archive_uri=_archive, allow_write_access=allow_write_access + ) + + archive_instances.update(loaded) + + for _archve_inst in archive_instances: + log_message( + "register.external.archive", + archive=_archve_inst.archive_alias, + allow_write_access=allow_write_access, + ) + + return list(archive_instances) + def create_manifest( self, module_or_operation: str, config: Union[Mapping[str, Any], None] = None ) -> Manifest: diff --git a/src/kiara/context/config.py b/src/kiara/context/config.py index 9c8e908de..473d2d998 100644 --- a/src/kiara/context/config.py +++ b/src/kiara/context/config.py @@ -34,7 +34,6 @@ from kiara.registries.ids import ID_REGISTRY from kiara.utils import log_message from kiara.utils.files import get_data_from_file -from kiara.utils.stores import create_store if TYPE_CHECKING: from kiara.context import Kiara @@ -62,16 +61,137 @@ def config_file_settings_source(settings: BaseSettings) -> Dict[str, Any]: class KiaraArchiveConfig(BaseModel): + """Configuration data that can be used to load an existing kiara archive.""" - archive_id: str = Field(description="The unique archive id.") + # archive_alias: str = Field(description="The unique archive id.") archive_type: str = Field(description="The archive type.") config: Mapping[str, Any] = Field( description="Archive type specific config.", default_factory=dict ) + +class KiaraArchiveReference(BaseModel): + @classmethod + def load_existing_archive( + cls, + archive_uri: str, + store_type: Union[str, None, Iterable[str]] = None, + allow_write_access: bool = False, + **kwargs: Any, + ) -> "KiaraArchiveReference": + + from kiara.utils.class_loading import find_all_archive_types + + archive_types = find_all_archive_types() + + archive_configs: List[KiaraArchiveConfig] = [] + archives: List[KiaraArchive] = [] + + archive_alias = archive_uri + + if store_type: + if isinstance(store_type, str): + archive_cls: Type[KiaraArchive] = archive_types.get(store_type, None) + if archive_cls is None: + raise Exception( + f"Can't create context: no archive type '{store_type}' available. Available types: {', '.join(archive_types.keys())}" + ) + data = archive_cls.load_store_config( + store_uri=archive_uri, + allow_write_access=allow_write_access, + **kwargs, + ) + archive: KiaraArchive = archive_cls( + archive_alias=archive_alias, archive_config=data + ) + archive_configs.append(data) + archives.append(archive) + else: + for st in store_type: + archive_cls = archive_types.get(st, None) + if archive_cls is None: + raise Exception( + f"Can't create context: no archive type '{store_type}' available. Available types: {', '.join(archive_types.keys())}" + ) + data = archive_cls.load_store_config( + store_uri=archive_uri, + allow_write_access=allow_write_access, + **kwargs, + ) + archive: KiaraArchive = archive_cls( + archive_alias=archive_alias, archive_config=data + ) + archive_configs.append(data) + archives.append(archive) + else: + for archive_type, archive_cls in archive_types.items(): + + data = archive_cls.load_store_config( + store_uri=archive_uri, + allow_write_access=allow_write_access, + **kwargs, + ) + if data is None: + continue + archive: KiaraArchive = archive_cls( + archive_alias=archive_alias, archive_config=data + ) + archive_configs.append(data) + archives.append(archive) + + if archives is None: + raise Exception( + f"Can't create context: no valid archive found at '{archive_uri}'" + ) + + result = cls( + archive_uri=archive_uri, + archive_alias=archive_alias, + allow_write_access=allow_write_access, + archive_configs=archive_configs, + ) + result._archives = archives + return result + + archive_uri: str = Field(description="The uri that points to the archive.") + archive_alias: str = Field( + description="The alias that is used for the archives contained in here." + ) + allow_write_access: bool = Field( + description="Whether to allow write access to the archives contained here.", + default=False, + ) + archive_configs: List[KiaraArchiveConfig] = Field( + description="All the archives this kiara context can use and the aliases they are registered with." + ) + _archives: Union[None, List["KiaraArchive"]] = PrivateAttr(default=None) + @property - def archive_uuid(self) -> uuid.UUID: - return uuid.UUID(self.archive_id) + def archives(self) -> List["KiaraArchive"]: + + if self._archives is not None: + return self._archives + + from kiara.utils.class_loading import find_all_archive_types + + archive_types = find_all_archive_types() + + result = [] + for config in self.archive_configs: + if config.archive_type not in archive_types.keys(): + raise Exception( + f"Can't create context: no archive type '{config.archive_type}' available. Available types: {', '.join(archive_types.keys())}" + ) + + archive_cls = archive_types[config.archive_type] + archive = archive_cls.load_store_config( + archive_uri=self.archive_uri, + allow_write_access=self.allow_write_access, + ) + result.append(archive) + + self._archives = result + return self._archives class KiaraContextConfig(BaseModel): @@ -98,22 +218,22 @@ def add_pipelines(self, *pipelines: str): "ignore.pipeline", reason="path does not exist", path=pipeline ) - def create_archive( - self, archive_alias: str, allow_write_access: bool = False - ) -> "KiaraArchive": - """Create the kiara archive with the specified alias. - - Make sure you know what you are doing when setting 'allow_write_access' to True. - """ - - store_config = self.archives[archive_alias] - store = create_store( - archive_id=store_config.archive_uuid, - store_type=store_config.archive_type, - store_config=store_config.config, - allow_write_access=allow_write_access, - ) - return store + # def create_archive( + # self, archive_alias: str, allow_write_access: bool = False + # ) -> "KiaraArchive": + # """Create the kiara archive with the specified alias. + # + # Make sure you know what you are doing when setting 'allow_write_access' to True. + # """ + # + # store_config = self.archives[archive_alias] + # store = create_store( + # archive_id=store_config.archive_uuid, + # store_type=store_config.archive_type, + # store_config=store_config.config, + # allow_write_access=allow_write_access, + # ) + # return store class KiaraSettings(BaseSettings): @@ -130,8 +250,8 @@ class KiaraSettings(BaseSettings): KIARA_SETTINGS = KiaraSettings() -def create_default_store( - store_id: uuid.UUID, store_type: str, stores_base_path: str +def create_default_store_config( + store_type: str, stores_base_path: str ) -> KiaraArchiveConfig: env_registry = EnvironmentRegistry.instance() @@ -144,10 +264,18 @@ def create_default_store( archive_info: ArchiveTypeInfo = available_archives.item_infos[store_type] cls: Type[BaseArchive] = archive_info.python_class.get_class() # type: ignore - config = cls.create_new_config(store_id=store_id, stores_base_path=stores_base_path) + + log_message( + "create_new_store", + stores_base_path=stores_base_path, + store_type=cls.__name__, + ) + + config = cls._config_cls.create_new_store_config(store_base_path=stores_base_path) + + # store_id: uuid.UUID = config.get_archive_id() data_store = KiaraArchiveConfig( - archive_id=store_id, archive_type=store_type, config=config.model_dump(), ) @@ -349,17 +477,18 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: changed = False - default_store_id = context_config.context_id + + sqlite_base_path = os.path.join(self.stores_base_path, "sqlite_stores") + filesystem_base_path = os.path.join(self.stores_base_path, "filesystem_stores") if DEFAULT_DATA_STORE_MARKER not in context_config.archives.keys(): # data_store_type = "filesystem_data_store" # TODO: change that back data_store_type = "sqlite_data_store" - data_store = create_default_store( - store_id=default_store_id, + data_store = create_default_store_config( store_type=data_store_type, - stores_base_path=self.stores_base_path, + stores_base_path=sqlite_base_path, ) context_config.archives[DEFAULT_DATA_STORE_MARKER] = data_store changed = True @@ -369,10 +498,9 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: # job_store_type = "filesystem_job_store" job_store_type = "sqlite_job_store" - job_store = create_default_store( - store_id=default_store_id, + job_store = create_default_store_config( store_type=job_store_type, - stores_base_path=self.stores_base_path, + stores_base_path=sqlite_base_path, ) context_config.archives[DEFAULT_JOB_STORE_MARKER] = job_store @@ -383,8 +511,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: alias_store_type = "filesystem_alias_store" alias_store_type = "sqlite_alias_store" - alias_store = create_default_store( - store_id=default_store_id, + alias_store = create_default_store_config( store_type=alias_store_type, stores_base_path=self.stores_base_path, ) @@ -394,10 +521,9 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: if DEFAULT_WORKFLOW_STORE_MARKER not in context_config.archives.keys(): workflow_store_type = "filesystem_workflow_store" - workflow_store = create_default_store( - store_id=default_store_id, + workflow_store = create_default_store_config( store_type=workflow_store_type, - stores_base_path=self.stores_base_path, + stores_base_path=os.path.join(filesystem_base_path, "workflows"), ) context_config.archives[DEFAULT_WORKFLOW_STORE_MARKER] = workflow_store changed = True @@ -406,10 +532,9 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: destiny_store_type = "filesystem_destiny_store" - destiny_store = create_default_store( - store_id=default_store_id, + destiny_store = create_default_store_config( store_type=destiny_store_type, - stores_base_path=self.stores_base_path, + stores_base_path=os.path.join(filesystem_base_path, "destinies"), ) context_config.archives[METADATA_DESTINY_STORE_MARKER] = destiny_store changed = True diff --git a/src/kiara/defaults.py b/src/kiara/defaults.py index e0824f99f..f9c2a03eb 100644 --- a/src/kiara/defaults.py +++ b/src/kiara/defaults.py @@ -123,6 +123,8 @@ KIARA_DEFAULT_ROOT_NODE_ID = "__self__" +KIARA_SQLITE_STORE_EXTENSION = "kiara" + VALUE_ATTR_DELIMITER = "::" VALID_VALUE_QUERY_CATEGORIES = ["data", "properties"] diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index 3317640cf..7d4ce7794 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -507,15 +507,35 @@ def filter_value( terminal_print(f"[red]Error saving results[/red]: {e}") sys.exit(1) - # if save_results: - # try: - # saved_results = kiara_op.save_result(job_id=job_id, aliases=final_aliases) - # if len(saved_results) == 1: - # title = "[b]Stored result value[/b]" - # else: - # title = "[b]Stored result values[/b]" - # terminal_print(saved_results, in_panel=title, empty_line_before=True) - # except Exception as e: - # log_exception(e) - # terminal_print(f"[red]Error saving results[/red]: {e}") - # sys.exit(1) + +@data.command(name="export") +@click.argument("alias", nargs=1, required=True) +@click.pass_context +def export_data_store(ctx, alias: str): + + from kiara.utils.stores import create_new_store + + kiara_api: KiaraAPI = ctx.obj.kiara_api + + value = kiara_api.get_value(alias) + base_path = "." + + store = create_new_store( + archive_alias=f"export_store_{alias}", + store_base_path=base_path, + store_type="sqlite_data_store", + file_name=f"{alias}.sqlite", + ) + + store_alias = kiara_api.context.data_registry.register_data_archive(store) + + try: + persisted_data = kiara_api.context.data_registry.store_value( + value, store_id=store_alias + ) + dbg(persisted_data) + except Exception as e: + store.delete_archive(archive_id=store.archive_id) + log_exception(e) + terminal_print(f"[red]Error saving results[/red]: {e}") + sys.exit(1) diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index 0ede7cad4..1381cbe4a 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -1321,7 +1321,7 @@ def list_values(self, **matcher_params: Any) -> ValueMapReadOnly: ) return result - def get_value(self, value: Union[str, Value, uuid.UUID]) -> Value: + def get_value(self, value: Union[str, Value, uuid.UUID, Path]) -> Value: """ Retrieve a value instance with the specified id or alias. @@ -1425,7 +1425,9 @@ def dict_path(path, my_dict, all_paths): return current_result - def retrieve_value_info(self, value: Union[str, uuid.UUID, Value]) -> ValueInfo: + def retrieve_value_info( + self, value: Union[str, uuid.UUID, Value, Path] + ) -> ValueInfo: """ Retrieve an info object for a value. diff --git a/src/kiara/registries/__init__.py b/src/kiara/registries/__init__.py index dba32f854..42fe3f2af 100644 --- a/src/kiara/registries/__init__.py +++ b/src/kiara/registries/__init__.py @@ -7,8 +7,10 @@ import abc import os +import typing import uuid -from typing import TYPE_CHECKING, Any, Generic, Iterable, Mapping, Type, TypeVar, Union +from pathlib import Path +from typing import TYPE_CHECKING, Generic, Iterable, Type, TypeVar, Union import structlog from pydantic import BaseModel, ConfigDict, Field @@ -22,21 +24,24 @@ if TYPE_CHECKING: from kiara.context import Kiara - from kiara.context.config import KiaraArchiveConfig class ArchiveConfig(BaseModel, abc.ABC): @classmethod @abc.abstractmethod - def create_new_store_config( - cls, store_id: uuid.UUID, stores_base_path: str - ) -> Self: + def create_new_store_config(cls, store_base_path: str, **kwargs) -> Self: raise NotImplementedError( f"Store config type '{cls}' does not implement 'create_new_config'." ) model_config = ConfigDict() + # @abc.abstractmethod + # def get_archive_id(self) -> uuid.UUID: + # raise NotImplementedError( + # f"Store config type '{self.__class__.__name__}' does not implement 'get_archive_id'." + # ) + ARCHIVE_CONFIG_CLS = TypeVar("ARCHIVE_CONFIG_CLS", bound=ArchiveConfig) @@ -54,30 +59,88 @@ class ArchiveDetails(BaseModel): NON_ARCHIVE_DETAILS = ArchiveDetails() -class KiaraArchive(abc.ABC): +class KiaraArchive(abc.ABC, typing.Generic[ARCHIVE_CONFIG_CLS]): + + _config_cls: Type[ARCHIVE_CONFIG_CLS] = None # type: ignore + + # @classmethod + # def create_store_config_instance( + # cls, config: Union[ARCHIVE_CONFIG_CLS, BaseModel, Mapping[str, Any]] + # ) -> "BaseArchive": + # """Create a store config instance from a config instance of a few different types.""" + # + # from kiara.context.config import KiaraArchiveConfig + # + # if isinstance(config, cls._config_cls): + # config = config + # elif isinstance(config, KiaraArchiveConfig): + # config = cls._config_cls(**config.config) + # elif isinstance(config, BaseModel): + # config = cls._config_cls(**config.model_dump()) + # elif isinstance(config, Mapping): + # config = cls._config_cls(**config) + # + # return config + + # @classmethod + # def is_valid_archive(cls, store_uri: str, **kwargs: Any) -> bool: + # return False + + @classmethod + def _load_store_config( + cls, store_uri: str, allow_write_access: bool, **kwargs + ) -> Union[ARCHIVE_CONFIG_CLS, None]: + """Tries to assemble an archive config from an uri (and optional paramters). + + If the archive type supports the archive at the uri, then a valid config will be returned, + otherwise 'None'. + """ + + return None + + @classmethod + def load_store_config( + cls, store_uri: str, allow_write_access: bool, **kwargs + ) -> Union[ARCHIVE_CONFIG_CLS, None]: + + log_message( + "attempt_loading_existing_store", + store_uri=store_uri, + store_type=cls.__name__, + ) - _config_cls = ArchiveConfig # type: ignore + return cls._load_store_config( + store_uri=store_uri, allow_write_access=allow_write_access, **kwargs + ) @classmethod - def create_config( - cls, config: Union["KiaraArchiveConfig", BaseModel, Mapping[str, Any]] - ) -> "BaseArchive": + def create_new_store_config( + cls, store_base_path: str, **kwargs + ) -> ARCHIVE_CONFIG_CLS: + + log_message( + "create_new_store", + store_base_path=store_base_path, + store_type=cls.__name__, + ) - from kiara.context.config import KiaraArchiveConfig + Path(store_base_path).mkdir(parents=True, exist_ok=True) - if isinstance(config, cls._config_cls): - config = config - elif isinstance(config, KiaraArchiveConfig): - config = cls._config_cls(**config.config) - elif isinstance(config, BaseModel): - config = cls._config_cls(**config.model_dump()) - elif isinstance(config, Mapping): - config = cls._config_cls(**config) + return cls._config_cls.create_new_store_config( + store_base_path=store_base_path, **kwargs + ) - return config + def __init__( + self, + archive_alias: str, + archive_config: ARCHIVE_CONFIG_CLS, + force_read_only: bool = False, + ): - def __init__(self, force_read_only: bool = False, **kwargs): + self._archive_alias: str = archive_alias + self._config: ARCHIVE_CONFIG_CLS = archive_config self._force_read_only: bool = force_read_only + self._archive_id: Union[uuid.UUID, None] = None @classmethod @abc.abstractmethod @@ -89,30 +152,36 @@ def supported_item_types(cls) -> Iterable[str]: def _is_writeable(cls) -> bool: pass + @abc.abstractmethod + def register_archive(self, kiara: "Kiara"): + pass + + @property + def archive_alias(self) -> str: + return self._archive_alias + def is_writeable(self) -> bool: if self._force_read_only: return False return self.__class__._is_writeable() - @abc.abstractmethod - def register_archive(self, kiara: "Kiara"): - pass + # @abc.abstractmethod + # def register_archive(self, kiara: "Kiara"): + # pass @abc.abstractmethod - def retrieve_archive_id(self) -> uuid.UUID: - pass + def _retrieve_archive_id(self) -> uuid.UUID: + raise NotImplementedError() @property def archive_id(self) -> uuid.UUID: - return self.retrieve_archive_id() + if self._archive_id is None: + self._archive_id = self._retrieve_archive_id() + return self._archive_id @property - def config(self) -> ArchiveConfig: - return self._get_config() - - @abc.abstractmethod - def _get_config(self) -> ArchiveConfig: - pass + def config(self) -> ARCHIVE_CONFIG_CLS: + return self._config def get_archive_details(self) -> ArchiveDetails: return NON_ARCHIVE_DETAILS @@ -148,48 +217,27 @@ def __eq__(self, other): return self.archive_id == other.archive_id -class BaseArchive(KiaraArchive, Generic[ARCHIVE_CONFIG_CLS]): - - _config_cls: Type[ARCHIVE_CONFIG_CLS] = None # type: ignore - - @classmethod - def create_new_config( - cls, store_id: uuid.UUID, stores_base_path: str - ) -> ARCHIVE_CONFIG_CLS: - - log_message( - "create_new_store", - store_id=store_id, - stores_base_path=stores_base_path, - store_type=cls.__name__, - ) - - return cls._config_cls.create_new_store_config( - store_id=store_id, stores_base_path=stores_base_path - ) +class BaseArchive(KiaraArchive[ARCHIVE_CONFIG_CLS], Generic[ARCHIVE_CONFIG_CLS]): + """A base class that can be used to implement a kiara archive.""" def __init__( self, - archive_id: uuid.UUID, - config: ARCHIVE_CONFIG_CLS, + archive_alias: str, + archive_config: ARCHIVE_CONFIG_CLS, force_read_only: bool = False, ): - super().__init__(force_read_only=force_read_only) - self._archive_id: uuid.UUID = archive_id - self._config: ARCHIVE_CONFIG_CLS = config + super().__init__( + archive_alias=archive_alias, + archive_config=archive_config, + force_read_only=force_read_only, + ) self._kiara: Union["Kiara", None] = None @classmethod def _is_writeable(cls) -> bool: return False - def _get_config(self) -> ARCHIVE_CONFIG_CLS: - return self._config - - def retrieve_archive_id(self) -> uuid.UUID: - return self._archive_id - @property def kiara_context(self) -> "Kiara": if self._kiara is None: @@ -213,14 +261,24 @@ def _delete_archive(self): class FileSystemArchiveConfig(ArchiveConfig): + @classmethod + def load_store_config(cls, store_uri: str, **kwargs) -> Self: + raise NotImplementedError( + f"Store config type '{cls}' does not implement 'create_config'." + ) + @classmethod def create_new_store_config( - cls, store_id: str, stores_base_path: str + cls, store_base_path: str, **kwargs ) -> "FileSystemArchiveConfig": - archive_path = os.path.abspath( - os.path.join(stores_base_path, "filesystem_data_store", store_id) - ) + store_id = str(uuid.uuid4()) + if "path" in kwargs: + file_name = kwargs["path"] + else: + file_name = store_id + + archive_path = os.path.abspath(os.path.join(store_base_path, file_name)) return FileSystemArchiveConfig(archive_path=archive_path) @@ -232,13 +290,47 @@ def create_new_store_config( class SqliteArchiveConfig(ArchiveConfig): @classmethod def create_new_store_config( - cls, store_id: str, stores_base_path: str + cls, store_base_path: str, **kwargs ) -> "SqliteArchiveConfig": - archive_path = os.path.abspath( - os.path.join(stores_base_path, "sqlite_stores", f"{store_id}.sqlite") + store_id = str(uuid.uuid4()) + + if "file_name" in kwargs: + file_name = kwargs["file_name"] + else: + file_name = f"{store_id}.sqlite" + + archive_path = os.path.abspath(os.path.join(store_base_path, file_name)) + + if os.path.exists(archive_path): + raise Exception(f"Archive path '{archive_path}' already exists.") + + Path(archive_path).parent.mkdir(exist_ok=True, parents=True) + + # Connect to the SQLite database (or create it if it doesn't exist) + import sqlite3 + + print(archive_path) + conn = sqlite3.connect(archive_path) + + # Create a cursor object + c = conn.cursor() + + # Create table + c.execute( + """CREATE TABLE archive_metadata + (key text PRIMARY KEY , value text NOT NULL)""" ) + # Insert a row of data + c.execute(f"INSERT INTO archive_metadata VALUES ('archive_id','{store_id}')") + + # Save (commit) the changes + conn.commit() + + # Close the connection + conn.close() + return SqliteArchiveConfig(sqlite_db_path=archive_path) sqlite_db_path: str = Field( diff --git a/src/kiara/registries/aliases/sqlite_store.py b/src/kiara/registries/aliases/sqlite_store.py index 44f418ca6..25d594a23 100644 --- a/src/kiara/registries/aliases/sqlite_store.py +++ b/src/kiara/registries/aliases/sqlite_store.py @@ -15,13 +15,25 @@ class SqliteAliasArchive(AliasArchive): _archive_type_name = "sqlite_alias_archive" _config_cls = SqliteArchiveConfig - def __init__(self, archive_id: uuid.UUID, config: SqliteArchiveConfig): + def __init__(self, archive_alias: str, archive_config: SqliteArchiveConfig): - AliasArchive.__init__(self, archive_id=archive_id, config=config) + AliasArchive.__init__( + self, archive_alias=archive_alias, archive_config=archive_config + ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None # self._lock: bool = True + def _retrieve_archive_id(self) -> uuid.UUID: + sql = text("SELECT value FROM archive_metadata WHERE key='archive_id'") + + with self.sqlite_engine.connect() as connection: + result = connection.execute(sql) + row = result.fetchone() + if row is None: + raise Exception("No archive ID found in metadata") + return uuid.UUID(row[0]) + @property def sqlite_path(self): diff --git a/src/kiara/registries/data/__init__.py b/src/kiara/registries/data/__init__.py index ce7e2bd07..34b7f4663 100644 --- a/src/kiara/registries/data/__init__.py +++ b/src/kiara/registries/data/__init__.py @@ -7,6 +7,7 @@ import abc import copy import uuid +from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -103,6 +104,9 @@ def resolve_alias(self, alias: str) -> uuid.UUID: pass +STORE_REF_TYPE_NAME = "store" + + class DefaultAliasResolver(AliasResolver): def __init__(self, kiara: "Kiara"): @@ -110,8 +114,6 @@ def __init__(self, kiara: "Kiara"): def resolve_alias(self, alias: str) -> uuid.UUID: - dbg(f"RESOLVE: {alias}") - if ":" in alias: ref_type, rest = alias.split(":", maxsplit=1) @@ -126,6 +128,20 @@ def resolve_alias(self, alias: str) -> uuid.UUID: alias=rest, msg=f"Can't retrive value for alias '{rest}': no such alias registered.", ) + elif ref_type == STORE_REF_TYPE_NAME: + + if "#" in rest: + raise NotImplementedError() + + archives = load_existing_archives(store=rest) + if archives: + for archive in archives: + archive_ref = self._kiara.data_registry.register_data_archive( + archive + ) + print(archive_ref) + + raise NotImplementedError("x") else: raise Exception( f"Can't retrieve value for '{alias}': invalid reference type '{ref_type}'." @@ -233,8 +249,6 @@ def retrieve_all_available_value_ids(self) -> Set[uuid.UUID]: result: Set[uuid.UUID] = set() for alias, store in self._data_archives.items(): - print(alias) - dbg(store.config.model_dump()) ids = store.value_ids if ids: result.update(ids) @@ -246,7 +260,7 @@ def register_data_archive( archive: DataArchive, alias: Union[str, None] = None, set_as_default_store: Union[bool, None] = None, - ): + ) -> str: data_store_id = archive.archive_id archive.register_archive(kiara=self._kiara) @@ -281,6 +295,8 @@ def register_data_archive( ) self._event_callback(event) + return alias + @property def default_data_store(self) -> str: if self._default_data_store is None: @@ -348,7 +364,7 @@ def find_store_id_for_value(self, value_id: uuid.UUID) -> Union[str, None]: self._value_archive_lookup_map[value_id] = matches[0] return matches[0] - def get_value(self, value: Union[uuid.UUID, ValueLink, str]) -> Value: + def get_value(self, value: Union[uuid.UUID, ValueLink, str, Path]) -> Value: _value_id = None if not isinstance(value, uuid.UUID): @@ -367,7 +383,8 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str]) -> Value: _value_id = None if _value_id is None: - + if isinstance(value, Path): + raise NotImplementedError() if not isinstance(value, str): raise Exception( f"Can't retrieve value for '{value}': invalid type '{type(value)}'." diff --git a/src/kiara/registries/data/data_store/__init__.py b/src/kiara/registries/data/data_store/__init__.py index 3d5cfbf40..3cb428ee8 100644 --- a/src/kiara/registries/data/data_store/__init__.py +++ b/src/kiara/registries/data/data_store/__init__.py @@ -6,6 +6,7 @@ # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) import abc +import typing import uuid from io import BytesIO from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Set, Union @@ -32,7 +33,7 @@ logger = structlog.getLogger() -class DataArchive(BaseArchive): +class DataArchive(BaseArchive[ARCHIVE_CONFIG_CLS], typing.Generic[ARCHIVE_CONFIG_CLS]): """Base class for data archiv implementationss.""" @classmethod @@ -43,13 +44,15 @@ def supported_item_types(cls) -> Iterable[str]: def __init__( self, - archive_id: uuid.UUID, - config: ARCHIVE_CONFIG_CLS, + archive_alias: str, + archive_config: ARCHIVE_CONFIG_CLS, force_read_only: bool = False, ): super().__init__( - archive_id=archive_id, config=config, force_read_only=force_read_only + archive_alias=archive_alias, + archive_config=archive_config, + force_read_only=force_read_only, ) self._env_cache: Dict[str, Dict[str, Mapping[str, Any]]] = {} diff --git a/src/kiara/registries/data/data_store/sqlite_store.py b/src/kiara/registries/data/data_store/sqlite_store.py index fa27d0280..f8ec06920 100644 --- a/src/kiara/registries/data/data_store/sqlite_store.py +++ b/src/kiara/registries/data/data_store/sqlite_store.py @@ -18,20 +18,57 @@ from kiara.utils.windows import fix_windows_longpath -class SqliteDataArchive(DataArchive): +class SqliteDataArchive(DataArchive[SqliteArchiveConfig]): _archive_type_name = "sqlite_data_archive" _config_cls = SqliteArchiveConfig + @classmethod + def _load_store_config( + cls, store_uri: str, allow_write_access: bool, **kwargs + ) -> Union[SqliteArchiveConfig, None]: + + if allow_write_access: + return None + + if not Path(store_uri).is_file(): + return None + + import sqlite3 + + con = sqlite3.connect(store_uri) + + cursor = con.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = {x[0] for x in cursor.fetchall()} + con.close() + + if tables != { + "values_pedigree", + "values_destinies", + "archive_metadata", + "persisted_values", + "values_metadata", + "values_data", + "environments", + }: + return None + + config = SqliteArchiveConfig(sqlite_db_path=store_uri) + return config + def __init__( self, - archive_id: uuid.UUID, - config: SqliteArchiveConfig, + archive_alias: str, + archive_config: SqliteArchiveConfig, force_read_only: bool = False, ): DataArchive.__init__( - self, archive_id=archive_id, config=config, force_read_only=force_read_only + self, + archive_alias=archive_alias, + archive_config=archive_config, + force_read_only=force_read_only, ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None @@ -42,6 +79,16 @@ def __init__( self._value_id_cache: Union[Iterable[uuid.UUID], None] = None # self._lock: bool = True + def _retrieve_archive_id(self) -> uuid.UUID: + sql = text("SELECT value FROM archive_metadata WHERE key='archive_id'") + + with self.sqlite_engine.connect() as connection: + result = connection.execute(sql) + row = result.fetchone() + if row is None: + raise Exception("No archive ID found in metadata") + return uuid.UUID(row[0]) + @property def sqlite_path(self): @@ -243,11 +290,48 @@ def retrieve_chunk( return chunk_path.as_posix() + def _delete_archive(self): + os.unlink(self.sqlite_path) + class SqliteDataStore(SqliteDataArchive, BaseDataStore): _archive_type_name = "sqlite_data_store" + @classmethod + def _load_store_config( + cls, store_uri: str, allow_write_access: bool, **kwargs + ) -> Union[SqliteArchiveConfig, None]: + + if not allow_write_access: + return None + + if not Path(store_uri).is_file(): + return None + + import sqlite3 + + con = sqlite3.connect(store_uri) + + cursor = con.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = {x[0] for x in cursor.fetchall()} + con.close() + + if tables != { + "values_pedigree", + "values_destinies", + "archive_metadata", + "persisted_values", + "values_metadata", + "values_data", + "environments", + }: + return None + + config = SqliteArchiveConfig(sqlite_db_path=store_uri) + return config + def _persist_environment_details( self, env_type: str, env_hash: str, env_data: Mapping[str, Any] ): diff --git a/src/kiara/registries/destinies/__init__.py b/src/kiara/registries/destinies/__init__.py index 91fe56f89..bebec7a5b 100644 --- a/src/kiara/registries/destinies/__init__.py +++ b/src/kiara/registries/destinies/__init__.py @@ -17,9 +17,9 @@ class DestinyArchive(BaseArchive): def supported_item_types(cls) -> Iterable[str]: return ["destiny"] - def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS): + def __init__(self, archive_alias: str, archive_config: ARCHIVE_CONFIG_CLS): - super().__init__(archive_id=archive_id, config=config) + super().__init__(archive_alias=archive_alias, archive_config=archive_config) @abc.abstractmethod def get_all_value_ids(self) -> Set[uuid.UUID]: diff --git a/src/kiara/registries/destinies/filesystem_store.py b/src/kiara/registries/destinies/filesystem_store.py index 5f02cb4d1..d7fbbc0cf 100644 --- a/src/kiara/registries/destinies/filesystem_store.py +++ b/src/kiara/registries/destinies/filesystem_store.py @@ -38,9 +38,9 @@ class FileSystemDestinyArchive(DestinyArchive): # ) # return result - def __init__(self, archive_id: uuid.UUID, config: FileSystemArchiveConfig): + def __init__(self, archive_alias: str, archive_config: FileSystemArchiveConfig): - super().__init__(archive_id=archive_id, config=config) + super().__init__(archive_alias=archive_alias, archive_config=archive_config) self._base_path: Union[Path, None] = None # base_path = config.archive_path @@ -54,6 +54,10 @@ def __init__(self, archive_id: uuid.UUID, config: FileSystemArchiveConfig): # self._destinies_path: Path = self._base_path / "destinies" # self._value_id_path: Path = self._base_path / "value_ids" + def _retrieve_archive_id(self) -> uuid.UUID: + + return uuid.UUID(self.destiny_store_path.name) + @property def destiny_store_path(self) -> Path: diff --git a/src/kiara/registries/jobs/job_store/sqlite_store.py b/src/kiara/registries/jobs/job_store/sqlite_store.py index 7b493726a..1269a2152 100644 --- a/src/kiara/registries/jobs/job_store/sqlite_store.py +++ b/src/kiara/registries/jobs/job_store/sqlite_store.py @@ -17,13 +17,25 @@ class SqliteJobArchive(JobArchive): _archive_type_name = "sqlite_job_archive" _config_cls = SqliteArchiveConfig - def __init__(self, archive_id: uuid.UUID, config: SqliteArchiveConfig): + def __init__(self, archive_alias: str, archive_config: SqliteArchiveConfig): - JobArchive.__init__(self, archive_id=archive_id, config=config) + JobArchive.__init__( + self, archive_alias=archive_alias, archive_config=archive_config + ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None # self._lock: bool = True + def _retrieve_archive_id(self) -> uuid.UUID: + sql = text("SELECT value FROM archive_metadata WHERE key='archive_id'") + + with self.sqlite_engine.connect() as connection: + result = connection.execute(sql) + row = result.fetchone() + if row is None: + raise Exception("No archive ID found in metadata") + return uuid.UUID(row[0]) + @property def sqlite_path(self): diff --git a/src/kiara/registries/workflows/archives.py b/src/kiara/registries/workflows/archives.py index 388689eff..731ef38c0 100644 --- a/src/kiara/registries/workflows/archives.py +++ b/src/kiara/registries/workflows/archives.py @@ -18,13 +18,17 @@ class FileSystemWorkflowArchive(WorkflowArchive): _archive_type_name = "filesystem_workflow_archive" _config_cls = FileSystemArchiveConfig # type: ignore - def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS): + def __init__(self, archive_alias: str, archive_config: ARCHIVE_CONFIG_CLS): - super().__init__(archive_id=archive_id, config=config) + super().__init__(archive_alias=archive_alias, archive_config=archive_config) self._base_path: Union[Path, None] = None self.alias_store_path.mkdir(parents=True, exist_ok=True) + def _retrieve_archive_id(self) -> uuid.UUID: + + return uuid.UUID(self.workflow_store_path.name) + @property def workflow_store_path(self) -> Path: diff --git a/src/kiara/utils/stores.py b/src/kiara/utils/stores.py index 0fbf84f3e..e8471f73f 100644 --- a/src/kiara/utils/stores.py +++ b/src/kiara/utils/stores.py @@ -1,19 +1,17 @@ # -*- coding: utf-8 -*- -import uuid -from typing import TYPE_CHECKING, Any, Mapping, Type, Union - -from pydantic import BaseModel +from typing import TYPE_CHECKING, Any, Type if TYPE_CHECKING: from kiara.registries import KiaraArchive -def create_store( - archive_id: uuid.UUID, +def create_new_store( + archive_alias: str, + store_base_path: str, store_type: str, - store_config: Union[Mapping[str, Any], BaseModel], allow_write_access: bool = False, -): + **kwargs: Any, +) -> "KiaraArchive": from kiara.utils.class_loading import find_all_archive_types @@ -25,10 +23,11 @@ def create_store( f"Can't create context: no archive type '{store_type}' available. Available types: {', '.join(archive_types.keys())}" ) - config = archive_cls.create_config(config=store_config) + # config = archive_cls.create_store_config_instance(config=store_config) + config = archive_cls.create_new_store_config(store_base_path, **kwargs) # TODO: make sure this constructor always exists? force_read_only = not allow_write_access - archive_instance = archive_cls(archive_id=archive_id, config=config, force_read_only=force_read_only) # type: ignore + archive_instance = archive_cls(archive_alias=archive_alias, archive_config=config, force_read_only=force_read_only) # type: ignore return archive_instance