feat: import from external store

Author: makkus

src/kiara/context/config.py

@@ -34,10 +34,12 @@
 from kiara.registries.ids import ID_REGISTRY
 from kiara.utils import log_message
 from kiara.utils.files import get_data_from_file
+from kiara.utils.stores import create_store
 
 if TYPE_CHECKING:
     from kiara.context import Kiara
     from kiara.models.context import ContextInfo
+    from kiara.registries import KiaraArchive
 
 logger = structlog.getLogger()
 
@@ -96,13 +98,22 @@ def add_pipelines(self, *pipelines: str):
                     "ignore.pipeline", reason="path does not exist", path=pipeline
                 )
 
-    # @property
-    # def db_url(self):
-    #     return get_kiara_db_url(self.context_folder)
-    #
-    # @property
-    # def data_directory(self) -> str:
-    #     return os.path.join(self.context_folder, "data")
+    def create_archive(
+        self, archive_alias: str, allow_write_access: bool = False
+    ) -> "KiaraArchive":
+        """Create the kiara archive with the specified alias.
+
+        Make sure you know what you are doing when setting 'allow_write_access' to True.
+        """
+
+        store_config = self.archives[archive_alias]
+        store = create_store(
+            archive_id=store_config.archive_uuid,
+            store_type=store_config.archive_type,
+            store_config=store_config.config,
+            allow_write_access=allow_write_access,
+        )
+        return store
 
 
 class KiaraSettings(BaseSettings):
@@ -120,7 +131,7 @@ class KiaraSettings(BaseSettings):
 
 
 def create_default_store(
-    store_id: str, store_type: str, stores_base_path: str
+    store_id: uuid.UUID, store_type: str, stores_base_path: str
 ) -> KiaraArchiveConfig:
 
     env_registry = EnvironmentRegistry.instance()
@@ -167,10 +178,12 @@ def create_in_folder(cls, path: Union[Path, str]) -> "KiaraConfig":
         return config
 
     @classmethod
-    def load_from_file(cls, path: Union[Path, None] = None) -> "KiaraConfig":
+    def load_from_file(cls, path: Union[Path, str, None] = None) -> "KiaraConfig":
 
         if path is None:
             path = Path(KIARA_MAIN_CONFIG_FILE)
+        elif isinstance(path, str):
+            path = Path(path)
 
         if not path.exists():
             raise Exception(

src/kiara/defaults.py

@@ -80,6 +80,7 @@
     "kiara",
     "callbacks",
 ]
+INVALID_ALIAS_NAMES = ["kiara", "__default__", "alias", "value", "value_id"]
 """List of reserved names, inputs/outputs can't use those."""
 
 DEFAULT_DATA_STORE_MARKER = "default_data_store"
@@ -94,7 +95,7 @@
 DEFAULT_WORKFLOW_STORE_MARKER = "default_workflow_store"
 """Name for the default context workflow store."""
 
-METADATA_DESTINY_STORE_MARKER = "default_destiny_store"
+METADATA_DESTINY_STORE_MARKER = "metadata"
 """Name for the default context destiny store."""
 
 PIPELINE_PARENT_MARKER = "__pipeline__"

src/kiara/interfaces/cli/data/commands.py

@@ -251,9 +251,8 @@ def explain_value(
 
     All of the 'show-additional-information' flags are only applied when the 'terminal' output format is selected. This might change in the future.
     """
-    from kiara.interfaces.python_api import ValueInfo
 
-    kiara_obj: Kiara = ctx.obj.kiara
+    kiara_api: KiaraAPI = ctx.obj.kiara_api
 
     render_config = {
         "show_pedigree": pedigree,
@@ -270,26 +269,26 @@ def explain_value(
     all_values = []
     for v_id in value_id:
         try:
-            value = kiara_obj.data_registry.get_value(v_id)
+            value_info = kiara_api.retrieve_value_info(v_id)
         except Exception as e:
             terminal_print()
             terminal_print(f"[red]Error[/red]: {e}")
             sys.exit(1)
-        if not value:
+        if not value_info:
             terminal_print(f"[red]Error[/red]: No value found for: {v_id}")
             sys.exit(1)
-        all_values.append(value)
+        all_values.append(value_info)
 
     if len(all_values) == 1:
         title = f"Value details for: [b i]{value_id[0]}[/b i]"
     else:
         title = "Value details"
 
-    v_infos = (
-        ValueInfo.create_from_instance(kiara=kiara_obj, instance=v) for v in all_values
-    )
+    # v_infos = (
+    #     ValueInfo.create_from_instance(kiara=kiara_obj, instance=v) for v in all_values
+    # )
 
-    terminal_print_model(*v_infos, format=format, in_panel=title, **render_config)
+    terminal_print_model(*all_values, format=format, in_panel=title, **render_config)
 
 
 @data.command(name="load")

src/kiara/interfaces/python_api/__init__.py

@@ -144,6 +144,8 @@ def instance(cls, context_name: Union[str, None] = None) -> "KiaraAPI":
     def __init__(self, kiara_config: Union["KiaraConfig", None] = None):
 
         if kiara_config is None:
+            from kiara.context import Kiara, KiaraConfig
+
             kiara_config = KiaraConfig()
 
         self._kiara_config: KiaraConfig = kiara_config

src/kiara/registries/__init__.py

@@ -8,7 +8,7 @@
 import abc
 import os
 import uuid
-from typing import TYPE_CHECKING, Generic, Iterable, Type, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Generic, Iterable, Mapping, Type, TypeVar, Union
 
 import structlog
 from pydantic import BaseModel, ConfigDict, Field
@@ -22,12 +22,15 @@
 
 if TYPE_CHECKING:
     from kiara.context import Kiara
+    from kiara.context.config import KiaraArchiveConfig
 
 
 class ArchiveConfig(BaseModel, abc.ABC):
     @classmethod
     @abc.abstractmethod
-    def create_new_store_config(cls, store_id: str, stores_base_path: str) -> Self:
+    def create_new_store_config(
+        cls, store_id: uuid.UUID, stores_base_path: str
+    ) -> Self:
         raise NotImplementedError(
             f"Store config type '{cls}' does not implement 'create_new_config'."
         )
@@ -55,16 +58,42 @@ class KiaraArchive(abc.ABC):
 
     _config_cls = ArchiveConfig  # type: ignore
 
+    @classmethod
+    def create_config(
+        cls, config: Union["KiaraArchiveConfig", BaseModel, Mapping[str, Any]]
+    ) -> "BaseArchive":
+
+        from kiara.context.config import KiaraArchiveConfig
+
+        if isinstance(config, cls._config_cls):
+            config = config
+        elif isinstance(config, KiaraArchiveConfig):
+            config = cls._config_cls(**config.config)
+        elif isinstance(config, BaseModel):
+            config = cls._config_cls(**config.model_dump())
+        elif isinstance(config, Mapping):
+            config = cls._config_cls(**config)
+
+        return config
+
+    def __init__(self, force_read_only: bool = False, **kwargs):
+        self._force_read_only: bool = force_read_only
+
     @classmethod
     @abc.abstractmethod
     def supported_item_types(cls) -> Iterable[str]:
         pass
 
     @classmethod
     @abc.abstractmethod
-    def is_writeable(cls) -> bool:
+    def _is_writeable(cls) -> bool:
         pass
 
+    def is_writeable(self) -> bool:
+        if self._force_read_only:
+            return False
+        return self.__class__._is_writeable()
+
     @abc.abstractmethod
     def register_archive(self, kiara: "Kiara"):
         pass
@@ -125,7 +154,7 @@ class BaseArchive(KiaraArchive, Generic[ARCHIVE_CONFIG_CLS]):
 
     @classmethod
     def create_new_config(
-        cls, store_id: str, stores_base_path: str
+        cls, store_id: uuid.UUID, stores_base_path: str
     ) -> ARCHIVE_CONFIG_CLS:
 
         log_message(
@@ -139,12 +168,22 @@ def create_new_config(
             store_id=store_id, stores_base_path=stores_base_path
         )
 
-    def __init__(self, archive_id: uuid.UUID, config: ARCHIVE_CONFIG_CLS):
+    def __init__(
+        self,
+        archive_id: uuid.UUID,
+        config: ARCHIVE_CONFIG_CLS,
+        force_read_only: bool = False,
+    ):
 
+        super().__init__(force_read_only=force_read_only)
         self._archive_id: uuid.UUID = archive_id
         self._config: ARCHIVE_CONFIG_CLS = config
         self._kiara: Union["Kiara", None] = None
 
+    @classmethod
+    def _is_writeable(cls) -> bool:
+        return False
+
     def _get_config(self) -> ARCHIVE_CONFIG_CLS:
         return self._config
 

src/kiara/registries/aliases/__init__.py

@@ -19,6 +19,8 @@
 
 import structlog
 
+from kiara.defaults import INVALID_ALIAS_NAMES
+from kiara.exceptions import KiaraException
 from kiara.models.events.alias_registry import AliasArchiveAddedEvent
 from kiara.registries import BaseArchive
 from kiara.registries.data import ValueLink
@@ -55,18 +57,14 @@ def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]:
     def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]:
         pass
 
-    @classmethod
-    def is_writeable(cls) -> bool:
-        return False
-
 
 class AliasStore(AliasArchive):
     @abc.abstractmethod
     def register_aliases(self, value_id: uuid.UUID, *aliases: str):
         pass
 
     @classmethod
-    def is_writeable(cls) -> bool:
+    def _is_writeable(cls) -> bool:
         return True
 
 
@@ -278,6 +276,13 @@ def register_aliases(
         allow_overwrite: bool = False,
     ):
 
+        for alias in aliases:
+            if alias in INVALID_ALIAS_NAMES:
+                raise KiaraException(
+                    msg=f"Invalid alias name: {alias}.",
+                    details=f"The following names can't be used as alias: {', '.join(INVALID_ALIAS_NAMES)}.",
+                )
+
         value_id = self._get_value_id(value_id=value_id)
         store_name = self.default_alias_store
         store: AliasStore = self.get_archive(archive_id=store_name)  # type: ignore

src/kiara/registries/aliases/archives.py

@@ -132,10 +132,6 @@ class FileSystemAliasStore(FileSystemAliasArchive, AliasStore):
 
     _archive_type_name = "filesystem_alias_store"
 
-    @classmethod
-    def is_writeable(cls) -> bool:
-        return True
-
     def register_aliases(self, value_id: uuid.UUID, *aliases: str):
 
         value_path = self._translate_value_id(value_id=value_id)

src/kiara/registries/data/__init__.py

@@ -110,6 +110,8 @@ def __init__(self, kiara: "Kiara"):
 
     def resolve_alias(self, alias: str) -> uuid.UUID:
 
+        dbg(f"RESOLVE: {alias}")
+
         if ":" in alias:
             ref_type, rest = alias.split(":", maxsplit=1)
 
@@ -230,7 +232,9 @@ def NONE_VALUE(self) -> Value:
     def retrieve_all_available_value_ids(self) -> Set[uuid.UUID]:
 
         result: Set[uuid.UUID] = set()
-        for store in self._data_archives.values():
+        for alias, store in self._data_archives.items():
+            print(alias)
+            dbg(store.config.model_dump())
             ids = store.value_ids
             if ids:
                 result.update(ids)
@@ -378,24 +382,35 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str]) -> Value:
             _value = self._registered_values[_value_id]
             return _value
 
-        matches = []
-        for store_id, store in self.data_archives.items():
-            match = store.has_value(value_id=_value_id)
-            if match:
-                matches.append(store_id)
+        default_store: DataArchive = self.get_archive(
+            archive_id=self.default_data_store
+        )
+        if not default_store.has_value(value_id=_value_id):
 
-        if len(matches) == 0:
-            raise NoSuchValueIdException(
-                value_id=_value_id, msg=f"No value registered with id: {value}"
-            )
-        elif len(matches) > 1:
-            raise NoSuchValueIdException(
-                value_id=_value_id,
-                msg=f"Found value with id '{value}' in multiple archives, this is not supported (yet): {matches}",
-            )
+            matches = []
+            for store_id, store in self.data_archives.items():
+                match = store.has_value(value_id=_value_id)
+                if match:
+                    matches.append(store_id)
+
+            if len(matches) == 0:
+                raise NoSuchValueIdException(
+                    value_id=_value_id, msg=f"No value registered with id: {value}"
+                )
+            elif len(matches) > 1:
+                raise NoSuchValueIdException(
+                    value_id=_value_id,
+                    msg=f"Found value with id '{value}' in multiple archives, this is not supported (yet): {matches}",
+                )
+            store_that_has_it = matches[0]
+        else:
+            store_that_has_it = self.default_data_store
 
-        self._value_archive_lookup_map[_value_id] = matches[0]
-        stored_value = self.get_archive(matches[0]).retrieve_value(value_id=_value_id)
+        self._value_archive_lookup_map[_value_id] = store_that_has_it
+
+        stored_value = self.get_archive(store_that_has_it).retrieve_value(
+            value_id=_value_id
+        )
         stored_value._set_registry(self)
         stored_value._is_stored = True