Skip to content

Commit 4e61969

Browse files
makkusmakkus
committed
feat: adding job & alias sql stores
1 parent c595892 commit 4e61969

File tree

16 files changed

+623
-121
lines changed

16 files changed

+623
-121
lines changed

‎pyproject.toml

+4
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,10 @@ filesystem_workflow_archive = "kiara.registries.workflows.archives:FileSystemWor
112112
filesystem_workflow_store = "kiara.registries.workflows.archives:FileSystemWorkflowStore"
113113
sqlite_data_archive = "kiara.registries.data.data_store.sqlite_store:SqliteDataArchive"
114114
sqlite_data_store = "kiara.registries.data.data_store.sqlite_store:SqliteDataStore"
115+
sqlite_alias_archive = "kiara.registries.aliases.sqlite_store:SqliteAliasArchive"
116+
sqlite_alias_store = "kiara.registries.aliases.sqlite_store:SqliteAliasStore"
117+
sqlite_job_archive = "kiara.registries.jobs.job_store.sqlite_store:SqliteJobArchive"
118+
sqlite_job_store = "kiara.registries.jobs.job_store.sqlite_store:SqliteJobStore"
115119

116120
[project.entry-points."kiara.cli_subcommands"]
117121

‎src/kiara/context/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ def __init__(
160160
self._archives: Dict[str, KiaraArchive] = {}
161161

162162
for archive_alias, archive in self._config.archives.items():
163+
163164
archive_cls = self._archive_types.get(archive.archive_type, None)
164165
if archive_cls is None:
165166
raise Exception(

‎src/kiara/context/config.py

+76-94
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import os
99
import uuid
1010
from pathlib import Path
11-
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Union
11+
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Type, Union
1212

1313
import structlog
1414
from pydantic import BaseModel, ConfigDict, field_validator, model_validator
@@ -32,6 +32,7 @@
3232
from kiara.exceptions import KiaraException
3333
from kiara.registries.environment import EnvironmentRegistry
3434
from kiara.registries.ids import ID_REGISTRY
35+
from kiara.utils import log_message
3536
from kiara.utils.files import get_data_from_file
3637

3738
if TYPE_CHECKING:
@@ -118,6 +119,30 @@ class KiaraSettings(BaseSettings):
118119
KIARA_SETTINGS = KiaraSettings()
119120

120121

122+
def create_default_store(
123+
store_id: str, store_type: str, stores_base_path: str
124+
) -> KiaraArchiveConfig:
125+
126+
env_registry = EnvironmentRegistry.instance()
127+
kiara_types: KiaraTypesRuntimeEnvironment = env_registry.environments["kiara_types"] # type: ignore
128+
available_archives = kiara_types.archive_types
129+
130+
assert store_type in available_archives.item_infos.keys()
131+
132+
from kiara.models.archives import ArchiveTypeInfo
133+
134+
archive_info: ArchiveTypeInfo = available_archives.item_infos[store_type]
135+
cls: Type[BaseArchive] = archive_info.python_class.get_class() # type: ignore
136+
config = cls.create_new_config(store_id=store_id, stores_base_path=stores_base_path)
137+
138+
data_store = KiaraArchiveConfig(
139+
archive_id=store_id,
140+
archive_type=store_type,
141+
config=config.model_dump(),
142+
)
143+
return data_store
144+
145+
121146
class KiaraConfig(BaseSettings):
122147
model_config = SettingsConfigDict(
123148
env_prefix="kiara_", extra="forbid", use_enum_values=True
@@ -309,121 +334,71 @@ def get_context_config(
309334

310335
def _validate_context(self, context_config: KiaraContextConfig) -> bool:
311336

312-
env_registry = EnvironmentRegistry.instance()
313-
from kiara.models.runtime_environment.kiara import KiaraTypesRuntimeEnvironment
337+
changed = False
314338

315-
kiara_types: KiaraTypesRuntimeEnvironment = env_registry.environments["kiara_types"] # type: ignore
316-
available_archives = kiara_types.archive_types
339+
default_store_id = context_config.context_id
317340

318-
changed = False
319341
if DEFAULT_DATA_STORE_MARKER not in context_config.archives.keys():
320342
# data_store_type = "filesystem_data_store"
321343
# TODO: change that back
322344
data_store_type = "sqlite_data_store"
323-
assert data_store_type in available_archives.item_infos.keys()
324-
325-
data_store_id = ID_REGISTRY.generate(comment="default data store id")
326-
data_archive_config = {
327-
"archive_path": os.path.abspath(
328-
os.path.join(
329-
self.stores_base_path, data_store_type, str(data_store_id)
330-
)
331-
)
332-
}
333-
data_store = KiaraArchiveConfig(
334-
archive_id=str(data_store_id),
335-
archive_type=data_store_type,
336-
config=data_archive_config,
345+
346+
data_store = create_default_store(
347+
store_id=default_store_id,
348+
store_type=data_store_type,
349+
stores_base_path=self.stores_base_path,
337350
)
338351
context_config.archives[DEFAULT_DATA_STORE_MARKER] = data_store
339-
340352
changed = True
341353

342354
if DEFAULT_JOB_STORE_MARKER not in context_config.archives.keys():
343-
job_store_type = "filesystem_job_store"
344-
assert job_store_type in available_archives.item_infos.keys()
345-
346-
job_store_id = ID_REGISTRY.generate(comment="default job store id")
347-
job_archive_config = {
348-
"archive_path": os.path.abspath(
349-
os.path.join(
350-
self.stores_base_path, job_store_type, str(job_store_id)
351-
)
352-
)
353-
}
354-
job_store = KiaraArchiveConfig(
355-
archive_id=str(job_store_id),
356-
archive_type=job_store_type,
357-
config=job_archive_config,
355+
356+
# job_store_type = "filesystem_job_store"
357+
job_store_type = "sqlite_job_store"
358+
359+
job_store = create_default_store(
360+
store_id=default_store_id,
361+
store_type=job_store_type,
362+
stores_base_path=self.stores_base_path,
358363
)
359-
context_config.archives[DEFAULT_JOB_STORE_MARKER] = job_store
360364

365+
context_config.archives[DEFAULT_JOB_STORE_MARKER] = job_store
361366
changed = True
362367

363368
if DEFAULT_ALIAS_STORE_MARKER not in context_config.archives.keys():
364369

365370
alias_store_type = "filesystem_alias_store"
366-
assert alias_store_type in available_archives.item_infos.keys()
367-
alias_store_id = ID_REGISTRY.generate(comment="default alias store id")
368-
alias_store_config = {
369-
"archive_path": os.path.abspath(
370-
os.path.join(
371-
self.stores_base_path, alias_store_type, str(alias_store_id)
372-
)
373-
)
374-
}
375-
alias_store = KiaraArchiveConfig(
376-
archive_id=str(alias_store_id),
377-
archive_type=alias_store_type,
378-
config=alias_store_config,
371+
alias_store_type = "sqlite_alias_store"
372+
373+
alias_store = create_default_store(
374+
store_id=default_store_id,
375+
store_type=alias_store_type,
376+
stores_base_path=self.stores_base_path,
379377
)
380378
context_config.archives[DEFAULT_ALIAS_STORE_MARKER] = alias_store
381-
382379
changed = True
383380

384381
if DEFAULT_WORKFLOW_STORE_MARKER not in context_config.archives.keys():
385382

386383
workflow_store_type = "filesystem_workflow_store"
387-
assert workflow_store_type in available_archives.item_infos.keys()
388-
workflow_store_id = ID_REGISTRY.generate(
389-
comment="default workflow store id"
390-
)
391-
workflow_store_config = {
392-
"archive_path": os.path.abspath(
393-
os.path.join(
394-
self.stores_base_path,
395-
workflow_store_type,
396-
str(workflow_store_id),
397-
)
398-
)
399-
}
400-
workflow_store = KiaraArchiveConfig(
401-
archive_id=str(workflow_store_id),
402-
archive_type=workflow_store_type,
403-
config=workflow_store_config,
384+
workflow_store = create_default_store(
385+
store_id=default_store_id,
386+
store_type=workflow_store_type,
387+
stores_base_path=self.stores_base_path,
404388
)
405389
context_config.archives[DEFAULT_WORKFLOW_STORE_MARKER] = workflow_store
406-
407390
changed = True
408391

409392
if METADATA_DESTINY_STORE_MARKER not in context_config.archives.keys():
393+
410394
destiny_store_type = "filesystem_destiny_store"
411-
assert destiny_store_type in available_archives.item_infos.keys()
412-
destiny_store_id = ID_REGISTRY.generate(comment="default destiny store id")
413-
destiny_store_config = {
414-
"archive_path": os.path.abspath(
415-
os.path.join(
416-
self.stores_base_path, destiny_store_type, str(destiny_store_id)
417-
)
418-
)
419-
}
420-
destiny_store = KiaraArchiveConfig(
421-
archive_id=str(destiny_store_id),
422-
archive_type=destiny_store_type,
423-
config=destiny_store_config,
395+
396+
destiny_store = create_default_store(
397+
store_id=default_store_id,
398+
store_type=destiny_store_type,
399+
stores_base_path=self.stores_base_path,
424400
)
425401
context_config.archives[METADATA_DESTINY_STORE_MARKER] = destiny_store
426-
427402
changed = True
428403

429404
return changed
@@ -547,7 +522,7 @@ def save(self, path: Union[Path, None] = None):
547522

548523
def delete(
549524
self, context_name: Union[str, None] = None, dry_run: bool = True
550-
) -> "ContextInfo":
525+
) -> Union["ContextInfo", None]:
551526

552527
if context_name is None:
553528
context_name = self.default_context
@@ -558,20 +533,27 @@ def delete(
558533
context_config = self.get_context_config(
559534
context_name=context_name, auto_generate=False
560535
)
561-
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
562536

563-
context_summary = ContextInfo.create_from_context(
564-
kiara=kiara, context_name=context_name
565-
)
537+
context_summary = None
538+
539+
try:
540+
kiara = Kiara(config=context_config, runtime_config=self.runtime_config)
541+
542+
context_summary = ContextInfo.create_from_context(
543+
kiara=kiara, context_name=context_name
544+
)
566545

567-
if dry_run:
568-
return context_summary
546+
if dry_run:
547+
return context_summary
569548

570-
for archive in kiara.get_all_archives().keys():
571-
archive.delete_archive(archive_id=archive.archive_id)
549+
for archive in kiara.get_all_archives().keys():
550+
archive.delete_archive(archive_id=archive.archive_id)
551+
except Exception as e:
552+
log_message("delete.context.error", context_name=context_name, error=e)
572553

573-
if context_config._context_config_path is not None:
574-
os.unlink(context_config._context_config_path)
554+
if not dry_run:
555+
if context_config._context_config_path is not None:
556+
os.unlink(context_config._context_config_path)
575557

576558
return context_summary
577559

‎src/kiara/defaults.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@
9494
DEFAULT_WORKFLOW_STORE_MARKER = "default_workflow_store"
9595
"""Name for the default context workflow store."""
9696

97-
METADATA_DESTINY_STORE_MARKER = "metadata"
97+
METADATA_DESTINY_STORE_MARKER = "default_destiny_store"
9898
"""Name for the default context destiny store."""
9999

100100
PIPELINE_PARENT_MARKER = "__pipeline__"

‎src/kiara/interfaces/cli/context/commands.py

+11-5
Original file line numberDiff line numberDiff line change
@@ -150,11 +150,17 @@ def delete_context(
150150
context_summary = kiara_config.delete(
151151
context_name=_context_name, dry_run=True
152152
)
153-
terminal_print_model(
154-
context_summary,
155-
full_details=True,
156-
in_panel=f"Context details: {_context_name}",
157-
)
153+
terminal_print()
154+
if not context_summary:
155+
terminal_print(
156+
f"Can't determine context details for context '{_context_name}'."
157+
)
158+
else:
159+
terminal_print_model(
160+
context_summary,
161+
full_details=True,
162+
in_panel=f"Context details: {_context_name}",
163+
)
158164
terminal_print()
159165
user_input = get_console().input(
160166
f"Deleting context '[b i]{_context_name}[/b i]', are you sure? \[yes/no]: "

0 commit comments

Comments
 (0)