From 6c86bfe2f173f53fdcda57a677a1316703ffa47d Mon Sep 17 00:00:00 2001 From: Alexander Zeitler Date: Wed, 4 Mar 2026 16:16:04 +0100 Subject: [PATCH 1/2] Add markdown file export for memories Export each memory as a markdown file with YAML frontmatter to the filesystem. Folder structure mirrors workspace/project hierarchy. Includes auto-sync on startup, bulk export via Tools UI, and automatic export on CRUD operations. --- docker-compose.yml | 2 + src/Memorizer/Actors/MarkdownExportActor.cs | 206 ++++++++ .../Actors/MarkdownExportMessages.cs | 36 ++ src/Memorizer/Controllers/ToolsController.cs | 91 ++++ .../Extensions/ServiceCollectionExtensions.cs | 24 + src/Memorizer/Routes.cs | 32 ++ .../Services/IMarkdownExportService.cs | 40 ++ .../Services/InitializationService.cs | 32 ++ .../Services/MarkdownExportService.cs | 463 ++++++++++++++++++ src/Memorizer/Services/Memory.cs | 86 +++- .../Settings/MarkdownExportSettings.cs | 7 + src/Memorizer/Views/Tools/Index.cshtml | 16 +- .../Views/Tools/MarkdownExport.cshtml | 237 +++++++++ src/Memorizer/appsettings.json | 4 + 14 files changed, 1265 insertions(+), 11 deletions(-) create mode 100644 src/Memorizer/Actors/MarkdownExportActor.cs create mode 100644 src/Memorizer/Actors/MarkdownExportMessages.cs create mode 100644 src/Memorizer/Services/IMarkdownExportService.cs create mode 100644 src/Memorizer/Services/MarkdownExportService.cs create mode 100644 src/Memorizer/Settings/MarkdownExportSettings.cs create mode 100644 src/Memorizer/Views/Tools/MarkdownExport.cshtml diff --git a/docker-compose.yml b/docker-compose.yml index bfecc27..1901efe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,8 @@ services: - MEMORIZER_LLM__Chunking__MinCharactersForChunking=2000 - MEMORIZER_LLM__Chunking__TargetChunkSize=1500 - MEMORIZER_Server__CanonicalUrl=http://localhost:5000 + - MEMORIZER_MarkdownExport__RootPath=/data/markdown + - MEMORIZER_MarkdownExport__AutoSyncOnStartup=false - ASPNETCORE_ENVIRONMENT=Development ports: - "5000:8080" diff --git a/src/Memorizer/Actors/MarkdownExportActor.cs b/src/Memorizer/Actors/MarkdownExportActor.cs new file mode 100644 index 0000000..22a83e9 --- /dev/null +++ b/src/Memorizer/Actors/MarkdownExportActor.cs @@ -0,0 +1,206 @@ +using Akka.Actor; +using Akka.Event; +using Akka.Streams; +using Memorizer.Services; +using Microsoft.Extensions.DependencyInjection; + +namespace Memorizer.Actors; + +public sealed class MarkdownExportActor : ReceiveActor +{ + private readonly IServiceProvider _serviceProvider; + private readonly ILoggingAdapter _logger; + private readonly IMaterializer _materializer; + + private ProgressJobManager? _jobManager; + private IServiceScope? _currentScope; + + public MarkdownExportActor(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + _logger = Context.GetLogger(); + _materializer = Context.System.Materializer(); + + Idle(); + } + + private void Idle() + { + ReceiveAsync(HandleStartExport); + + Receive(msg => + { + _logger.Debug("Subscription requested while idle, subscriber: {0}", msg.SubscriberId); + var tempManager = new ProgressJobManager(_logger, _materializer); + var reader = tempManager.CreateIdleSubscription(msg.SubscriberId); + Sender.Tell(new ProgressSubscription(msg.SubscriberId, reader)); + }); + + Receive(msg => + { + _logger.Debug("Unsubscribe requested while idle, subscriber: {0}", msg.SubscriberId); + }); + + Receive(_ => HandleGetStatusIdle()); + } + + private void Running() + { + Receive(msg => + { + _logger.Warning("Markdown export already running, rejecting new request from {0}", msg.RequestedBy); + Sender.Tell(new MarkdownExportStatus( + IsRunning: true, + Status: "Already running" + )); + }); + + Receive(msg => + { + if (_jobManager != null) + { + _logger.Debug("Adding subscriber to running job: {0}", msg.SubscriberId); + var reader = _jobManager.AddSubscriber(msg.SubscriberId); + Sender.Tell(new ProgressSubscription(msg.SubscriberId, reader)); + } + }); + + Receive(msg => + { + _logger.Debug("Removing subscriber: {0}", msg.SubscriberId); + _jobManager?.RemoveSubscriber(msg.SubscriberId); + }); + + Receive(_ => HandleGetStatusRunning()); + Receive(HandleExportCompleted); + } + + private async Task HandleStartExport(StartMarkdownExport message) + { + var sender = Sender; + var self = Self; + + _logger.Info("Starting markdown export, requested by {0}", message.RequestedBy); + + try + { + _currentScope = _serviceProvider.CreateScope(); + var exportService = _currentScope.ServiceProvider.GetRequiredService(); + + if (!exportService.IsEnabled) + { + sender.Tell(new MarkdownExportStatus( + IsRunning: false, + Status: "Feature disabled - RootPath not configured" + )); + _currentScope.Dispose(); + _currentScope = null; + return; + } + + _jobManager = new ProgressJobManager(_logger, _materializer); + _jobManager.StartJob(1, message.RequestedBy); + + Become(Running); + + sender.Tell(new MarkdownExportStatus( + IsRunning: true, + Status: "Running", + TotalProcessed: 0, + TotalSuccessful: 0, + TotalFailed: 0, + Outstanding: 1, + StartTime: _jobManager.StartTime, + Duration: TimeSpan.Zero, + RequestedBy: message.RequestedBy + )); + + var progress = new Progress(p => + { + // Update job manager with progress - we track total items dynamically + if (p.TotalItems > 0 && _jobManager.TotalItems != p.TotalItems) + { + // Re-initialize with correct total + // Note: ProgressJobManager doesn't support changing total mid-job, + // but we broadcast via the SSE event + } + }); + + var result = await exportService.ExportAllAsync( + message.WorkspaceFilter, + message.ProjectFilter, + progress, + CancellationToken.None); + + var completed = new MarkdownExportCompleted( + RequestedBy: message.RequestedBy, + StartTime: _jobManager.StartTime, + TotalExported: result.TotalExported, + TotalFailed: result.TotalFailed, + TotalSkipped: result.TotalSkipped, + Duration: DateTime.UtcNow - _jobManager.StartTime + ); + + self.Tell(completed); + } + catch (Exception ex) + { + _logger.Error(ex, "Error during markdown export: {0}", ex.Message); + sender.Tell(new MarkdownExportStatus( + IsRunning: false, + Status: "Failed: " + ex.Message + )); + _jobManager?.Fail(ex.Message); + _jobManager = null; + _currentScope?.Dispose(); + _currentScope = null; + Become(Idle); + } + } + + private void HandleExportCompleted(MarkdownExportCompleted message) + { + _logger.Info("Markdown export completed: {0} exported, {1} failed, {2} skipped, duration: {3}ms", + message.TotalExported, message.TotalFailed, message.TotalSkipped, message.Duration.TotalMilliseconds); + + Context.System.EventStream.Publish(message); + + _jobManager?.RecordSuccess(); + _jobManager?.Complete(); + _jobManager = null; + + _currentScope?.Dispose(); + _currentScope = null; + + Become(Idle); + } + + private void HandleGetStatusIdle() + { + Sender.Tell(new MarkdownExportStatus( + IsRunning: false, + Status: "idle" + )); + } + + private void HandleGetStatusRunning() + { + if (_jobManager == null) + { + HandleGetStatusIdle(); + return; + } + + Sender.Tell(new MarkdownExportStatus( + IsRunning: true, + Status: "Running", + TotalProcessed: _jobManager.ProcessedCount, + TotalSuccessful: _jobManager.SuccessCount, + TotalFailed: _jobManager.FailureCount, + Outstanding: _jobManager.TotalItems - _jobManager.ProcessedCount, + StartTime: _jobManager.StartTime, + Duration: DateTime.UtcNow - _jobManager.StartTime, + RequestedBy: _jobManager.RequestedBy + )); + } +} diff --git a/src/Memorizer/Actors/MarkdownExportMessages.cs b/src/Memorizer/Actors/MarkdownExportMessages.cs new file mode 100644 index 0000000..970109f --- /dev/null +++ b/src/Memorizer/Actors/MarkdownExportMessages.cs @@ -0,0 +1,36 @@ +using Memorizer.Models; + +namespace Memorizer.Actors; + +public sealed record StartMarkdownExport +{ + public required string RequestedBy { get; init; } + public WorkspaceId? WorkspaceFilter { get; init; } + public ProjectId? ProjectFilter { get; init; } +} + +public record GetMarkdownExportStatus; + +public record MarkdownExportStatus( + bool IsRunning, + string Status, + int? TotalProcessed = null, + int? TotalSuccessful = null, + int? TotalFailed = null, + int? TotalSkipped = null, + int? Outstanding = null, + DateTime? StartTime = null, + TimeSpan? Duration = null, + string? RequestedBy = null +); + +public sealed record MarkdownExportCompleted( + string RequestedBy, + DateTime StartTime, + int TotalExported, + int TotalFailed, + int TotalSkipped, + TimeSpan Duration +); + +public sealed class MarkdownExportActorKey; diff --git a/src/Memorizer/Controllers/ToolsController.cs b/src/Memorizer/Controllers/ToolsController.cs index bc39b2a..abc70d1 100644 --- a/src/Memorizer/Controllers/ToolsController.cs +++ b/src/Memorizer/Controllers/ToolsController.cs @@ -1,6 +1,7 @@ using Akka.Actor; using Akka.Hosting; using Memorizer.Actors; +using Memorizer.Models; using Memorizer.Services; using Memorizer.Settings; using Microsoft.AspNetCore.Mvc; @@ -16,9 +17,11 @@ public class ToolsController : Controller private readonly IActorRef _embeddingRegenerationActor; private readonly IActorRef _versionPurgeActor; private readonly IActorRef _dimensionMigrationActor; + private readonly IActorRef _markdownExportActor; private readonly IOptionsSnapshot _llmSettingsSnapshot; private readonly IEmbeddingDimensionService _dimensionService; private readonly IDimensionMismatchState _mismatchState; + private readonly MarkdownExportSettings _markdownExportSettings; private readonly ILogger _logger; // Convenience property to access current settings @@ -29,18 +32,22 @@ public ToolsController( IRequiredActor embeddingRegenerationActor, IRequiredActor versionPurgeActor, IRequiredActor dimensionMigrationActor, + IRequiredActor markdownExportActor, IOptionsSnapshot llmSettingsSnapshot, IEmbeddingDimensionService dimensionService, IDimensionMismatchState mismatchState, + MarkdownExportSettings markdownExportSettings, ILogger logger) { _titleGenerationActor = titleGenerationActor.ActorRef; _embeddingRegenerationActor = embeddingRegenerationActor.ActorRef; _versionPurgeActor = versionPurgeActor.ActorRef; _dimensionMigrationActor = dimensionMigrationActor.ActorRef; + _markdownExportActor = markdownExportActor.ActorRef; _llmSettingsSnapshot = llmSettingsSnapshot; _dimensionService = dimensionService; _mismatchState = mismatchState; + _markdownExportSettings = markdownExportSettings; _logger = logger; } @@ -557,4 +564,88 @@ public class ResumeMigrationRequest } #endregion + + #region Markdown Export + + [HttpGet] + [Route("markdown-export")] + public IActionResult MarkdownExport() + { + ViewData["IsEnabled"] = !string.IsNullOrWhiteSpace(_markdownExportSettings.RootPath); + ViewData["RootPath"] = _markdownExportSettings.RootPath ?? "(not configured)"; + return View(); + } + + [HttpPost] + [Route("start-markdown-export")] + public async Task StartMarkdownExport(Guid? workspaceId = null, Guid? projectId = null) + { + try + { + if (string.IsNullOrWhiteSpace(_markdownExportSettings.RootPath)) + { + return Json(new { success = false, message = "Markdown export is not configured. Set MarkdownExport:RootPath in configuration." }); + } + + var status = await _markdownExportActor.Ask(new GetMarkdownExportStatus(), TimeSpan.FromSeconds(5)); + if (status.IsRunning) + { + return Json(new { + success = false, + message = "A markdown export is already in progress.", + isRunning = true + }); + } + + var startMessage = new StartMarkdownExport + { + RequestedBy = User.Identity?.Name ?? "Anonymous", + WorkspaceFilter = workspaceId.HasValue ? new WorkspaceId(workspaceId.Value) : null, + ProjectFilter = projectId.HasValue ? new ProjectId(projectId.Value) : null + }; + + var startStatus = await _markdownExportActor.Ask( + startMessage, TimeSpan.FromSeconds(30)); + + return Json(new { + success = true, + message = "Markdown export started", + isRunning = startStatus.IsRunning + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error starting markdown export: {Error}", ex.Message); + return Json(new { success = false, message = $"Error: {ex.Message}" }); + } + } + + [HttpGet] + [Route("markdown-export-status")] + public async Task GetMarkdownExportStatus() + { + try + { + var status = await _markdownExportActor.Ask(new GetMarkdownExportStatus(), TimeSpan.FromSeconds(5)); + return Json(new { + success = true, + status = status.Status, + isRunning = status.IsRunning, + outstanding = status.Outstanding, + totalProcessed = status.TotalProcessed, + totalSuccessful = status.TotalSuccessful, + totalFailed = status.TotalFailed, + requestedBy = status.RequestedBy, + startTime = status.StartTime, + duration = status.Duration?.TotalSeconds + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting markdown export status: {Error}", ex.Message); + return Json(new { success = false, message = $"Error: {ex.Message}" }); + } + } + + #endregion } \ No newline at end of file diff --git a/src/Memorizer/Extensions/ServiceCollectionExtensions.cs b/src/Memorizer/Extensions/ServiceCollectionExtensions.cs index 9f210c2..3e87d44 100644 --- a/src/Memorizer/Extensions/ServiceCollectionExtensions.cs +++ b/src/Memorizer/Extensions/ServiceCollectionExtensions.cs @@ -24,6 +24,8 @@ public static IServiceCollection AddMemorizer( services.AddSimilaritySettings(); services.AddSearchSettings(); services.AddCanonicalUrlService(); + services.AddMarkdownExportSettings(); + services.AddMarkdownExportService(); if(initialize) services.AddHostedService(); services.AutoRegisterTypesInAssemblies(typeof(Storage).Assembly); @@ -100,6 +102,11 @@ public static IServiceCollection AddActorSystem( var dimensionMigrationActorProps = resolver.Props(); var dimensionMigrationActor = system.ActorOf(dimensionMigrationActorProps, "dimension-migration"); registry.Register(dimensionMigrationActor); + + // Create and register the MarkdownExportActor + var markdownExportActorProps = resolver.Props(); + var markdownExportActor = system.ActorOf(markdownExportActorProps, "markdown-export"); + registry.Register(markdownExportActor); }); // TODO: Configure Akka.Persistence.Sql with PostgreSQL @@ -182,6 +189,23 @@ public static IServiceCollection AddCanonicalUrlService( return services; } + public static IServiceCollection AddMarkdownExportSettings( + this IServiceCollection services) + { + services.AddSingleton(sp => + sp.GetRequiredService().GetSection("MarkdownExport").Get() ?? + new MarkdownExportSettings()); + + return services; + } + + public static IServiceCollection AddMarkdownExportService( + this IServiceCollection services) + { + services.AddScoped(); + return services; + } + public static IServiceCollection AddHealthChecks( this IServiceCollection services, IConfiguration configuration) diff --git a/src/Memorizer/Routes.cs b/src/Memorizer/Routes.cs index b4635e5..41fc50e 100644 --- a/src/Memorizer/Routes.cs +++ b/src/Memorizer/Routes.cs @@ -143,6 +143,38 @@ async IAsyncEnumerable> StreamProgress() } }); + // SSE endpoint for markdown export progress + app.MapGet("/tools/markdown-export-progress", + async (IActorRegistry actorRegistry, CancellationToken ct) => + { + var markdownExportActor = await actorRegistry.GetAsync(ct); + var subscriberId = Guid.NewGuid().ToString(); + + var subscription = await markdownExportActor.Ask( + new SubscribeToProgress(subscriberId), + ct); + + return TypedResults.ServerSentEvents(StreamProgress()); + + async IAsyncEnumerable> StreamProgress() + { + try + { + await foreach (var progress in subscription.Reader.ReadAllAsync(ct)) + { + yield return new SseItem(progress, "progress") + { + EventId = Guid.NewGuid().ToString() + }; + } + } + finally + { + markdownExportActor.Tell(new UnsubscribeFromProgress(subscriberId), ActorRefs.NoSender); + } + } + }); + return app; } } diff --git a/src/Memorizer/Services/IMarkdownExportService.cs b/src/Memorizer/Services/IMarkdownExportService.cs new file mode 100644 index 0000000..ae010b8 --- /dev/null +++ b/src/Memorizer/Services/IMarkdownExportService.cs @@ -0,0 +1,40 @@ +using Memorizer.Models; + +namespace Memorizer.Services; + +public class ExportProgress +{ + public int TotalItems { get; set; } + public int ProcessedItems { get; set; } + public int SuccessfulItems { get; set; } + public int FailedItems { get; set; } +} + +public class ExportResult +{ + public int TotalExported { get; set; } + public int TotalFailed { get; set; } + public int TotalSkipped { get; set; } + public List Errors { get; set; } = []; +} + +public interface IMarkdownExportService +{ + bool IsEnabled { get; } + + Task ExportMemoryAsync(Models.Memory memory, CancellationToken ct = default); + + Task DeleteMemoryFileAsync(MemoryId id, CancellationToken ct = default); + + Task RenameWorkspaceFolderAsync(WorkspaceId id, string oldSlug, string newSlug, CancellationToken ct = default); + + Task RenameProjectFolderAsync(ProjectId id, string oldSlug, string newSlug, CancellationToken ct = default); + + Task MoveMemoryFileAsync(MemoryId id, MemoryOwner oldOwner, MemoryOwner newOwner, CancellationToken ct = default); + + Task ExportAllAsync( + WorkspaceId? workspaceFilter = null, + ProjectId? projectFilter = null, + IProgress? progress = null, + CancellationToken ct = default); +} diff --git a/src/Memorizer/Services/InitializationService.cs b/src/Memorizer/Services/InitializationService.cs index e01577a..32e50f2 100644 --- a/src/Memorizer/Services/InitializationService.cs +++ b/src/Memorizer/Services/InitializationService.cs @@ -46,6 +46,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // Seed sample data for local development if enabled await SeedSampleDataAsync(stoppingToken); + + // Auto-sync markdown export if enabled + await AutoSyncMarkdownExportAsync(stoppingToken); } private async Task SeedSampleDataAsync(CancellationToken ct = default) @@ -645,6 +648,35 @@ private async Task CheckAndTriggerEmbeddingMigration(CancellationToken ct = defa } } + private async Task AutoSyncMarkdownExportAsync(CancellationToken ct = default) + { + try + { + var markdownSettings = _services.GetRequiredService(); + if (!markdownSettings.AutoSyncOnStartup || string.IsNullOrWhiteSpace(markdownSettings.RootPath)) + { + _logger.LogDebug("Markdown export auto-sync disabled or not configured, skipping"); + return; + } + + _logger.LogInformation("Auto-syncing markdown export to {RootPath}", markdownSettings.RootPath); + + using var scope = _services.CreateScope(); + var exportService = scope.ServiceProvider.GetRequiredService(); + + var result = await exportService.ExportAllAsync(ct: ct); + + _logger.LogInformation( + "Markdown export auto-sync completed: {Exported} exported, {Failed} failed, {Skipped} skipped", + result.TotalExported, result.TotalFailed, result.TotalSkipped); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to auto-sync markdown export: {Message}", ex.Message); + // Don't fail startup + } + } + /// /// Runs one-time data migration to seed system memories for existing projects and workspaces. /// Uses the data_migrations table to track whether this migration has already been executed. diff --git a/src/Memorizer/Services/MarkdownExportService.cs b/src/Memorizer/Services/MarkdownExportService.cs new file mode 100644 index 0000000..4c003a8 --- /dev/null +++ b/src/Memorizer/Services/MarkdownExportService.cs @@ -0,0 +1,463 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using Memorizer.Models; +using Memorizer.Models.Enums; +using Memorizer.Settings; + +namespace Memorizer.Services; + +public partial class MarkdownExportService : IMarkdownExportService +{ + private readonly MarkdownExportSettings _settings; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + private static readonly SemaphoreSlim _fileLock = new(1, 1); + + public bool IsEnabled => !string.IsNullOrWhiteSpace(_settings.RootPath); + + public MarkdownExportService( + MarkdownExportSettings settings, + IServiceProvider serviceProvider, + ILogger logger) + { + _settings = settings; + _serviceProvider = serviceProvider; + _logger = logger; + } + + private IStorage GetStorage() => _serviceProvider.GetRequiredService(); + + public async Task ExportMemoryAsync(Models.Memory memory, CancellationToken ct = default) + { + if (!IsEnabled) return; + + var folderPath = await ResolveOwnerFolderAsync(memory.Owner, ct); + var fileName = BuildFileName(memory.Title ?? "untitled", memory.Id); + var filePath = Path.Combine(folderPath, fileName); + + var content = BuildMarkdownContent(memory); + + await _fileLock.WaitAsync(ct); + try + { + Directory.CreateDirectory(folderPath); + + // Remove any existing file for this memory ID (title might have changed) + RemoveExistingFileForId(folderPath, memory.Id); + + await File.WriteAllTextAsync(filePath, content, Encoding.UTF8, ct); + _logger.LogDebug("Exported memory {MemoryId} to {FilePath}", memory.Id, filePath); + } + finally + { + _fileLock.Release(); + } + } + + public async Task DeleteMemoryFileAsync(MemoryId id, CancellationToken ct = default) + { + if (!IsEnabled) return; + + await _fileLock.WaitAsync(ct); + try + { + var idSuffix = GetIdSuffix(id); + var files = Directory.GetFiles(_settings.RootPath!, $"*--{idSuffix}.md", SearchOption.AllDirectories); + foreach (var file in files) + { + File.Delete(file); + _logger.LogDebug("Deleted memory file {FilePath}", file); + } + } + finally + { + _fileLock.Release(); + } + } + + public async Task RenameWorkspaceFolderAsync(WorkspaceId id, string oldSlug, string newSlug, CancellationToken ct = default) + { + if (!IsEnabled) return; + + // Build the path to the workspace folder by walking the ancestor chain + var workspace = await GetStorage().GetWorkspaceAsync(id, ct); + if (workspace == null) return; + + var parentPath = await BuildWorkspaceParentPath(id, ct); + var oldPath = Path.Combine(parentPath, oldSlug); + var newPath = Path.Combine(parentPath, newSlug); + + await _fileLock.WaitAsync(ct); + try + { + if (Directory.Exists(oldPath) && oldPath != newPath) + { + Directory.Move(oldPath, newPath); + _logger.LogInformation("Renamed workspace folder from {OldPath} to {NewPath}", oldPath, newPath); + } + } + finally + { + _fileLock.Release(); + } + } + + public async Task RenameProjectFolderAsync(ProjectId id, string oldSlug, string newSlug, CancellationToken ct = default) + { + if (!IsEnabled) return; + + var project = await GetStorage().GetProjectAsync(id, ct); + if (project == null) return; + + var projectParentPath = await BuildProjectParentPath(project, ct); + var oldPath = Path.Combine(projectParentPath, oldSlug); + var newPath = Path.Combine(projectParentPath, newSlug); + + await _fileLock.WaitAsync(ct); + try + { + if (Directory.Exists(oldPath) && oldPath != newPath) + { + Directory.Move(oldPath, newPath); + _logger.LogInformation("Renamed project folder from {OldPath} to {NewPath}", oldPath, newPath); + } + } + finally + { + _fileLock.Release(); + } + } + + public async Task MoveMemoryFileAsync(MemoryId id, MemoryOwner oldOwner, MemoryOwner newOwner, CancellationToken ct = default) + { + if (!IsEnabled) return; + + var idSuffix = GetIdSuffix(id); + + // Find existing file + var oldFolder = await ResolveOwnerFolderAsync(oldOwner, ct); + string? existingFile = null; + + if (Directory.Exists(oldFolder)) + { + var files = Directory.GetFiles(oldFolder, $"*--{idSuffix}.md"); + existingFile = files.FirstOrDefault(); + } + + if (existingFile == null) + { + // No file to move; export fresh + var memory = await GetStorage().Get(id, ct); + if (memory != null) + await ExportMemoryAsync(memory, ct); + return; + } + + var newFolder = await ResolveOwnerFolderAsync(newOwner, ct); + var newFilePath = Path.Combine(newFolder, Path.GetFileName(existingFile)); + + await _fileLock.WaitAsync(ct); + try + { + Directory.CreateDirectory(newFolder); + File.Move(existingFile, newFilePath, overwrite: true); + _logger.LogDebug("Moved memory file from {OldPath} to {NewPath}", existingFile, newFilePath); + } + finally + { + _fileLock.Release(); + } + } + + public async Task ExportAllAsync( + WorkspaceId? workspaceFilter = null, + ProjectId? projectFilter = null, + IProgress? progress = null, + CancellationToken ct = default) + { + if (!IsEnabled) + return new ExportResult(); + + var result = new ExportResult(); + + // Get all memories via pagination + var allMemories = new List(); + const int pageSize = 100; + int page = 1; + + while (!ct.IsCancellationRequested) + { + var (memories, totalCount) = await GetStorage().GetMemoriesPaginated(page, pageSize, ct); + if (memories.Count == 0) break; + + foreach (var m in memories) + { + // Apply filters + if (workspaceFilter.HasValue) + { + if (m.Owner.Type == OwnerTypeEnum.Workspace && m.Owner.WorkspaceId != workspaceFilter.Value) + continue; + if (m.Owner.Type == OwnerTypeEnum.Project) + { + var project = await GetStorage().GetProjectAsync(m.Owner.ProjectId!.Value, ct); + if (project == null || project.WorkspaceId != workspaceFilter.Value) + continue; + } + } + + if (projectFilter.HasValue) + { + if (m.Owner.Type != OwnerTypeEnum.Project || m.Owner.ProjectId != projectFilter.Value) + continue; + } + + allMemories.Add(m); + } + + if (memories.Count < pageSize) break; + page++; + } + + // Also include unfiled memories if no filter or workspace filter matches unfiled + if (!projectFilter.HasValue && (!workspaceFilter.HasValue || workspaceFilter.Value.Value == Guid.Empty)) + { + // Unfiled memories are already included via GetMemoriesPaginated + } + + var progressState = new ExportProgress { TotalItems = allMemories.Count }; + progress?.Report(progressState); + + foreach (var memory in allMemories) + { + if (ct.IsCancellationRequested) break; + + try + { + // Skip system and archived memories + if (memory.Archetype is ArchetypeEnum.System or ArchetypeEnum.Archived) + { + result.TotalSkipped++; + progressState.ProcessedItems++; + progress?.Report(progressState); + continue; + } + + await ExportMemoryAsync(memory, ct); + result.TotalExported++; + progressState.SuccessfulItems++; + } + catch (Exception ex) + { + result.TotalFailed++; + result.Errors.Add($"Memory {memory.Id}: {ex.Message}"); + progressState.FailedItems++; + _logger.LogWarning(ex, "Failed to export memory {MemoryId}", memory.Id); + } + + progressState.ProcessedItems++; + progress?.Report(progressState); + } + + return result; + } + + private async Task ResolveOwnerFolderAsync(MemoryOwner owner, CancellationToken ct) + { + var root = _settings.RootPath!; + + if (owner.IsUnfiled) + return Path.Combine(root, "unfiled"); + + if (owner.Type == OwnerTypeEnum.Workspace) + { + var workspace = await GetStorage().GetWorkspaceAsync(owner.WorkspaceId!.Value, ct); + if (workspace == null) + return Path.Combine(root, "unfiled"); + + return await BuildWorkspaceFolderPath(workspace, ct); + } + + if (owner.Type == OwnerTypeEnum.Project) + { + var project = await GetStorage().GetProjectAsync(owner.ProjectId!.Value, ct); + if (project == null) + return Path.Combine(root, "unfiled"); + + return await BuildProjectFolderPath(project, ct); + } + + return Path.Combine(root, "unfiled"); + } + + private async Task BuildWorkspaceFolderPath(Workspace workspace, CancellationToken ct) + { + var root = _settings.RootPath!; + var segments = new List { root }; + + // Get ancestor path + var ancestors = await GetStorage().GetWorkspacePathAsync(workspace.Id, ct); + foreach (var ancestor in ancestors) + { + var ancestorWs = await GetStorage().GetWorkspaceAsync(ancestor.Id, ct); + segments.Add(ancestorWs?.Slug ?? GenerateSlug(ancestor.Name)); + } + + segments.Add(workspace.Slug); + return Path.Combine(segments.ToArray()); + } + + private async Task BuildWorkspaceParentPath(WorkspaceId id, CancellationToken ct) + { + var root = _settings.RootPath!; + var segments = new List { root }; + + var ancestors = await GetStorage().GetWorkspacePathAsync(id, ct); + foreach (var ancestor in ancestors) + { + var ancestorWs = await GetStorage().GetWorkspaceAsync(ancestor.Id, ct); + segments.Add(ancestorWs?.Slug ?? GenerateSlug(ancestor.Name)); + } + + return Path.Combine(segments.ToArray()); + } + + private async Task BuildProjectFolderPath(Project project, CancellationToken ct) + { + // Start with workspace path + var workspace = await GetStorage().GetWorkspaceAsync(project.WorkspaceId, ct); + string workspacePath; + if (workspace != null) + { + workspacePath = await BuildWorkspaceFolderPath(workspace, ct); + } + else + { + workspacePath = Path.Combine(_settings.RootPath!, "unfiled"); + } + + // Build project ancestor chain + var projectSegments = new List(); + var projectPath = await GetStorage().GetProjectPathAsync(project.Id, ct); + foreach (var ancestor in projectPath.ProjectAncestors) + { + if (!ancestor.IsWorkspace) + { + var ancestorProject = await GetStorage().GetProjectAsync(new ProjectId(ancestor.Id), ct); + projectSegments.Add(ancestorProject?.Slug ?? GenerateSlug(ancestor.Name)); + } + } + + projectSegments.Add(project.Slug); + + return Path.Combine(workspacePath, Path.Combine(projectSegments.ToArray())); + } + + private async Task BuildProjectParentPath(Project project, CancellationToken ct) + { + var workspace = await GetStorage().GetWorkspaceAsync(project.WorkspaceId, ct); + string workspacePath; + if (workspace != null) + { + workspacePath = await BuildWorkspaceFolderPath(workspace, ct); + } + else + { + workspacePath = Path.Combine(_settings.RootPath!, "unfiled"); + } + + // Build project ancestor chain (excluding the project itself) + var projectPath = await GetStorage().GetProjectPathAsync(project.Id, ct); + var segments = new List { workspacePath }; + foreach (var ancestor in projectPath.ProjectAncestors) + { + if (!ancestor.IsWorkspace) + { + var ancestorProject = await GetStorage().GetProjectAsync(new ProjectId(ancestor.Id), ct); + segments.Add(ancestorProject?.Slug ?? GenerateSlug(ancestor.Name)); + } + } + + return Path.Combine(segments.ToArray()); + } + + private static string BuildMarkdownContent(Models.Memory memory) + { + var sb = new StringBuilder(); + + // YAML Frontmatter + sb.AppendLine("---"); + sb.Append("title: \"").Append(EscapeYamlString(memory.Title ?? "Untitled")).AppendLine("\""); + sb.Append("id: ").AppendLine(memory.Id.Value.ToString()); + sb.Append("type: ").AppendLine(memory.Type ?? "reference"); + + if (memory.Tags is { Length: > 0 }) + { + sb.AppendLine("tags:"); + foreach (var tag in memory.Tags) + { + sb.Append(" - ").AppendLine(tag); + } + } + + sb.Append("confidence: ").AppendLine(((double)memory.Confidence).ToString(CultureInfo.InvariantCulture)); + sb.Append("source: ").AppendLine(memory.Source ?? "unknown"); + sb.Append("archetype: ").AppendLine(memory.Archetype.ToString().ToLowerInvariant()); + sb.Append("version: ").AppendLine(((int)memory.CurrentVersion).ToString()); + sb.Append("created: ").AppendLine(memory.CreatedAt.ToString("O")); + sb.Append("updated: ").AppendLine(memory.UpdatedAt.ToString("O")); + sb.AppendLine("---"); + sb.AppendLine(); + + // Body + sb.Append(memory.Text); + + return sb.ToString(); + } + + internal static string BuildFileName(string title, MemoryId id) + { + var slug = GenerateSlug(title); + var idSuffix = GetIdSuffix(id); + return $"{slug}--{idSuffix}.md"; + } + + private static string GetIdSuffix(MemoryId id) + { + return id.Value.ToString("N")[..8]; + } + + private void RemoveExistingFileForId(string folder, MemoryId id) + { + if (!Directory.Exists(folder)) return; + + var idSuffix = GetIdSuffix(id); + var existing = Directory.GetFiles(folder, $"*--{idSuffix}.md"); + foreach (var file in existing) + { + File.Delete(file); + } + } + + private static string GenerateSlug(string name) + { + var slug = name.ToLowerInvariant() + .Replace(' ', '-') + .Replace("--", "-"); + + slug = SlugRegex().Replace(slug, ""); + slug = slug.Trim('-'); + + return string.IsNullOrEmpty(slug) ? "unnamed" : slug; + } + + private static string EscapeYamlString(string value) + { + return value + .Replace("\\", "\\\\") + .Replace("\"", "\\\""); + } + + [GeneratedRegex(@"[^a-z0-9\-]")] + private static partial Regex SlugRegex(); +} diff --git a/src/Memorizer/Services/Memory.cs b/src/Memorizer/Services/Memory.cs index 90de083..37e6a0e 100644 --- a/src/Memorizer/Services/Memory.cs +++ b/src/Memorizer/Services/Memory.cs @@ -447,13 +447,15 @@ public class Storage : IStorage private readonly IEmbeddingService _embeddingService; private readonly IDiffService _diffService; private readonly VersioningSettings _versioningSettings; + private readonly IMarkdownExportService? _markdownExportService; - public Storage(NpgsqlDataSource dataSource, IEmbeddingService embeddingService, IDiffService diffService, VersioningSettings versioningSettings) + public Storage(NpgsqlDataSource dataSource, IEmbeddingService embeddingService, IDiffService diffService, VersioningSettings versioningSettings, IMarkdownExportService? markdownExportService = null) { _dataSource = dataSource; _embeddingService = embeddingService; _diffService = diffService; _versioningSettings = versioningSettings; + _markdownExportService = markdownExportService; } /// @@ -606,6 +608,13 @@ INSERT INTO memories (id, type_legacy, content, text, source, embedding, embeddi await CreateRelationship(memory.Id, relatedTo.Value, relationshipType, cancellationToken); } + // Export to markdown file if enabled + if (_markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.ExportMemoryAsync(memory, cancellationToken); } + catch { /* Don't fail the store operation */ } + } + return memory; } @@ -740,6 +749,13 @@ public async Task Delete( cmd.Parameters.AddWithValue("id", id.Value); int rowsAffected = await cmd.ExecuteNonQueryAsync(cancellationToken); + + if (rowsAffected > 0 && _markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.DeleteMemoryFileAsync(id, cancellationToken); } + catch { /* Don't fail the delete operation */ } + } + return rowsAffected > 0; } @@ -1099,7 +1115,13 @@ UPDATE memories // Return the updated memory if successful if (rowsAffected > 0) { - return await Get(id, cancellationToken); + var updatedMemory = await Get(id, cancellationToken); + if (updatedMemory != null && _markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.ExportMemoryAsync(updatedMemory, cancellationToken); } + catch { /* Don't fail the update operation */ } + } + return updatedMemory; } return null; @@ -2588,6 +2610,14 @@ public async Task UpdateWorkspaceAsync( bool updateParent = newParentId != null || makeTopLevel; var slug = name != null ? GenerateSlug(name) : null; + // Capture old slug before updating (needed for markdown export folder rename) + string? oldSlug = null; + if (name != null && _markdownExportService is { IsEnabled: true }) + { + var existingWorkspace = await GetWorkspaceAsync(id, cancellationToken); + oldSlug = existingWorkspace?.Slug; + } + var sql = @" UPDATE workspaces SET name = COALESCE(@name, name), @@ -2619,6 +2649,13 @@ UPDATE workspaces SET await CreateOrUpdateWorkspaceSystemMemoryAsync(workspace, cancellationToken); } + // Rename markdown export folder if name changed + if (name != null && oldSlug != null && oldSlug != workspace.Slug && _markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.RenameWorkspaceFolderAsync(id, oldSlug, workspace.Slug, cancellationToken); } + catch { /* Don't fail the update operation */ } + } + return workspace; } @@ -3011,6 +3048,14 @@ public async Task UpdateProjectAsync( var slug = name != null ? GenerateSlug(name) : null; + // Capture old slug before updating (needed for markdown export folder rename) + string? oldProjectSlug = null; + if (name != null && _markdownExportService is { IsEnabled: true }) + { + var existingProject = await GetProjectAsync(id, cancellationToken); + oldProjectSlug = existingProject?.Slug; + } + // Determine how to handle parent_id in the UPDATE // If newParentId is specified, set parent_id to that value // If makeTopLevel is true, set parent_id to NULL @@ -3062,6 +3107,13 @@ ELSE completed_at await CreateOrUpdateProjectSystemMemoryAsync(project, cancellationToken); } + // Rename markdown export folder if name changed + if (name != null && oldProjectSlug != null && oldProjectSlug != project.Slug && _markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.RenameProjectFolderAsync(id, oldProjectSlug, project.Slug, cancellationToken); } + catch { /* Don't fail the update operation */ } + } + return project; } @@ -3773,6 +3825,14 @@ FROM projects p public async Task SetMemoryOwnerAsync(MemoryId memoryId, MemoryOwner owner, CancellationToken cancellationToken = default) { + // Capture old owner before updating (needed for markdown export file move) + MemoryOwner? oldOwner = null; + if (_markdownExportService is { IsEnabled: true }) + { + var existing = await Get(memoryId, cancellationToken); + oldOwner = existing?.Owner; + } + await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); await using var cmd = new NpgsqlCommand(@" UPDATE memories SET @@ -3788,6 +3848,12 @@ UPDATE memories SET var affected = await cmd.ExecuteNonQueryAsync(cancellationToken); if (affected == 0) throw new InvalidOperationException($"Memory {memoryId} not found"); + + if (oldOwner.HasValue && _markdownExportService is { IsEnabled: true }) + { + try { await _markdownExportService.MoveMemoryFileAsync(memoryId, oldOwner.Value, owner, cancellationToken); } + catch { /* Don't fail the owner change operation */ } + } } public Task MoveMemoryToUnfiledAsync(MemoryId memoryId, CancellationToken cancellationToken = default) @@ -4053,7 +4119,21 @@ UPDATE memories return null; // Return the updated memory - return await Get(memoryId, cancellationToken); + var updatedMemory = await Get(memoryId, cancellationToken); + + if (_markdownExportService is { IsEnabled: true } && updatedMemory != null) + { + try + { + if (newArchetype == ArchetypeEnum.Archived) + await _markdownExportService.DeleteMemoryFileAsync(memoryId, cancellationToken); + else + await _markdownExportService.ExportMemoryAsync(updatedMemory, cancellationToken); + } + catch { /* Don't fail the archetype update */ } + } + + return updatedMemory; } public async Task<(IReadOnlyList Memories, int TotalCount)> GetArchivedMemoriesAsync( diff --git a/src/Memorizer/Settings/MarkdownExportSettings.cs b/src/Memorizer/Settings/MarkdownExportSettings.cs new file mode 100644 index 0000000..13677d1 --- /dev/null +++ b/src/Memorizer/Settings/MarkdownExportSettings.cs @@ -0,0 +1,7 @@ +namespace Memorizer.Settings; + +public class MarkdownExportSettings +{ + public string? RootPath { get; set; } + public bool AutoSyncOnStartup { get; set; } = false; +} diff --git a/src/Memorizer/Views/Tools/Index.cshtml b/src/Memorizer/Views/Tools/Index.cshtml index 064a6a1..f2daa67 100644 --- a/src/Memorizer/Views/Tools/Index.cshtml +++ b/src/Memorizer/Views/Tools/Index.cshtml @@ -107,21 +107,21 @@
- - Performance Analytics + + Markdown Export

- View analytics and performance metrics for automated tools. + Export memories as Markdown files with YAML frontmatter.

- Coming soon: Track how tools perform, success rates, and processing times. + Export all memories to the filesystem as .md files organized by workspace and project hierarchy.

- + + + Export Memories +
diff --git a/src/Memorizer/Views/Tools/MarkdownExport.cshtml b/src/Memorizer/Views/Tools/MarkdownExport.cshtml new file mode 100644 index 0000000..baf7b13 --- /dev/null +++ b/src/Memorizer/Views/Tools/MarkdownExport.cshtml @@ -0,0 +1,237 @@ +@{ + ViewData["Title"] = "Markdown Export Tool"; + var isEnabled = (bool)ViewData["IsEnabled"]!; + var rootPath = (string)ViewData["RootPath"]!; +} + +
+
+
+ + +

+ + Markdown Export Tool +

+

Export memories as Markdown files with YAML frontmatter to the filesystem.

+
+
+ + @if (!isEnabled) + { +
+
+ +
+
+ } + +
+
+
+
+
Export Memories
+
+
+
+ +
+ @if (isEnabled) + { + Enabled + } + else + { + Disabled + } + Root path: @rootPath +
+
+ +
+
+ + +
+
+
+
+ + + + + + +
+ +
+
+
+
How It Works
+
+
+
    +
  1. + Scan Memories
    + Iterate through all memories in the database +
    2. +
  2. + Build Folder Structure
    + Create folders mirroring workspace/project hierarchy +
    3. +
  3. + Generate Files
    + Write .md files with YAML frontmatter and memory content +
    4. +
  4. + Report Results
    + Show export summary with success/failure counts +
    5. +
+
+
+ +
+
+
File Format
+
+
+

Each memory is exported as:

+ 
---
+title: "Memory Title"
+id: a1b2c3d4-...
+type: reference
+tags:
+  - tag1
+  - tag2
+confidence: 0.95
+source: LLM
+archetype: document
+version: 3
+created: 2026-03-04T12:00:00Z
+updated: 2026-03-04T14:30:00Z
+---
+
+Memory content as markdown...
+

+ Filename: {title-slug}--{short-id}.md +

+
+
+
+
+
+ +@section Scripts { + +} diff --git a/src/Memorizer/appsettings.json b/src/Memorizer/appsettings.json index 9342da8..68d4808 100644 --- a/src/Memorizer/appsettings.json +++ b/src/Memorizer/appsettings.json @@ -41,5 +41,9 @@ }, "Seeding": { "Enabled": true + }, + "MarkdownExport": { + "RootPath": null, + "AutoSyncOnStartup": false } } \ No newline at end of file From 284d5f0b4f1480952df9a22e8b219bc04fae4157 Mon Sep 17 00:00:00 2001 From: Alexander Zeitler Date: Sun, 8 Mar 2026 04:48:39 +0100 Subject: [PATCH 2/2] Fix GetMemoriesPaginated call passing CancellationToken as memoryType parameter --- src/Memorizer/Services/MarkdownExportService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Memorizer/Services/MarkdownExportService.cs b/src/Memorizer/Services/MarkdownExportService.cs index 4c003a8..c5a3112 100644 --- a/src/Memorizer/Services/MarkdownExportService.cs +++ b/src/Memorizer/Services/MarkdownExportService.cs @@ -188,7 +188,7 @@ public async Task ExportAllAsync( while (!ct.IsCancellationRequested) { - var (memories, totalCount) = await GetStorage().GetMemoriesPaginated(page, pageSize, ct); + var (memories, totalCount) = await GetStorage().GetMemoriesPaginated(page, pageSize, cancellationToken: ct); if (memories.Count == 0) break; foreach (var m in memories)