From 53953dc960087311c053c79769fb69a43eae69e9 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Wed, 17 Sep 2025 12:56:25 +1200 Subject: [PATCH 1/5] SF-3567 Add support for replacing rich-text docs in the Realtime Server --- src/RealtimeServer/common/index.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/RealtimeServer/common/index.ts b/src/RealtimeServer/common/index.ts index d1dc9f6b05d..0960de95d48 100644 --- a/src/RealtimeServer/common/index.ts +++ b/src/RealtimeServer/common/index.ts @@ -342,13 +342,24 @@ export = { } // Build the ops from a diff - // NOTE: We do not use diff-patch-match, as that may result in - // op conflicts when ops are submitted from multiple sources. - // diff-patch-match mutates the string, but we want to replace it. - const ops = json0OtDiff(doc.data, data); + let ops: any; + let hasOps: boolean; + if (doc.type?.name == OTJson0.type.name) { + // NOTE: We do not use diff-patch-match, as that may result in + // op conflicts when ops are submitted from multiple sources. + // diff-patch-match mutates the string, but we want to replace it. + ops = json0OtDiff(doc.data, data); + hasOps = ops.length > 0; + } else if (doc.type?.name == RichText.type.name) { + ops = new RichText.Delta(doc.data.ops).diff(new RichText.Delta(data.ops)); + hasOps = ops.ops.length > 0; + } else { + callback(new Error('Unsupported document type.')); + return; + } // Submit the ops - if (ops.length > 0) { + if (hasOps) { const options: any = {}; doc.submitSource = source != null; if (source != null) { From 109290dea0e3cdb3d290f908619ad2a1c95a4421 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Tue, 16 Sep 2025 16:07:28 +1200 Subject: [PATCH 2/5] SF-3567 Add new books to a project from a draft --- .../Controllers/SFProjectsRpcController.cs | 39 ++ .../Models/DraftApplyResult.cs | 24 + .../Models/DraftApplyState.cs | 8 + .../Services/DeltaUsxMapper.cs | 12 +- .../Services/IMachineApiService.cs | 11 + .../Services/INotifier.cs | 1 + .../Services/ISFProjectService.cs | 1 + .../Services/MachineApiService.cs | 497 ++++++++++++++- .../Services/NotificationHub.cs | 9 + .../Services/NotificationHubExtensions.cs | 6 + .../Services/ParatextSyncRunner.cs | 2 +- .../Services/SFProjectService.cs | 3 +- .../SFProjectsRpcControllerTests.cs | 24 + .../Services/DeltaUsxMapperTests.cs | 14 +- .../Services/MachineApiServiceTests.cs | 564 +++++++++++++++++- .../Services/ParatextSyncRunnerTests.cs | 3 + 16 files changed, 1193 insertions(+), 25 deletions(-) create mode 100644 src/SIL.XForge.Scripture/Models/DraftApplyResult.cs create mode 100644 src/SIL.XForge.Scripture/Models/DraftApplyState.cs diff --git a/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs b/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs index 422b3666ec0..950d669896f 100644 --- a/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs +++ b/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using EdjCase.JsonRpc.Router.Abstractions; @@ -34,6 +35,44 @@ IUserAccessor userAccessor // Keep a reference in this class to prevent duplicate allocation (Warning CS9107) private readonly IExceptionHandler _exceptionHandler = exceptionHandler; + public IRpcMethodResult ApplyPreTranslationToProject( + string projectId, + string scriptureRange, + string targetProjectId, + DateTime timestamp + ) + { + try + { + // Run the background job + backgroundJobClient.Enqueue(r => + r.ApplyPreTranslationToProjectAsync( + UserId, + projectId, + scriptureRange, + targetProjectId, + timestamp, + CancellationToken.None + ) + ); + return Ok(); + } + catch (Exception) + { + _exceptionHandler.RecordEndpointInfoForException( + new Dictionary + { + { "method", "ApplyPreTranslationToProject" }, + { "projectId", projectId }, + { "scriptureRange", scriptureRange }, + { "targetProjectId", targetProjectId }, + { "timestamp", timestamp.ToString(CultureInfo.InvariantCulture) }, + } + ); + throw; + } + } + public async Task Create(SFProjectCreateSettings settings) { try diff --git a/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs b/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs new file mode 100644 index 00000000000..d8e0973545b --- /dev/null +++ b/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +namespace SIL.XForge.Scripture.Models; + +/// +/// The result of applying a draft. +/// +public class DraftApplyResult +{ + /// + /// Whether changes were saved to the database. + /// + public bool ChangesSaved { get; set; } + + /// + /// A list of any chapters that failed to apply in the format "GEN 1". + /// + public List Failures = []; + + /// + /// A log containing any warnings or errors that occurred while applying the draft. + /// + public string Log { get; set; } = string.Empty; +} diff --git a/src/SIL.XForge.Scripture/Models/DraftApplyState.cs b/src/SIL.XForge.Scripture/Models/DraftApplyState.cs new file mode 100644 index 00000000000..46fb274c649 --- /dev/null +++ b/src/SIL.XForge.Scripture/Models/DraftApplyState.cs @@ -0,0 +1,8 @@ +namespace SIL.XForge.Scripture.Models; + +public class DraftApplyState +{ + public bool Failed { get; set; } + public string? State { get; set; } + public bool Success { get; set; } +} diff --git a/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs b/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs index 310ea4fd1dd..94d224ed234 100644 --- a/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs +++ b/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using System.Xml.Linq; using System.Xml.Schema; using Microsoft.Extensions.Logging; @@ -11,7 +12,7 @@ namespace SIL.XForge.Scripture.Services; -public class DeltaUsxMapper( +public partial class DeltaUsxMapper( IGuidService guidService, ILogger logger, IExceptionHandler exceptionHandler @@ -171,6 +172,15 @@ private static bool CanParaContainVerseText(string? style) return ParagraphPoetryListStyles.Contains(style); } + [GeneratedRegex(@"\\id\s+(\w+)", RegexOptions.Compiled)] + private static partial Regex BookIdRegex(); + + public static string ExtractBookId(string usfm) + { + string firstLine = usfm.Split('\n').FirstOrDefault()?.Trim() ?? string.Empty; + return BookIdRegex().Match(firstLine).Groups[1].Value; + } + /// /// Create list of ChapterDelta objects from USX. /// diff --git a/src/SIL.XForge.Scripture/Services/IMachineApiService.cs b/src/SIL.XForge.Scripture/Services/IMachineApiService.cs index 951c4e2eda9..ff3372a6183 100644 --- a/src/SIL.XForge.Scripture/Services/IMachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/IMachineApiService.cs @@ -14,6 +14,17 @@ namespace SIL.XForge.Scripture.Services; [Intercept(typeof(EventMetricLogger))] public interface IMachineApiService { + [Mutex] + [LogEventMetric(EventScope.Drafting, nameof(curUserId), nameof(sfProjectId), captureReturnValue: true)] + Task ApplyPreTranslationToProjectAsync( + string curUserId, + string sfProjectId, + string scriptureRange, + string targetProjectId, + DateTime timestamp, + CancellationToken cancellationToken + ); + [LogEventMetric(EventScope.Drafting, nameof(sfProjectId))] Task BuildCompletedAsync(string sfProjectId, string buildId, string buildState, Uri websiteUrl); diff --git a/src/SIL.XForge.Scripture/Services/INotifier.cs b/src/SIL.XForge.Scripture/Services/INotifier.cs index e3b45da2b89..7b421df877a 100644 --- a/src/SIL.XForge.Scripture/Services/INotifier.cs +++ b/src/SIL.XForge.Scripture/Services/INotifier.cs @@ -6,6 +6,7 @@ namespace SIL.XForge.Scripture.Services; public interface INotifier { Task NotifyBuildProgress(string sfProjectId, ServalBuildState buildState); + Task NotifyDraftApplyProgress(string sfProjectId, DraftApplyState draftApplyState); Task NotifySyncProgress(string sfProjectId, ProgressState progressState); Task SubscribeToProject(string projectId); } diff --git a/src/SIL.XForge.Scripture/Services/ISFProjectService.cs b/src/SIL.XForge.Scripture/Services/ISFProjectService.cs index 780d42d6c0e..4c12526cf2d 100644 --- a/src/SIL.XForge.Scripture/Services/ISFProjectService.cs +++ b/src/SIL.XForge.Scripture/Services/ISFProjectService.cs @@ -64,6 +64,7 @@ Task UpdatePermissionsAsync( string curUserId, IDocument projectDoc, IReadOnlyList? users = null, + IReadOnlyList? books = null, CancellationToken token = default ); Task EnsureWritingSystemTagIsSetAsync(string curUserId, string projectId); diff --git a/src/SIL.XForge.Scripture/Services/MachineApiService.cs b/src/SIL.XForge.Scripture/Services/MachineApiService.cs index 6582c991080..43bb5457241 100644 --- a/src/SIL.XForge.Scripture/Services/MachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineApiService.cs @@ -23,11 +23,15 @@ using SIL.XForge.Models; using SIL.XForge.Realtime; using SIL.XForge.Realtime.Json0; +using SIL.XForge.Realtime.RichText; using SIL.XForge.Scripture.Models; using SIL.XForge.Services; using SIL.XForge.Utils; +using Chapter = SIL.XForge.Scripture.Models.Chapter; using TextInfo = SIL.XForge.Scripture.Models.TextInfo; +#pragma warning disable CA2254 + namespace SIL.XForge.Scripture.Services; /// @@ -79,15 +83,502 @@ IRepository userSecrets /// internal const string BuildStateFinishing = "FINISHING"; - private static readonly IEqualityComparer> _listIntComparer = SequenceEqualityComparer.Create( - EqualityComparer.Default - ); private static readonly IEqualityComparer> _listStringComparer = SequenceEqualityComparer.Create( EqualityComparer.Default ); private static readonly IEqualityComparer> _listProjectScriptureRangeComparer = SequenceEqualityComparer.Create(EqualityComparer.Default); + public async Task ApplyPreTranslationToProjectAsync( + string curUserId, + string sfProjectId, + string scriptureRange, + string targetProjectId, + DateTime timestamp, + CancellationToken cancellationToken + ) + { + // Ensure that the user has permission to access the draft project + SFProject project = await EnsureProjectPermissionAsync( + curUserId, + sfProjectId, + isServalAdmin: false, + cancellationToken + ); + + // Connect to the realtime server + await using IConnection connection = await realtimeService.ConnectAsync(curUserId); + + // Retrieve the chapter deltas + var result = new DraftApplyResult(); + IDocument targetProjectDoc; + List createdBooks = []; + Dictionary> createdChapters = []; + List<(ChapterDelta chapterDelta, int bookNum)> chapterDeltas = []; + try + { + // Retrieve the user secret + Attempt attempt = await userSecrets.TryGetAsync(curUserId, cancellationToken); + if (!attempt.TryResult(out UserSecret userSecret)) + { + throw new DataNotFoundException("The user does not exist."); + } + + // Load the target project + targetProjectDoc = await connection.FetchAsync(targetProjectId); + if (!targetProjectDoc.IsLoaded) + { + throw new DataNotFoundException("The project does not exist"); + } + + // Get the draft project versification + ScrVers versification = + paratextService.GetParatextSettings(userSecret, project.ParatextId)?.Versification + ?? VerseRef.defaultVersification; + + // Get the target project versification + ScrVers targetVersification = + paratextService.GetParatextSettings(userSecret, targetProjectDoc.Data.ParatextId)?.Versification + ?? VerseRef.defaultVersification; + + // Parse the scripture range + ScriptureRangeParser scriptureRangeParser = new ScriptureRangeParser(versification); + Dictionary> booksAndChapters = scriptureRangeParser.GetChapters(scriptureRange); + + // Get the drafts for the scripture range + foreach ((string book, List bookChapters) in booksAndChapters) + { + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState { State = $"Retrieving draft for {Canon.BookIdToEnglishName(book)}." } + ); + int bookNum = Canon.BookIdToNumber(book); + + // Warn if the last chapter is different (this will affect chapter creation + int lastChapter = versification.GetLastChapter(bookNum); + int targetLastChapter = targetVersification.GetLastChapter(bookNum); + if (lastChapter != targetLastChapter) + { + string message = + $"The draft project ({project.ShortName.Sanitize()}) versification for {book} has {lastChapter} chapters," + + $" while the target project ({targetProjectDoc.Data.ShortName.Sanitize()}) has {targetLastChapter} chapters."; + logger.LogWarning(message); + result.Log += $"{message}\n"; + await hubContext.NotifyDraftApplyProgress(sfProjectId, new DraftApplyState { State = message }); + } + + // Ensure that if chapters is blank, it contains every chapter in the book + List chapters = bookChapters; + if (chapters.Count == 0) + { + chapters = [.. Enumerable.Range(1, lastChapter)]; + } + + // Store the USJ for each chapter, so if we download form Serval we only do it once per book + List chapterUsj = []; + foreach (int chapterNum in chapters.Where(c => c > 0)) + { + // See if we have a draft locally + string id = TextDocument.GetDocId(sfProjectId, bookNum, chapterNum, TextDocument.Draft); + IDocument textDocument = await connection.FetchAsync(id); + IUsj usj; + if (textDocument.IsLoaded) + { + // Retrieve the snapshot if it exists, or use the latest available if none + Snapshot snapshot = await connection.FetchSnapshotAsync( + id, + timestamp + ); + usj = snapshot.Data ?? textDocument.Data; + } + else + { + // We do not have a draft locally, so we should retrieve it from Serval, and save it locally + if (chapterUsj.Count < chapterNum) + { + DraftUsfmConfig config = + project.TranslateConfig.DraftConfig.UsfmConfig ?? new DraftUsfmConfig(); + string usfm = await preTranslationService.GetPreTranslationUsfmAsync( + sfProjectId, + bookNum, + chapterNum: 0, + config, + cancellationToken + ); + + // If the usfm is invalid, skip this book + if (string.IsNullOrWhiteSpace(usfm)) + { + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = $"No book number for {Canon.BookNumberToEnglishName(bookNum)}.", + } + ); + break; + } + + // If the book id is invalid, skip this book + if (DeltaUsxMapper.ExtractBookId(usfm) != book) + { + result.Failures.Add(book); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = + $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)}.", + } + ); + + break; + } + + // Get the USFM as a list of USJ chapters + chapterUsj = + [ + .. paratextService.GetChaptersAsUsj(userSecret, project.ParatextId, bookNum, usfm), + ]; + + // If the chapter is still not present, go to the next book + if (chapterUsj.Count < chapterNum) + { + // Don't report an error here, as sometimes the versification will report more chapters than the USFM has + break; + } + } + + // Get the chapter USJ + usj = chapterUsj[chapterNum - 1]; + + // If the chapter is invalid, skip it + if (usj.Content.Count == 0) + { + // A blank chapter from Serval + result.Failures.Add($"{Canon.BookNumberToId(bookNum)} {chapterNum}"); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = + $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterNum}.", + } + ); + continue; + } + + // Save the chapter to the realtime server + await SaveTextDocumentAsync(textDocument, usj); + } + + // If the chapter is invalid, skip it + if (usj.Content.Count == 0) + { + // Likely a blank draft in the database + result.Failures.Add($"{Canon.BookNumberToId(bookNum)} {chapterNum}"); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = + $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterNum}.", + } + ); + continue; + } + + // Then convert it to USX + XDocument usxDoc = UsjToUsx.UsjToUsxXDocument(usj); + + // Then convert it to a Delta + IEnumerable deltas = deltaUsxMapper.ToChapterDeltas(usxDoc); + + // Ensure that the chapter was present in the USFM + ChapterDelta chapterDelta = deltas.FirstOrDefault(); + if (chapterDelta is not null) + { + chapterDeltas.Add((chapterDelta, bookNum)); + } + } + } + } + catch (Exception e) + { + // Log the error, report to bugsnag, and report to the user via SignalR + string message = + $"Apply pre-translation draft exception occurred for project {sfProjectId.Sanitize()} running in background job."; + logger.LogError(e, message); + exceptionHandler.ReportException(e); + result.Log += $"{message}\n"; + result.Log += $"{e}\n"; + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState { Failed = true, State = result.Log } + ); + + // Do not proceed to save the draft to the project + return result; + } + + bool successful = false; + try + { + // Being the transaction + connection.BeginTransaction(); + + // Begin a transaction, and update the project + foreach ((ChapterDelta chapterDelta, int bookNum) in chapterDeltas) + { + // Create the new chapter record + Chapter chapter = new Chapter + { + DraftApplied = true, + IsValid = chapterDelta.IsValid, + Number = chapterDelta.Number, + LastVerse = chapterDelta.LastVerse, + }; + + // Create or update the relevant book and chapter records in the project + int textIndex = targetProjectDoc.Data.Texts.FindIndex(t => t.BookNum == bookNum); + if (textIndex == -1) + { + // Create the new book record with the chapter + TextInfo text = new TextInfo { BookNum = bookNum, Chapters = [chapter] }; + await targetProjectDoc.SubmitJson0OpAsync(op => op.Add(pd => pd.Texts, text)); + + // Record that the book and chapter were created + createdBooks.Add(bookNum); + createdChapters.Add(bookNum, [chapterDelta.Number]); + } + else + { + int chapterIndex = targetProjectDoc + .Data.Texts[textIndex] + .Chapters.FindIndex(c => c.Number == chapterDelta.Number); + if (chapterIndex == -1) + { + // Create a new chapter record + await targetProjectDoc.SubmitJson0OpAsync(op => + op.Add(pd => pd.Texts[textIndex].Chapters, chapter) + ); + + // Record that the chapter was created + if (createdChapters.TryGetValue(bookNum, out List chapters)) + { + chapters.Add(chapterDelta.Number); + } + else + { + createdChapters.Add(bookNum, [chapterDelta.Number]); + } + } + else + { + // Update the existing chapter record + await targetProjectDoc.SubmitJson0OpAsync(op => + { + op.Set(pd => pd.Texts[textIndex].Chapters[chapterIndex].DraftApplied, chapter.DraftApplied); + op.Set(pd => pd.Texts[textIndex].Chapters[chapterIndex].IsValid, chapter.IsValid); + op.Set(pd => pd.Texts[textIndex].Chapters[chapterIndex].LastVerse, chapter.LastVerse); + }); + } + } + } + + // Update the permissions + if (chapterDeltas.Count > 0) + { + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState { State = "Loading permissions from Paratext." } + ); + await projectService.UpdatePermissionsAsync( + curUserId, + targetProjectDoc, + users: null, + books: chapterDeltas.Select(c => c.bookNum).Distinct().ToList(), + cancellationToken + ); + } + + // Create the text data documents, using the permissions matrix calculated above for permissions + foreach ((ChapterDelta chapterDelta, int bookNum) in chapterDeltas) + { + // Ensure that the user has permission to write the book + int textIndex = targetProjectDoc.Data.Texts.FindIndex(t => t.BookNum == bookNum); + if (textIndex == -1) + { + string bookId = Canon.BookNumberToId(bookNum); + if (result.Failures.LastOrDefault() != bookId) + { + // Only list the book failure once + result.Failures.Add(bookId); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)}.", + } + ); + } + + continue; + } + + bool canWriteBook = + targetProjectDoc.Data.Texts[textIndex].Permissions.TryGetValue(curUserId, out string bookPermission) + && bookPermission == TextInfoPermission.Write; + if (!canWriteBook) + { + // Remove the book from the project if we created it, and proceed to add the next chapter + if (createdBooks.Contains(bookNum)) + { + await targetProjectDoc.SubmitJson0OpAsync(op => op.Remove(pd => pd.Texts, textIndex)); + } + + string bookId = Canon.BookNumberToId(bookNum); + if (result.Failures.LastOrDefault() != bookId) + { + // Only list the book failure once + result.Failures.Add(bookId); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)}.", + } + ); + } + + continue; + } + + // Ensure that the user has permission to write the chapter + int chapterIndex = targetProjectDoc + .Data.Texts[textIndex] + .Chapters.FindIndex(c => c.Number == chapterDelta.Number); + if (chapterIndex == -1) + { + result.Failures.Add($"{Canon.BookNumberToId(bookNum)} {chapterDelta.Number}"); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = + $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + } + ); + continue; + } + + bool canWriteChapter = + targetProjectDoc + .Data.Texts[textIndex] + .Chapters[chapterIndex] + .Permissions.TryGetValue(curUserId, out string chapterPermission) + && chapterPermission == TextInfoPermission.Write; + if (!canWriteChapter) + { + // Remove the chapter from the project if we created it, and proceed to add the next chapter + if ( + createdChapters.TryGetValue(bookNum, out List chapters) + && chapters.Contains(chapterDelta.Number) + ) + { + await targetProjectDoc.SubmitJson0OpAsync(op => + op.Remove(pd => pd.Texts[textIndex].Chapters, chapterIndex) + ); + } + + result.Failures.Add($"{Canon.BookNumberToId(bookNum)} {chapterDelta.Number}"); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = + $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + } + ); + continue; + } + + // Create or update the chapter's text document + string id = TextData.GetTextDocId(targetProjectDoc.Id, bookNum, chapterDelta.Number); + TextData newTextData = new TextData(chapterDelta.Delta); + IDocument textDataDoc = connection.Get(id); + await textDataDoc.FetchAsync(); + if (textDataDoc.IsLoaded) + { + // Update the existing text data document + Delta diffDelta = textDataDoc.Data.Diff(newTextData); + if (diffDelta.Ops.Count > 0) + { + await textDataDoc.SubmitOpAsync(diffDelta, OpSource.Draft); + } + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = $"Updating {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + } + ); + } + else + { + // Create a new text data document + await textDataDoc.CreateAsync(newTextData); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState + { + State = $"Creating {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + } + ); + } + + // A draft has been applied + successful = true; + } + } + catch (Exception e) + { + // Log the error and report to bugsnag + string message = + $"Apply pre-translation draft exception occurred for project {sfProjectId.Sanitize()} running in background job."; + logger.LogError(e, message); + exceptionHandler.ReportException(e); + result.Log += $"{message}\n"; + result.Log += $"{e}\n"; + + // Do not commit the transaction + successful = false; + } + finally + { + if (successful) + { + await connection.CommitTransactionAsync(); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState { Success = true, State = result.Log } + ); + } + else + { + connection.RollbackTransaction(); + await hubContext.NotifyDraftApplyProgress( + sfProjectId, + new DraftApplyState { Failed = true, State = result.Log } + ); + } + + result.ChangesSaved = successful; + } + + return result; + } + public async Task BuildCompletedAsync(string sfProjectId, string buildId, string buildState, Uri websiteUrl) { try diff --git a/src/SIL.XForge.Scripture/Services/NotificationHub.cs b/src/SIL.XForge.Scripture/Services/NotificationHub.cs index 8f17d103ab4..462dd51aeb5 100644 --- a/src/SIL.XForge.Scripture/Services/NotificationHub.cs +++ b/src/SIL.XForge.Scripture/Services/NotificationHub.cs @@ -21,6 +21,15 @@ public class NotificationHub : Hub, INotifier public async Task NotifyBuildProgress(string projectId, ServalBuildState buildState) => await Clients.Group(projectId).NotifyBuildProgress(projectId, buildState); + /// + /// Notifies subscribers to a project of draft application progress. + /// + /// The Scripture Forge project identifier. + /// The state of the draft being applied. + /// The asynchronous task. + public async Task NotifyDraftApplyProgress(string projectId, DraftApplyState draftApplyState) => + await Clients.Group(projectId).NotifyDraftApplyProgress(projectId, draftApplyState); + /// /// Notifies subscribers to a project of sync progress. /// diff --git a/src/SIL.XForge.Scripture/Services/NotificationHubExtensions.cs b/src/SIL.XForge.Scripture/Services/NotificationHubExtensions.cs index 7568cbb3db9..8d4747e7820 100644 --- a/src/SIL.XForge.Scripture/Services/NotificationHubExtensions.cs +++ b/src/SIL.XForge.Scripture/Services/NotificationHubExtensions.cs @@ -12,6 +12,12 @@ public static Task NotifyBuildProgress( ServalBuildState buildState ) => hubContext.Clients.Groups(projectId).NotifyBuildProgress(projectId, buildState); + public static Task NotifyDraftApplyProgress( + this IHubContext hubContext, + string projectId, + DraftApplyState draftApplyState + ) => hubContext.Clients.Groups(projectId).NotifyDraftApplyProgress(projectId, draftApplyState); + public static Task NotifySyncProgress( this IHubContext hubContext, string projectId, diff --git a/src/SIL.XForge.Scripture/Services/ParatextSyncRunner.cs b/src/SIL.XForge.Scripture/Services/ParatextSyncRunner.cs index bd9da6da018..c7f39a45e64 100644 --- a/src/SIL.XForge.Scripture/Services/ParatextSyncRunner.cs +++ b/src/SIL.XForge.Scripture/Services/ParatextSyncRunner.cs @@ -447,7 +447,7 @@ await UpdateBiblicalTermsAsync( if (!_paratextService.IsResource(targetParatextId) || resourceNeedsUpdating) { LogMetric("Updating permissions"); - await _projectService.UpdatePermissionsAsync(userId, _projectDoc, _paratextUsers, token); + await _projectService.UpdatePermissionsAsync(userId, _projectDoc, users: _paratextUsers, token: token); } await NotifySyncProgress(SyncPhase.Phase9, 40.0); diff --git a/src/SIL.XForge.Scripture/Services/SFProjectService.cs b/src/SIL.XForge.Scripture/Services/SFProjectService.cs index a91aa28f9cf..0ad8a44c86a 100644 --- a/src/SIL.XForge.Scripture/Services/SFProjectService.cs +++ b/src/SIL.XForge.Scripture/Services/SFProjectService.cs @@ -1400,6 +1400,7 @@ public async Task UpdatePermissionsAsync( string curUserId, IDocument projectDoc, IReadOnlyList? users = null, + IReadOnlyList? books = null, CancellationToken token = default ) { @@ -1410,7 +1411,7 @@ public async Task UpdatePermissionsAsync( } string paratextId = projectDoc.Data.ParatextId; - HashSet booksInProject = [.. _paratextService.GetBookList(userSecret, paratextId)]; + HashSet booksInProject = [.. books ?? _paratextService.GetBookList(userSecret, paratextId)]; users ??= await _paratextService.GetParatextUsersAsync(userSecret, projectDoc.Data, token); IReadOnlyDictionary ptUsernameMapping = users .Where(u => !string.IsNullOrWhiteSpace(u.Id) && !string.IsNullOrWhiteSpace(u.Username)) diff --git a/test/SIL.XForge.Scripture.Tests/Controllers/SFProjectsRpcControllerTests.cs b/test/SIL.XForge.Scripture.Tests/Controllers/SFProjectsRpcControllerTests.cs index f9f61d468b1..58c850556f5 100644 --- a/test/SIL.XForge.Scripture.Tests/Controllers/SFProjectsRpcControllerTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Controllers/SFProjectsRpcControllerTests.cs @@ -39,6 +39,30 @@ public class SFProjectsRpcControllerTests const string Locale = "en"; private static readonly Uri WebsiteUrl = new Uri("https://scriptureforge.org", UriKind.Absolute); + [Test] + public void ApplyPreTranslationToProject_Success() + { + var env = new TestEnvironment(); + + // SUT + var result = env.Controller.ApplyPreTranslationToProject(Project01, "GEN-EXO", Project01, DateTime.UtcNow); + Assert.IsInstanceOf(result); + env.BackgroundJobClient.Received().Create(Arg.Any(), Arg.Any()); + } + + [Test] + public void ApplyPreTranslationToProject_UnknownError() + { + var env = new TestEnvironment(); + env.BackgroundJobClient.Create(Arg.Any(), Arg.Any()).Throws(new ArgumentNullException()); + + // SUT + Assert.Throws(() => + env.Controller.ApplyPreTranslationToProject(Project01, "GEN-EXO", Project01, DateTime.UtcNow) + ); + env.ExceptionHandler.Received().RecordEndpointInfoForException(Arg.Any>()); + } + [Test] public async Task Delete_Success() { diff --git a/test/SIL.XForge.Scripture.Tests/Services/DeltaUsxMapperTests.cs b/test/SIL.XForge.Scripture.Tests/Services/DeltaUsxMapperTests.cs index 67502f4a031..54e4c492dae 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/DeltaUsxMapperTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/DeltaUsxMapperTests.cs @@ -19,7 +19,7 @@ namespace SIL.XForge.Scripture.Services; [TestFixture] -public partial class DeltaUsxMapperTests +public class DeltaUsxMapperTests { private TestGuidService _mapperGuidService; private TestGuidService _testGuidService; @@ -4091,19 +4091,9 @@ private async Task RoundTripTestHelper(string projectZipFilename, string project private void AssertRoundtrips(string bookUsfm) => Assert.That(DoesRoundtrip(bookUsfm, out string errorMessage), Is.True, errorMessage); - [GeneratedRegex(@"\\id\s+(\w+)", RegexOptions.Compiled)] - private static partial Regex BookCodeRegex(); - - private static string ExtractBookCode(string bookUsfm) - { - string firstLine = bookUsfm.Split('\n').FirstOrDefault()?.Trim() ?? string.Empty; - string bookCode = BookCodeRegex().Match(firstLine).Groups[1].Value; - return bookCode; - } - private bool DoesRoundtrip(string bookUsfm, out string errorMessage) { - string bookCode = ExtractBookCode(bookUsfm); + string bookCode = DeltaUsxMapper.ExtractBookId(bookUsfm); XmlDocument bookUsxLoading = UsfmToUsx.ConvertToXmlDocument( _scrText, diff --git a/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs b/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs index c9eaa944538..66fc26b641c 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; @@ -48,6 +49,7 @@ public class MachineApiServiceTests private const string User01 = "user01"; private const string User02 = "user02"; private const string Paratext01 = "paratext01"; + private const string Paratext02 = "paratext02"; private const string ParatextUserId01 = "paratextUser01"; private const string Segment = "segment"; private const string TargetSegment = "targetSegment"; @@ -57,7 +59,7 @@ public class MachineApiServiceTests private const string JsonPayload = """{"event":"TranslationBuildFinished","payload":{"build":{"id":"65f0c455682bb17bc4066917","url":"/api/v1/translation/engines/translationEngine01/builds/65f0c455682bb17bc4066917"},"engine":{"id":"translationEngine01","url":"/api/v1/translation/engines/translationEngine01"},"buildState":"Completed","dateFinished":"2024-03-12T21:14:10.789Z"}}"""; - private const string TestUsfm = "\\c 1 \\v1 Verse 1"; + private const string TestUsfm = "\\c 1 \\v 1 Verse 1"; private const string TestUsx = "" + "Verse 1"; @@ -107,6 +109,334 @@ public class MachineApiServiceTests DateFinished = DateTimeOffset.UtcNow, }; + [Test] + public async Task ApplyPreTranslationToProjectAsync_BlankUsjFromMongo() + { + // Set up test environment + var env = new TestEnvironment(); + TextDocument textDocument = new TextDocument + { + Id = TextDocument.GetDocId(Project01, 31, 1, TextDocument.Draft), + Type = Usj.UsjType, + Version = Usj.UsjVersion, + Content = [], + }; + env.TextDocuments.Add(textDocument); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "OBA", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + Assert.That(actual.Log, Is.Empty); + Assert.That(actual.Failures, Is.Not.Empty); + Assert.That(actual.Failures.First(), Is.EqualTo("OBA 1")); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_BlankUsjFromServal() + { + // Set up test environment + var env = new TestEnvironment(); + const string usfm = "\\id OBA\r\n\\c 1\r\n\\v 1 Verse 1"; + env.PreTranslationService.GetPreTranslationUsfmAsync( + Project01, + bookNum: 31, + chapterNum: 0, + Arg.Any(), + CancellationToken.None + ) + .Returns(Task.FromResult(usfm)); + env.ParatextService.GetChaptersAsUsj(Arg.Any(), Paratext01, bookNum: 31, usfm) + .Returns([UsxToUsj.UsxXmlDocumentToUsj(null)]); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "OBA", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + Assert.That(actual.Log, Is.Empty); + Assert.That(actual.Failures, Is.Not.Empty); + Assert.That(actual.Failures.First(), Is.EqualTo("OBA 1")); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_DifferentVersification() + { + // Set up test environment + var env = new TestEnvironment(); + env.ConfigureDraft( + Project01, + bookNum: 106, + numberOfChapters: 2, + bookExists: true, + draftExists: true, + canWriteBook: true, + writeChapters: 1 + ); + + // 6 Ezra has max 12 chapters (either 1-2 or 11-12) in Vulgate and 1 chapter in English + env.ParatextService.GetParatextSettings(Arg.Any(), Paratext01) + .Returns(new ParatextSettings { Versification = ScrVers.Vulgate }); + env.ParatextService.GetParatextSettings(Arg.Any(), Paratext02) + .Returns(new ParatextSettings { Versification = ScrVers.English }); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "6EZ", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + await env.VerifyDraftAsync( + actual, + Project02, + numberOfChapters: 2, + bookExists: true, + canWriteBook: true, + writeChapters: 1 + ); + env.MockLogger.AssertHasEvent(logEvent => logEvent.LogLevel == LogLevel.Warning); + Assert.That(actual.Log, Is.Not.Empty); + Assert.That(actual.Failures, Is.Not.Empty); + Assert.That(actual.Failures.First(), Is.EqualTo("6EZ 2")); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_ExceptionFromParatext() + { + // Set up test environment + var env = new TestEnvironment(); + env.ConfigureDraft( + Project01, + bookNum: 39, + numberOfChapters: 3, + bookExists: true, + draftExists: true, + canWriteBook: true, + writeChapters: 3 + ); + env.ProjectService.UpdatePermissionsAsync( + Arg.Any(), + Arg.Any>(), + users: null, + books: Arg.Any>(), + CancellationToken.None + ) + .ThrowsAsync(new NotSupportedException()); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "MAL", + targetProjectId: Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + env.MockLogger.AssertHasEvent(logEvent => logEvent.Exception?.GetType() == typeof(NotSupportedException)); + env.ExceptionHandler.Received().ReportException(Arg.Any()); + Assert.That(actual.Log, Is.Not.Empty); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_InvalidBookId() + { + // Set up test environment + var env = new TestEnvironment(); + const string usfm = "\\id EXO Should Be GEN\r\n\\c 1\r\n\\v 1 Verse 1"; + env.PreTranslationService.GetPreTranslationUsfmAsync( + Project01, + 1, + chapterNum: 0, + Arg.Any(), + CancellationToken.None + ) + .Returns(Task.FromResult(usfm)); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + Assert.That(actual.Log, Is.Empty); + Assert.That(actual.Failures, Is.Not.Empty); + Assert.That(actual.Failures.First(), Is.EqualTo("GEN")); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_RequestEarlierThanLocalDraft() + { + // Set up test environment + var env = new TestEnvironment(); + env.ConfigureDraft( + Project01, + bookNum: 1, + numberOfChapters: 50, + bookExists: true, + draftExists: true, + canWriteBook: true, + writeChapters: 50 + ); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + Project02, + DateTime.MinValue, + CancellationToken.None + ); + + await env.VerifyDraftAsync( + actual, + Project02, + numberOfChapters: 50, + bookExists: true, + canWriteBook: true, + writeChapters: 50 + ); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_MissingTargetProject() + { + // Set up test environment + var env = new TestEnvironment(); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + targetProjectId: "invalid_project_id", + DateTime.UtcNow, + CancellationToken.None + ); + + env.MockLogger.AssertHasEvent(logEvent => logEvent.Exception?.GetType() == typeof(DataNotFoundException)); + env.ExceptionHandler.Received() + .ReportException(Arg.Is(e => e.Message.Contains("project"))); + Assert.That(actual.Log, Is.Not.Empty); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_MissingUserSecret() + { + // Set up test environment + var env = new TestEnvironment(); + await env.UserSecrets.DeleteAllAsync(_ => true); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + env.MockLogger.AssertHasEvent(logEvent => logEvent.Exception?.GetType() == typeof(DataNotFoundException)); + env.ExceptionHandler.Received().ReportException(Arg.Is(e => e.Message.Contains("user"))); + Assert.That(actual.Log, Is.Not.Empty); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_NoChapterDeltas() + { + // Set up test environment + var env = new TestEnvironment(); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + Project02, + DateTime.UtcNow, + CancellationToken.None + ); + + await env + .ProjectService.DidNotReceive() + .UpdatePermissionsAsync( + User01, + Arg.Any>(), + users: null, + books: Arg.Any>(), + CancellationToken.None + ); + Assert.That(actual.ChangesSaved, Is.False); + } + + [Test] + public async Task ApplyPreTranslationToProjectAsync_Success( + [Values(Project01, Project02)] string targetProjectId, + [Values(true, false)] bool bookExists, + [Values(true, false)] bool draftExists, + [Values(true, false)] bool canWriteBook, + [Values(true, false)] bool canWriteChapter + ) + { + // Set up test environment + var env = new TestEnvironment(); + env.ConfigureDraft( + Project01, + bookNum: 1, + numberOfChapters: 50, + bookExists, + draftExists, + canWriteBook, + writeChapters: canWriteChapter ? 50 : 0 + ); + + // SUT + DraftApplyResult actual = await env.Service.ApplyPreTranslationToProjectAsync( + User01, + Project01, + scriptureRange: "GEN", + targetProjectId, + DateTime.UtcNow, + CancellationToken.None + ); + + await env.VerifyDraftAsync( + actual, + targetProjectId, + numberOfChapters: 50, + bookExists, + canWriteBook, + writeChapters: canWriteChapter ? 50 : 0 + ); + } + [Test] public async Task BuildCompletedAsync_EventMetricInvalid() { @@ -1551,8 +1881,6 @@ public async Task GetLastCompletedPreTranslationBuildAsync_RetrievePreTranslatio const double percentCompleted = 0; const int revision = 43; const JobState state = JobState.Completed; - env.ParatextService.GetParatextSettings(Arg.Any(), Arg.Any()) - .Returns(new ParatextSettings { Versification = ScrVers.English }); env.TranslationEnginesClient.GetAllBuildsAsync(TranslationEngine01, CancellationToken.None) .Returns( Task.FromResult>( @@ -1623,8 +1951,6 @@ public async Task GetLastCompletedPreTranslationBuildAsync_NoRetrievePreTranslat const double percentCompleted = 0; const int revision = 43; const JobState state = JobState.Completed; - env.ParatextService.GetParatextSettings(Arg.Any(), Arg.Any()) - .Returns(new ParatextSettings { Versification = ScrVers.English }); env.TranslationEnginesClient.GetAllBuildsAsync(TranslationEngine01, CancellationToken.None) .Returns( Task.FromResult>( @@ -1680,8 +2006,6 @@ public async Task GetLastCompletedPreTranslationBuildAsync_NullScriptureRange_Su const double percentCompleted = 0; const int revision = 43; const JobState state = JobState.Completed; - env.ParatextService.GetParatextSettings(Arg.Any(), Arg.Any()) - .Returns(new ParatextSettings { Versification = ScrVers.English }); env.TranslationEnginesClient.GetAllBuildsAsync(TranslationEngine01, CancellationToken.None) .Returns( Task.FromResult>( @@ -3995,6 +4319,9 @@ public TestEnvironment() MachineProjectService = Substitute.For(); MockLogger = new MockLogger(); ParatextService = Substitute.For(); + ParatextService + .GetParatextSettings(Arg.Any(), Arg.Any()) + .Returns(new ParatextSettings { Versification = ScrVers.English }); PreTranslationService = Substitute.For(); ProjectSecrets = new MemoryRepository( [ @@ -4059,6 +4386,7 @@ public TestEnvironment() new SFProject { Id = Project02, + ParatextId = Paratext02, TranslateConfig = new TranslateConfig { DraftConfig = new DraftConfig @@ -4085,6 +4413,7 @@ public TestEnvironment() ] ); TextDocuments = new MemoryRepository(); + Texts = new MemoryRepository(); ProjectRights = Substitute.For(); ProjectRights .HasRight(Arg.Any(), User01, SFProjectDomain.Drafts, Operation.Create) @@ -4094,6 +4423,7 @@ public TestEnvironment() RealtimeService = new SFMemoryRealtimeService(); RealtimeService.AddRepository("sf_projects", OTType.Json0, Projects); RealtimeService.AddRepository("text_documents", OTType.Json0, TextDocuments); + RealtimeService.AddRepository("texts", OTType.RichText, Texts); ServalOptions = Options.Create(new ServalOptions { WebhookSecret = "this_is_a_secret" }); SyncService = Substitute.For(); SyncService.SyncAsync(Arg.Any()).Returns(Task.FromResult(JobId)); @@ -4155,6 +4485,7 @@ public TestEnvironment() public MemoryRepository Projects { get; } public MemoryRepository ProjectSecrets { get; } public MemoryRepository TextDocuments { get; } + public MemoryRepository Texts { get; } public ISFProjectRights ProjectRights { get; } public ISFProjectService ProjectService { get; } public SFMemoryRealtimeService RealtimeService { get; } @@ -4165,6 +4496,225 @@ public TestEnvironment() public ITranslationEngineTypesClient TranslationEngineTypesClient { get; } public MemoryRepository UserSecrets { get; } + public void ConfigureDraft( + string projectId, + int bookNum, + int numberOfChapters, + bool bookExists, + bool draftExists, + bool canWriteBook, + int writeChapters + ) + { + List chapterDeltas = []; + StringBuilder sb = new StringBuilder(); + sb.Append($"\\id {Canon.BookNumberToId(bookNum)}\r\n"); + List usx = []; + for (int chapterNum = 1; chapterNum <= numberOfChapters; chapterNum++) + { + // Build the USFM and USX for the chapter + sb.Append($"\\c {chapterNum}\r\n\\p\r\n\\v 1 First verse\r\n\\v 2 Second verse, same as the first\r\n"); + usx.Add( + $"" + + $"" + + "First verse" + + "Second verse, same as the first" + + "" + ); + + // Return chapter deltas for that USFM + JToken draftToken1 = JToken.Parse( + $"{{\"insert\": {{ \"chapter\": {{ \"number\": \"{chapterNum}\", \"style\": \"c\" }} }} }}" + ); + JToken draftToken2 = JToken.Parse( + "{\"insert\": { \"verse\": { \"number\": \"1\", \"style\": \"v\" } } }" + ); + JToken draftToken3 = JToken.Parse( + "{\"insert\": \"New verse 1 text\", \"attributes\": { \"segment\": \"verse_1_1\" } }" + ); + JToken draftToken4 = JToken.Parse( + "{\"insert\": { \"verse\": { \"number\": \"2\", \"style\": \"v\" } } }" + ); + JToken draftToken5 = JToken.Parse( + "{\"insert\": \"New verse 2 text\"," + "\"attributes\": { \"segment\": \"verse_1_2\" } }" + ); + var chapterDelta = new ChapterDelta( + chapterNum, + 2, + true, + new Delta([draftToken1, draftToken2, draftToken3, draftToken4, draftToken5]) + ); + chapterDeltas.Add(chapterDelta); + + // Create the book in the realtime server if required + if (bookExists) + { + JToken textToken1 = JToken.Parse( + $"{{\"insert\": {{ \"chapter\": {{ \"number\": \"{chapterNum}\", \"style\": \"c\" }} }} }}" + ); + JToken textToken2 = JToken.Parse( + "{\"insert\": { \"verse\": { \"number\": \"1\", \"style\": \"v\" } } }" + ); + JToken textToken3 = JToken.Parse( + "{\"insert\": { \"blank\": true }, \"attributes\": { \"segment\": \"verse_1_1\" } }" + ); + JToken textToken4 = JToken.Parse( + "{\"insert\": { \"verse\": { \"number\": \"2\", \"style\": \"v\" } } }" + ); + JToken textToken5 = JToken.Parse( + "{\"insert\": \"Old verse 2 text\"," + "\"attributes\": { \"segment\": \"verse_1_2\" } }" + ); + + TextData textData = new TextData( + new Delta([textToken1, textToken2, textToken3, textToken4, textToken5]) + ) + { + Id = TextData.GetTextDocId(projectId, bookNum, chapterNum), + }; + Texts.Add(textData); + } + + // Create a local copy of the draft in the realtime server if required + if (draftExists) + { + TextDocument textDocument = new TextDocument + { + Id = TextDocument.GetDocId(projectId, bookNum, chapterNum, TextDocument.Draft), + Type = Usj.UsjType, + Version = Usj.UsjVersion, + Content = + [ + new UsjMarker + { + Type = "book", + Marker = "id", + Code = Canon.BookNumberToId(bookNum), + }, + new UsjMarker + { + Type = "chapter", + Marker = "c", + Number = chapterNum.ToString(), + }, + new UsjMarker + { + Type = "verse", + Marker = "v", + Number = "1", + }, + "Previous verse 1 draft", + ], + }; + TextDocuments.Add(textDocument); + } + } + + string usfm = sb.ToString(); + PreTranslationService + .GetPreTranslationUsfmAsync( + projectId, + bookNum, + chapterNum: 0, + Arg.Any(), + CancellationToken.None + ) + .Returns(Task.FromResult(usfm)); + + ParatextService + .GetChaptersAsUsj(Arg.Any(), Paratext01, bookNum, usfm) + .Returns(usx.Select(UsxToUsj.UsxStringToUsj)); + + // Return the chapter deltas for the specific chapter + DeltaUsxMapper + .ToChapterDeltas(Arg.Any()) + .Returns(callInfo => + { + var usxDoc = callInfo.Arg(); + string chapterNumber = usxDoc.Descendants("chapter").First().Attribute("number")!.Value; + return new[] { chapterDeltas.Single(c => c.Number == int.Parse(chapterNumber)) }; + }); + + // Update the permissions for the user applying the draft + ProjectService + .When(x => + x.UpdatePermissionsAsync( + Arg.Any(), + Arg.Any>(), + users: null, + books: Arg.Any>(), + CancellationToken.None + ) + ) + .Do(callInfo => + { + string userId = callInfo.ArgAt(0); + var projectDoc = callInfo.ArgAt>(1); + foreach (var text in projectDoc.Data.Texts) + { + text.Permissions.TryAdd( + userId, + canWriteBook ? TextInfoPermission.Write : TextInfoPermission.Read + ); + foreach (var chapter in text.Chapters) + { + chapter.Permissions.TryAdd( + userId, + chapter.Number <= writeChapters ? TextInfoPermission.Write : TextInfoPermission.Read + ); + } + } + }); + } + + public async Task VerifyDraftAsync( + DraftApplyResult result, + string targetProjectId, + int numberOfChapters, + bool bookExists, + bool canWriteBook, + int writeChapters + ) + { + await ProjectService + .Received() + .UpdatePermissionsAsync( + User01, + Arg.Any>(), + users: null, + books: Arg.Any>(), + CancellationToken.None + ); + ExceptionHandler.DidNotReceive().ReportException(Arg.Any()); + + await Assert.ThatAsync( + () => TextDocuments.CountDocumentsAsync(t => t != null), + Is.EqualTo(numberOfChapters) + ); + + int numberOfTexts = 0; + if (canWriteBook && writeChapters > 0) + { + // The number of texts will correspond to the number of written chapters + numberOfTexts = writeChapters; + } + else if (bookExists && targetProjectId == Project01) + { + // The number of texts will correspond to the number of chapters in the book to start with + numberOfTexts = numberOfChapters; + } + + await Assert.ThatAsync( + () => Texts.CountDocumentsAsync(t => t.Id.Contains(targetProjectId)), + Is.EqualTo(numberOfTexts) + ); + + Assert.That(result.ChangesSaved, Is.EqualTo(canWriteBook && writeChapters > 0)); + if (writeChapters < numberOfChapters) + { + Assert.That(result.Failures, Is.Not.Empty); + } + } + public TranslationBuild ConfigureTranslationBuild(TranslationBuild? translationBuild = null) { const string message = "Finalizing"; diff --git a/test/SIL.XForge.Scripture.Tests/Services/ParatextSyncRunnerTests.cs b/test/SIL.XForge.Scripture.Tests/Services/ParatextSyncRunnerTests.cs index 9a8e8d0e186..39e6b306269 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/ParatextSyncRunnerTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/ParatextSyncRunnerTests.cs @@ -663,6 +663,7 @@ await env sfProjDoc.Data.Id == "project01" && sfProjDoc.Data.ParatextId == "target" ), Arg.Any>(), + Arg.Any>(), Arg.Any() ); } @@ -2396,6 +2397,7 @@ await env sfProjDoc.Data.Id == "project01" && sfProjDoc.Data.ParatextId == "target" ), Arg.Any>(), + Arg.Any>(), Arg.Any() ); } @@ -2445,6 +2447,7 @@ await env sfProjDoc.Data.Id == "project01" && sfProjDoc.Data.ParatextId == "target" ), Arg.Any>(), + Arg.Any>(), Arg.Any() ); } From 18975a79d065327698d9939f2aa79be938d22473 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Wed, 24 Sep 2025 16:22:58 +1200 Subject: [PATCH 3/5] SF-3567 Add frontend service --- .../src/app/core/sf-project.service.spec.ts | 13 +++++++++++++ .../ClientApp/src/app/core/sf-project.service.ts | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts index 9f5b187af8b..2e6f3fd4482 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts @@ -74,6 +74,19 @@ describe('SFProjectService', () => { })); }); + describe('onlineApplyPreTranslationToProject', () => { + it('should invoke the command service', fakeAsync(async () => { + const env = new TestEnvironment(); + const projectId = 'project01'; + const scriptureRange = 'GEN-REV'; + const targetProjectId = 'project01'; + const timestamp = new Date(); + await env.service.onlineApplyPreTranslationToProject(projectId, scriptureRange, targetProjectId, timestamp); + verify(mockedCommandService.onlineInvoke(anything(), 'applyPreTranslationToProject', anything())).once(); + expect().nothing(); + })); + }); + class TestEnvironment { readonly httpTestingController: HttpTestingController; readonly service: SFProjectService; diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts index f980699cc89..995ad65f1d7 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts @@ -180,6 +180,20 @@ export class SFProjectService extends ProjectService { return this.onlineInvoke('cancelSync', { projectId: id }); } + onlineApplyPreTranslationToProject( + projectId: string, + scriptureRange: string, + targetProjectId: string, + timestamp: Date + ): Promise { + return this.onlineInvoke('applyPreTranslationToProject', { + projectId, + scriptureRange, + targetProjectId, + timestamp: timestamp.toISOString() + }); + } + onlineSetPreTranslate(projectId: string, preTranslate: boolean): Promise { return this.onlineInvoke('setPreTranslate', { projectId, From 42801d60fa79595f9719b9baaa95bb390cf0a034 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Mon, 29 Sep 2025 09:21:06 +1300 Subject: [PATCH 4/5] Updates from code review feedback --- src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs | 4 ++-- src/SIL.XForge.Scripture/Services/MachineApiService.cs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs b/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs index 94d224ed234..2171603798d 100644 --- a/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs +++ b/src/SIL.XForge.Scripture/Services/DeltaUsxMapper.cs @@ -177,8 +177,8 @@ private static bool CanParaContainVerseText(string? style) public static string ExtractBookId(string usfm) { - string firstLine = usfm.Split('\n').FirstOrDefault()?.Trim() ?? string.Empty; - return BookIdRegex().Match(firstLine).Groups[1].Value; + Match match = BookIdRegex().Match(usfm); + return match.Success ? match.Groups[1].Value : string.Empty; } /// diff --git a/src/SIL.XForge.Scripture/Services/MachineApiService.cs b/src/SIL.XForge.Scripture/Services/MachineApiService.cs index 43bb5457241..e15aecebb0b 100644 --- a/src/SIL.XForge.Scripture/Services/MachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineApiService.cs @@ -30,6 +30,7 @@ using Chapter = SIL.XForge.Scripture.Models.Chapter; using TextInfo = SIL.XForge.Scripture.Models.TextInfo; +// Disable notice "The logging message template should not vary between calls to..." #pragma warning disable CA2254 namespace SIL.XForge.Scripture.Services; @@ -167,7 +168,9 @@ await hubContext.NotifyDraftApplyProgress( await hubContext.NotifyDraftApplyProgress(sfProjectId, new DraftApplyState { State = message }); } - // Ensure that if chapters is blank, it contains every chapter in the book + // Ensure that if chapters is blank, it contains every chapter in the book. + // ScriptureRangeParser will return no chapters, meaning all chapters, + // if the scripture range just specifies a book without chapter numbers. List chapters = bookChapters; if (chapters.Count == 0) { From 207f3c0169cce77563e8c0773d308f9878245804 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 2 Oct 2025 08:08:32 +1300 Subject: [PATCH 5/5] Further updates from code review feedback --- .../src/app/core/sf-project.service.spec.ts | 15 +++++-- .../src/app/core/sf-project.service.ts | 4 +- .../Controllers/SFProjectsRpcController.cs | 4 +- .../Models/DraftApplyResult.cs | 4 +- .../Services/MachineApiService.cs | 44 +++++++------------ 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts index 2e6f3fd4482..843ac57546e 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.spec.ts @@ -1,7 +1,7 @@ import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'; import { HttpTestingController, provideHttpClientTesting } from '@angular/common/http/testing'; import { fakeAsync, TestBed } from '@angular/core/testing'; -import { anything, mock, verify } from 'ts-mockito'; +import { anything, mock, verify, when } from 'ts-mockito'; import { CommandService } from 'xforge-common/command.service'; import { RealtimeService } from 'xforge-common/realtime.service'; import { configureTestingModule } from 'xforge-common/test-utils'; @@ -76,14 +76,23 @@ describe('SFProjectService', () => { describe('onlineApplyPreTranslationToProject', () => { it('should invoke the command service', fakeAsync(async () => { + const jobId: string = 'job01'; const env = new TestEnvironment(); + when(mockedCommandService.onlineInvoke(anything(), 'applyPreTranslationToProject', anything())).thenReturn( + Promise.resolve(jobId) + ); const projectId = 'project01'; const scriptureRange = 'GEN-REV'; const targetProjectId = 'project01'; const timestamp = new Date(); - await env.service.onlineApplyPreTranslationToProject(projectId, scriptureRange, targetProjectId, timestamp); + const actual = await env.service.onlineApplyPreTranslationToProject( + projectId, + scriptureRange, + targetProjectId, + timestamp + ); verify(mockedCommandService.onlineInvoke(anything(), 'applyPreTranslationToProject', anything())).once(); - expect().nothing(); + expect(actual).toBe(jobId); })); }); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts index 995ad65f1d7..4e7835d51d6 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts @@ -185,8 +185,8 @@ export class SFProjectService extends ProjectService { scriptureRange: string, targetProjectId: string, timestamp: Date - ): Promise { - return this.onlineInvoke('applyPreTranslationToProject', { + ): Promise { + return this.onlineInvoke('applyPreTranslationToProject', { projectId, scriptureRange, targetProjectId, diff --git a/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs b/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs index 950d669896f..c59a7348277 100644 --- a/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs +++ b/src/SIL.XForge.Scripture/Controllers/SFProjectsRpcController.cs @@ -45,7 +45,7 @@ DateTime timestamp try { // Run the background job - backgroundJobClient.Enqueue(r => + string jobId = backgroundJobClient.Enqueue(r => r.ApplyPreTranslationToProjectAsync( UserId, projectId, @@ -55,7 +55,7 @@ DateTime timestamp CancellationToken.None ) ); - return Ok(); + return Ok(jobId); } catch (Exception) { diff --git a/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs b/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs index d8e0973545b..c36184461ab 100644 --- a/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs +++ b/src/SIL.XForge.Scripture/Models/DraftApplyResult.cs @@ -13,9 +13,9 @@ public class DraftApplyResult public bool ChangesSaved { get; set; } /// - /// A list of any chapters that failed to apply in the format "GEN 1". + /// A list of any books and chapters that failed to apply in the format "GEN" and "GEN 1". /// - public List Failures = []; + public HashSet Failures = []; /// /// A log containing any warnings or errors that occurred while applying the draft. diff --git a/src/SIL.XForge.Scripture/Services/MachineApiService.cs b/src/SIL.XForge.Scripture/Services/MachineApiService.cs index e15aecebb0b..b996c77302d 100644 --- a/src/SIL.XForge.Scripture/Services/MachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineApiService.cs @@ -151,7 +151,7 @@ CancellationToken cancellationToken { await hubContext.NotifyDraftApplyProgress( sfProjectId, - new DraftApplyState { State = $"Retrieving draft for {Canon.BookIdToEnglishName(book)}." } + new DraftApplyState { State = $"Retrieving draft for {book}." } ); int bookNum = Canon.BookIdToNumber(book); @@ -212,11 +212,12 @@ await hubContext.NotifyDraftApplyProgress( // If the usfm is invalid, skip this book if (string.IsNullOrWhiteSpace(usfm)) { + result.Failures.Add(book); await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = $"No book number for {Canon.BookNumberToEnglishName(bookNum)}.", + State = $"No draft available for {Canon.BookNumberToId(bookNum)}.", } ); break; @@ -231,7 +232,7 @@ await hubContext.NotifyDraftApplyProgress( new DraftApplyState { State = - $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)}.", + $"Could not retrieve a valid draft for {Canon.BookNumberToId(bookNum)}.", } ); @@ -265,7 +266,7 @@ await hubContext.NotifyDraftApplyProgress( new DraftApplyState { State = - $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterNum}.", + $"Could not retrieve draft for {Canon.BookNumberToId(bookNum)} {chapterNum}.", } ); continue; @@ -284,8 +285,7 @@ await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = - $"Could not retrieve draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterNum}.", + State = $"Could not retrieve draft for {Canon.BookNumberToId(bookNum)} {chapterNum}.", } ); continue; @@ -327,7 +327,7 @@ await hubContext.NotifyDraftApplyProgress( bool successful = false; try { - // Being the transaction + // Begin the transaction connection.BeginTransaction(); // Begin a transaction, and update the project @@ -413,16 +413,12 @@ await projectService.UpdatePermissionsAsync( if (textIndex == -1) { string bookId = Canon.BookNumberToId(bookNum); - if (result.Failures.LastOrDefault() != bookId) + if (result.Failures.Add(bookId)) { - // Only list the book failure once - result.Failures.Add(bookId); + // Only notify the book failure once per book await hubContext.NotifyDraftApplyProgress( sfProjectId, - new DraftApplyState - { - State = $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)}.", - } + new DraftApplyState { State = $"Could not save draft for {Canon.BookNumberToId(bookNum)}." } ); } @@ -441,16 +437,12 @@ await hubContext.NotifyDraftApplyProgress( } string bookId = Canon.BookNumberToId(bookNum); - if (result.Failures.LastOrDefault() != bookId) + if (result.Failures.Add(bookId)) { - // Only list the book failure once - result.Failures.Add(bookId); + // Only notify the book failure once per book await hubContext.NotifyDraftApplyProgress( sfProjectId, - new DraftApplyState - { - State = $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)}.", - } + new DraftApplyState { State = $"Could not save draft for {Canon.BookNumberToId(bookNum)}." } ); } @@ -468,8 +460,7 @@ await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = - $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + State = $"Could not save draft for {Canon.BookNumberToId(bookNum)} {chapterDelta.Number}.", } ); continue; @@ -499,8 +490,7 @@ await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = - $"Could not save draft for {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + State = $"Could not save draft for {Canon.BookNumberToId(bookNum)} {chapterDelta.Number}.", } ); continue; @@ -523,7 +513,7 @@ await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = $"Updating {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + State = $"Updating {Canon.BookNumberToId(bookNum)} {chapterDelta.Number}.", } ); } @@ -535,7 +525,7 @@ await hubContext.NotifyDraftApplyProgress( sfProjectId, new DraftApplyState { - State = $"Creating {Canon.BookNumberToEnglishName(bookNum)} {chapterDelta.Number}.", + State = $"Creating {Canon.BookNumberToId(bookNum)} {chapterDelta.Number}.", } ); }