Skip to content

SF-3163 Store Draft Sources as Arrays #3508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 29, 2025
Merged

SF-3163 Store Draft Sources as Arrays #3508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2570fdd
Fix e2e test time out caused by latency
pmachapman Oct 13, 2025
0daee9e
Update MongoDB query
pmachapman Oct 12, 2025
ae77f4d
Realtime server updates
pmachapman Oct 7, 2025
fae2cd7
C# backend updates
pmachapman Oct 8, 2025
da07640
Frontend updates
pmachapman Oct 7, 2025
1e7bc38
Merge branch 'master' into feature/SF-3163
Nateowami Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions mongodb/Projects/FindProjectsUsingProjectAsSource.mongodb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,22 @@
use('xforge')
use("xforge");

// Query to find what project is using a project as a source (and therefore preventing the specified project from being
// deleted). This is a re-implementation of the logic in SFProjectService.IsSourceProject

const shortName = '';
const shortName = "";

const id = db.sf_projects.findOne({ shortName })._id
const id = db.sf_projects.findOne({ shortName })._id;

db.sf_projects.find({
$or: [{
'translateConfig.source.projectRef': id
}, {
'translateConfig.draftConfig.alternateSource.projectRef': id
}, {
'translateConfig.draftConfig.alternateTrainingSource.projectRef': id
}, {
'translateConfig.draftConfig.additionalTrainingSource.projectRef': id
}]
})
$or: [
{
"translateConfig.source.projectRef": id
},
{
"translateConfig.draftConfig.draftingSources.projectRef": id
},
{
"translateConfig.draftConfig.trainingSources.projectRef": id
}
]
});
5 changes: 2 additions & 3 deletions src/RealtimeServer/scriptureforge/models/sf-project-test-data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ function testProjectProfile(ordinal: number): SFProjectProfile {
preTranslate: false,
defaultNoteTagId: 1,
draftConfig: {
additionalTrainingSourceEnabled: false,
alternateSourceEnabled: false,
alternateTrainingSourceEnabled: false,
draftingSources: [],
trainingSources: [],
lastSelectedTrainingDataFiles: []
}
},
Expand Down
5 changes: 2 additions & 3 deletions src/RealtimeServer/scriptureforge/models/sf-project.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,8 @@ export const SF_PROJECT_INDEX_PATHS: (string | [string, CreateIndexesOptions])[]
obj<SFProject>().pathStr(p => p.shortName),
// Indexes for SFProjectService.IsSourceProject() in .NET
[obj<SFProject>().pathStr(p => p.translateConfig.source!.projectRef), { sparse: true }],
[obj<SFProject>().pathStr(p => p.translateConfig.draftConfig.additionalTrainingSource!.projectRef), { sparse: true }],
[obj<SFProject>().pathStr(p => p.translateConfig.draftConfig.alternateSource!.projectRef), { sparse: true }],
[obj<SFProject>().pathStr(p => p.translateConfig.draftConfig.alternateTrainingSource!.projectRef), { sparse: true }]
['translateConfig.draftConfig.draftingSources.projectRef', { sparse: true }],
['translateConfig.draftConfig.trainingSources.projectRef', { sparse: true }]
];

/** Length of id for a DBL resource. */
Expand Down
8 changes: 2 additions & 6 deletions src/RealtimeServer/scriptureforge/models/translate-config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,8 @@ export interface DraftUsfmConfig {
}

export interface DraftConfig {
additionalTrainingSourceEnabled: boolean;
additionalTrainingSource?: TranslateSource;
alternateSourceEnabled: boolean;
alternateSource?: TranslateSource;
alternateTrainingSourceEnabled: boolean;
alternateTrainingSource?: TranslateSource;
draftingSources: TranslateSource[];
trainingSources: TranslateSource[];
lastSelectedTrainingDataFiles: string[];
lastSelectedTrainingScriptureRanges?: ProjectScriptureRange[];
lastSelectedTranslationScriptureRanges?: ProjectScriptureRange[];
Expand Down
147 changes: 147 additions & 0 deletions src/RealtimeServer/scriptureforge/services/sf-project-migrations.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -905,6 +905,153 @@ describe('SFProjectMigrations', () => {
]);
});
});

describe('version 27', () => {
it('adds empty arrays if preTranslate is false', async () => {
const env = new TestEnvironment(26);
const conn = env.server.connect();
await createDoc(conn, SF_PROJECTS_COLLECTION, 'project01', {
translateConfig: { preTranslate: false, draftConfig: {} }
});
let projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toBeUndefined();

await env.server.migrateIfNecessary();

projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toEqual([]);
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toEqual([]);
});

it('adds empty arrays if preTranslate is true but sources were disabled', async () => {
const env = new TestEnvironment(26);
const conn = env.server.connect();
await createDoc(conn, SF_PROJECTS_COLLECTION, 'project01', {
translateConfig: {
preTranslate: true,
draftConfig: {
alternateTrainingSource: { projectRef: 'project02' },
alternateTrainingSourceEnabled: false,
alternateSource: { projectRef: 'project03' },
alternateSourceEnabled: false,
additionalTrainingSource: { projectRef: 'project04' },
additionalTrainingSourceEnabled: false
}
}
});
let projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toBeUndefined();

await env.server.migrateIfNecessary();

projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toEqual([]);
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toEqual([]);
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSourceEnabled).toBeUndefined();
});

it('adds empty arrays if preTranslate is true and sources are enabled but undefined', async () => {
const env = new TestEnvironment(26);
const conn = env.server.connect();
await createDoc(conn, SF_PROJECTS_COLLECTION, 'project01', {
translateConfig: {
preTranslate: true,
draftConfig: {
alternateTrainingSourceEnabled: true,
alternateSourceEnabled: true,
additionalTrainingSourceEnabled: true
}
}
});
let projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toBeUndefined();

await env.server.migrateIfNecessary();

projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toEqual([]);
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toEqual([]);
});

it('adds source to both arrays if preTranslate is true but sources were disabled', async () => {
const env = new TestEnvironment(26);
const conn = env.server.connect();
await createDoc(conn, SF_PROJECTS_COLLECTION, 'project01', {
translateConfig: {
preTranslate: true,
draftConfig: {
alternateTrainingSource: { projectRef: 'project02' },
alternateTrainingSourceEnabled: false,
alternateSource: { projectRef: 'project03' },
alternateSourceEnabled: false,
additionalTrainingSource: { projectRef: 'project04' },
additionalTrainingSourceEnabled: false
},
source: { projectRef: 'project05' }
}
});
let projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toBeUndefined();

await env.server.migrateIfNecessary();

projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toEqual([{ projectRef: 'project05' }]);
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toEqual([{ projectRef: 'project05' }]);
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSourceEnabled).toBeUndefined();
});

it('adds sources to the appropriate arrays if preTranslate is true and sources are configured', async () => {
const env = new TestEnvironment(26);
const conn = env.server.connect();
await createDoc(conn, SF_PROJECTS_COLLECTION, 'project01', {
translateConfig: {
preTranslate: true,
draftConfig: {
alternateTrainingSource: { projectRef: 'project02' },
alternateTrainingSourceEnabled: true,
alternateSource: { projectRef: 'project03' },
alternateSourceEnabled: true,
additionalTrainingSource: { projectRef: 'project04' },
additionalTrainingSourceEnabled: true
},
source: { projectRef: 'project05' }
}
});
let projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toBeUndefined();

await env.server.migrateIfNecessary();

projectDoc = await fetchDoc(conn, SF_PROJECTS_COLLECTION, 'project01');
expect(projectDoc.data.translateConfig.draftConfig.draftingSources).toEqual([{ projectRef: 'project03' }]);
expect(projectDoc.data.translateConfig.draftConfig.trainingSources).toEqual([
{ projectRef: 'project02' },
{ projectRef: 'project04' }
]);
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateTrainingSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.alternateSourceEnabled).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSource).toBeUndefined();
expect(projectDoc.data.translateConfig.draftConfig.additionalTrainingSourceEnabled).toBeUndefined();
});
});
});

class TestEnvironment {
Expand Down
82 changes: 80 additions & 2 deletions src/RealtimeServer/scriptureforge/services/sf-project-migrations.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { NoteTag } from '../models/note-tag';
import { SF_PROJECT_RIGHTS, SFProjectDomain } from '../models/sf-project-rights';
import { SFProjectRole } from '../models/sf-project-role';
import { TextInfoPermission } from '../models/text-info-permission';
import { TranslateShareLevel } from '../models/translate-config';
import { TranslateShareLevel, TranslateSource } from '../models/translate-config';

class SFProjectMigration1 extends DocMigration {
static readonly VERSION = 1;
Expand Down Expand Up @@ -553,6 +553,83 @@ class SFProjectMigration26 extends DocMigration {
}
}

class SFProjectMigration27 extends DocMigration {
static readonly VERSION = 27;

async migrateDoc(doc: Doc): Promise<void> {
const ops: Op[] = [];
const draftingSources: TranslateSource[] = [];
const trainingSources: TranslateSource[] = [];

// Migrate the old values to the new structure
if (doc.data.translateConfig.preTranslate === true) {
const translateConfig = doc.data.translateConfig;
const draftConfig = translateConfig.draftConfig;
if (draftConfig.alternateTrainingSourceEnabled && draftConfig.alternateTrainingSource != null) {
trainingSources.push(draftConfig.alternateTrainingSource);
} else if (translateConfig.source != null) {
trainingSources.push(translateConfig.source);
}
if (draftConfig.additionalTrainingSourceEnabled && draftConfig.additionalTrainingSource != null) {
trainingSources.push(draftConfig.additionalTrainingSource);
}
if (draftConfig.alternateSourceEnabled && draftConfig.alternateSource != null) {
draftingSources.push(draftConfig.alternateSource);
} else if (translateConfig.source != null) {
draftingSources.push(translateConfig.source);
}
}

// Create the new structure
if (doc.data.translateConfig.draftConfig.draftingSources == null) {
ops.push({ p: ['translateConfig', 'draftConfig', 'draftingSources'], oi: draftingSources });
}
if (doc.data.translateConfig.draftConfig.trainingSources == null) {
ops.push({ p: ['translateConfig', 'draftConfig', 'trainingSources'], oi: trainingSources });
}

// Remove the old values
if (doc.data.translateConfig.draftConfig.alternateSourceEnabled != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'alternateSourceEnabled'],
od: doc.data.translateConfig.draftConfig.alternateSourceEnabled
});
}
if (doc.data.translateConfig.draftConfig.alternateSource != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'alternateSource'],
od: doc.data.translateConfig.draftConfig.alternateSource
});
}
if (doc.data.translateConfig.draftConfig.alternateTrainingSourceEnabled != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'alternateTrainingSourceEnabled'],
od: doc.data.translateConfig.draftConfig.alternateTrainingSourceEnabled
});
}
if (doc.data.translateConfig.draftConfig.alternateTrainingSource != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'alternateTrainingSource'],
od: doc.data.translateConfig.draftConfig.alternateTrainingSource
});
}
if (doc.data.translateConfig.draftConfig.additionalTrainingSourceEnabled != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'additionalTrainingSourceEnabled'],
od: doc.data.translateConfig.draftConfig.additionalTrainingSourceEnabled
});
}
if (doc.data.translateConfig.draftConfig.additionalTrainingSource != null) {
ops.push({
p: ['translateConfig', 'draftConfig', 'additionalTrainingSource'],
od: doc.data.translateConfig.draftConfig.additionalTrainingSource
});
}

await submitMigrationOp(SFProjectMigration27.VERSION, doc, ops);
}
}

export const SF_PROJECT_MIGRATIONS: MigrationConstructor[] = monotonicallyIncreasingMigrationList([
SFProjectMigration1,
SFProjectMigration2,
Expand All @@ -579,5 +656,6 @@ export const SF_PROJECT_MIGRATIONS: MigrationConstructor[] = monotonicallyIncrea
SFProjectMigration23,
SFProjectMigration24,
SFProjectMigration25,
SFProjectMigration26
SFProjectMigration26,
SFProjectMigration27
]);
Loading
Loading