Show a progress dialog when exporting user data

This does the following:
- Ensure the user has visual feedback that an export is in progress
  (and no surprise file dialog will appear when it finishes)
- Allow the user to cancel an export if it hasn't finished
- Makes exports load multiple groups at once to reduce requests, thus
  improving performance significantly

Closes #4345

Author: paw-hub

src/common/misc/TranslationKey.ts

@@ -2037,3 +2037,4 @@ export type TranslationKeyType =
 	| "unread_label"
 	| "replied_label"
 	| "forwarded_label"
+	| "userExportProgress_msg"

src/common/settings/UserDataExporter.ts

@@ -1,11 +1,14 @@
 import { LoginController } from "../api/main/LoginController.js"
-import { CustomerTypeRef, GroupInfoTypeRef, GroupTypeRef } from "../api/entities/sys/TypeRefs.js"
-import { assertNotNull, mapNullable, neverNull, pad, promiseMap, renderCsv, stringToUtf8Uint8Array } from "@tutao/tutanota-utils"
+import { GroupInfoTypeRef, GroupTypeRef } from "../api/entities/sys/TypeRefs.js"
+import { assertNotNull, mapNullable, pad, promiseMap, renderCsv, splitInChunks, stringToUtf8Uint8Array } from "@tutao/tutanota-utils"
 import { EntityClient } from "../api/common/EntityClient.js"
 import { FileController } from "../file/FileController.js"
 import { createDataFile } from "../api/common/DataFile.js"
 import { CounterFacade } from "../api/worker/facades/lazy/CounterFacade.js"
 import { CounterType } from "../api/common/TutanotaConstants.js"
+import { CancelledError } from "../api/common/error/CancelledError"
+
+const GROUP_DOWNLOAD_SIZE = 50
 
 export const CSV_MIMETYPE = "text/csv"
 export const USER_CSV_FILENAME = "users.csv"
@@ -19,8 +22,7 @@ interface UserExportData {
 	aliases: Array<string>
 }
 
-export async function exportUserCsv(entityClient: EntityClient, logins: LoginController, fileController: FileController, counterFacade: CounterFacade) {
-	const data = await loadUserExportData(entityClient, logins, counterFacade)
+export async function exportUserCsv(data: readonly UserExportData[], fileController: FileController) {
 	const csv = renderCsv(
 		["name", "mail address", "date created", "date deleted", "storage used (in bytes)", "aliases"],
 		data.map((user) => [
@@ -43,25 +45,58 @@ function formatDate(date: Date): string {
 /**
  * Load data for each user administrated by the logged in user, in order to be exported
  */
-export async function loadUserExportData(entityClient: EntityClient, logins: LoginController, counterFacade: CounterFacade): Promise<Array<UserExportData>> {
-	const { user } = logins.getUserController()
-	const { userGroups } = await entityClient.load(CustomerTypeRef, assertNotNull(user.customer))
-
-	const groupsAdministeredByUser = await entityClient.loadAll(GroupInfoTypeRef, userGroups)
-
-	const usedCustomerStorageCounterValues = await counterFacade.readAllCustomerCounterValues(CounterType.UserStorageLegacy, neverNull(user.customer))
-	return promiseMap(groupsAdministeredByUser, async (info) => {
-		const group = await entityClient.load(GroupTypeRef, info.group)
-		const userStorageCounterValue = usedCustomerStorageCounterValues.find((counterValue) => counterValue.counterId === group.storageCounter)
-		const usedStorage = userStorageCounterValue != null ? Number(userStorageCounterValue.value) : 0
-
-		return {
-			name: info.name,
-			mailAddress: info.mailAddress ?? "",
-			created: info.created,
-			deleted: info.deleted,
-			usedStorage,
-			aliases: info.mailAddressAliases.map((alias) => alias.mailAddress),
+export async function loadUserExportData(
+	entityClient: EntityClient,
+	logins: LoginController,
+	counterFacade: CounterFacade,
+	onProgress?: (complete: number, total: number) => unknown,
+	abortSignal?: AbortSignal,
+): Promise<UserExportData[]> {
+	const customer = await logins.getUserController().loadCustomer()
+	const groupsAdministeredByUser = await entityClient.loadAll(GroupInfoTypeRef, customer.userGroups)
+	const usedCustomerStorageCounterValues = await counterFacade.readAllCustomerCounterValues(CounterType.UserStorageLegacy, customer._id)
+
+	let isCancelled = false
+	abortSignal?.addEventListener("abort", () => (isCancelled = true))
+
+	let total = groupsAdministeredByUser.length
+	let completed = 0
+	onProgress?.(completed, total)
+
+	const downloaded = await promiseMap(splitInChunks(GROUP_DOWNLOAD_SIZE, groupsAdministeredByUser), async (infos) => {
+		if (isCancelled) {
+			throw new CancelledError("user export cancelled by user")
 		}
+
+		const groups = await entityClient.loadMultiple(
+			GroupTypeRef,
+			null,
+			infos.map((group) => group.group),
+		)
+
+		const mapped = groups.map((group) => {
+			const info = assertNotNull(groupsAdministeredByUser.find((groupInfo) => groupInfo.group === group._id))
+			const userStorageCounterValue = usedCustomerStorageCounterValues.find((counterValue) => counterValue.counterId === group.storageCounter)
+			const usedStorage = Number(userStorageCounterValue?.value ?? "0")
+			return {
+				name: info.name,
+				mailAddress: info.mailAddress ?? "",
+				created: info.created,
+				deleted: info.deleted,
+				usedStorage,
+				aliases: info.mailAddressAliases.map((alias) => alias.mailAddress),
+			}
+		})
+
+		completed += mapped.length
+
+		// in case we did not get every single group back that we requested (something may have changed in the meantime)
+		total -= infos.length - mapped.length
+
+		onProgress?.(completed, total)
+
+		return mapped
 	})
+
+	return downloaded.flat()
 }

src/mail-app/settings/SettingsView.ts

@@ -17,7 +17,7 @@ import { GroupListView } from "./groups/GroupListView.js"
 import { WhitelabelSettingsViewer } from "../../common/settings/whitelabel/WhitelabelSettingsViewer"
 import { Icons } from "../../common/gui/base/icons/Icons"
 import { theme } from "../../common/gui/theme"
-import { FeatureType, GroupType, LegacyPlans } from "../../common/api/common/TutanotaConstants"
+import { FeatureType, GroupType } from "../../common/api/common/TutanotaConstants"
 import { BootIcons } from "../../common/gui/base/icons/BootIcons"
 import { locator } from "../../common/api/main/CommonLocator"
 import { SubscriptionViewer } from "../../common/subscription/SubscriptionViewer"
@@ -44,7 +44,7 @@ import { showProgressDialog } from "../../common/gui/dialogs/ProgressDialog"
 import { createGroupSettings, createUserAreaGroupDeleteData, UserSettingsGroupRootTypeRef } from "../../common/api/entities/tutanota/TypeRefs.js"
 import { GroupInvitationFolderRow } from "../../common/sharing/view/GroupInvitationFolderRow"
 import { TemplateGroupService } from "../../common/api/entities/tutanota/Services"
-import { exportUserCsv } from "../../common/settings/UserDataExporter.js"
+import { exportUserCsv, loadUserExportData } from "../../common/settings/UserDataExporter.js"
 import { IconButton } from "../../common/gui/base/IconButton.js"
 import { BottomNav } from "../gui/BottomNav.js"
 import { getAvailableDomains } from "../../common/settings/mailaddress/MailAddressesUtils.js"
@@ -78,6 +78,8 @@ import { MailExportViewer } from "./MailExportViewer"
 import { getSupportUsageTestStage } from "../../common/support/SupportUsageTestUtils.js"
 import { LockedError } from "../../common/api/common/error/RestError"
 import { shouldHideBusinessPlans } from "../../common/subscription/SubscriptionUtils"
+import { ButtonType } from "../../common/gui/base/Button"
+import { CancelledError } from "../../common/api/common/error/CancelledError"
 
 assertMainOrNode()
 
@@ -404,7 +406,6 @@ export class SettingsView extends BaseTopLevelView implements TopLevelView<Setti
 		await this.updateShowBusinessSettings()
 		await this.updateShowAffiliateSettings()
 		const currentPlanType = await this.logins.getUserController().getPlanType()
-		const isLegacyPlan = LegacyPlans.includes(currentPlanType)
 
 		if (await this.logins.getUserController().canHaveUsers()) {
 			this._adminFolders.push(
@@ -418,7 +419,7 @@ export class SettingsView extends BaseTopLevelView implements TopLevelView<Setti
 							() => this.focusSettingsDetailsColumn(),
 							() => !isApp() && this._customDomains.isLoaded() && this._customDomains.getLoaded().length > 0,
 							() => showUserImportDialog(this._customDomains.getLoaded()),
-							() => exportUserCsv(locator.entityClient, this.logins, locator.fileController, locator.counterFacade),
+							() => this.doExportUsers(),
 						),
 					undefined,
 				),
@@ -932,6 +933,48 @@ export class SettingsView extends BaseTopLevelView implements TopLevelView<Setti
 		const customer = await this.logins.getUserController().loadCustomer()
 		this.showAffiliateSettings = isCustomizationEnabledForCustomer(customer, FeatureType.AffiliatePartner)
 	}
+
+	private async doExportUsers() {
+		try {
+			const progress = stream(0)
+			let progressText = lang.getTranslation("pleaseWait_msg")
+			const abortController = new AbortController()
+
+			const data = await showProgressDialog(
+				() => progressText,
+				loadUserExportData(
+					locator.entityClient,
+					this.logins,
+					locator.counterFacade,
+					(complete: number, total: number) => {
+						progressText = lang.getTranslation("userExportProgress_msg", {
+							"{current}": complete,
+							"{total}": total,
+						})
+						progress((complete / total) * 100)
+					},
+					abortController.signal,
+				),
+				progress,
+				{
+					middle: "exportUsers_action",
+					left: [
+						{
+							label: "cancel_action",
+							type: ButtonType.Primary,
+							click: () => abortController.abort(),
+						},
+					],
+				},
+			)
+			await exportUserCsv(data, locator.fileController)
+		} catch (e) {
+			if (e instanceof CancelledError) {
+				return
+			}
+			throw e
+		}
+	}
 }
 
 function showRenameTemplateListDialog(instance: TemplateGroupInstance) {

src/mail-app/translations/de.ts

@@ -2062,5 +2062,6 @@ export default {
 		"unread_label": "Unread",
 		"replied_label": "Replied",
 		"forwarded_label": "Forwarded",
-	}
+		      "userExportProgress_msg": "Preparing {current} of {total} user(s) for export...",
+    }
 }

src/mail-app/translations/de_sie.ts

@@ -2062,5 +2062,6 @@ export default {
         "unread_label": "Unread",
         "replied_label": "Replied",
         "forwarded_label": "Forwarded",
+        "userExportProgress_msg": "Preparing {current} of {total} user(s) for export...",
     }
 }

src/mail-app/translations/en.ts

@@ -2058,5 +2058,6 @@ export default {
         "unread_label": "Unread",
         "replied_label": "Replied",
         "forwarded_label": "Forwarded",
+        "userExportProgress_msg": "Preparing {current} of {total} user(s) for export...",
     }
 }

test/tests/settings/UserDataExportTest.ts

@@ -2,14 +2,14 @@ import o from "@tutao/otest"
 import { loadUserExportData } from "../../../src/common/settings/UserDataExporter.js"
 import { EntityClient } from "../../../src/common/api/common/EntityClient.js"
 import { LoginController } from "../../../src/common/api/main/LoginController.js"
-import { FileController } from "../../../src/common/file/FileController.js"
-import { object, when } from "testdouble"
+import { matchers, object, when } from "testdouble"
 import { CustomerTypeRef, Group, GroupInfo, GroupInfoTypeRef, GroupTypeRef, User } from "../../../src/common/api/entities/sys/TypeRefs.js"
 import { formatDateTimeUTC } from "../../../src/calendar-app/calendar/export/CalendarExporter.js"
 import { CounterFacade } from "../../../src/common/api/worker/facades/lazy/CounterFacade.js"
 import { CounterType } from "../../../src/common/api/common/TutanotaConstants.js"
 import { CounterValueTypeRef } from "../../../src/common/api/entities/monitor/TypeRefs.js"
 import { createTestEntity } from "../TestUtils.js"
+import { TypeRef } from "@tutao/tutanota-utils"
 
 o.spec("user data export", function () {
 	const customerId = "customerId"
@@ -19,34 +19,42 @@ o.spec("user data export", function () {
 		customer: customerId,
 	} as User
 
-	let allUserGroupInfos: Array<GroupInfo>
+	let allUserGroupInfos: GroupInfo[]
 
 	let entityClientMock: EntityClient
 	let counterFacadeMock: CounterFacade
 	let loginsMock: LoginController
-	let fileControllerMock: FileController
+	let allGroups: Group[]
 
 	o.beforeEach(function () {
 		allUserGroupInfos = []
 
 		loginsMock = object()
 		when(loginsMock.getUserController()).thenReturn({
-			user,
+			loadCustomer: () =>
+				Promise.resolve(
+					createTestEntity(CustomerTypeRef, {
+						userGroups: userGroupsId,
+						_id: customerId,
+					}),
+				),
 			// we only test the case where we are global admin for now
 			isGlobalAdmin: () => true,
 		})
 
 		entityClientMock = object()
-		when(entityClientMock.load(CustomerTypeRef, customerId)).thenResolve({
-			userGroups: userGroupsId,
-		})
 		when(entityClientMock.loadAll(GroupInfoTypeRef, userGroupsId)).thenResolve(allUserGroupInfos)
+		when(entityClientMock.loadMultiple(GroupTypeRef, null, matchers.anything())).thenDo(
+			(_typeref: TypeRef<Group>, _list: Id | null, groups: readonly Id[]) => {
+				return Promise.resolve(allGroups.filter((g) => groups.includes(g._id)))
+			},
+		)
 
 		counterFacadeMock = object()
-		fileControllerMock = object()
+		allGroups = []
 	})
 
-	o("should load and return correct user data ", async function () {
+	o.test("should load and return correct user data ", async function () {
 		const oneCreated = new Date(1655294400000) // 2022-06-15 12:00:00 GMT+0
 		const oneDeleted = new Date(1655469000000) // "2022-06-17 12:30:00 GMT +0"
 		const twoCreated = new Date(1657886400000) // "2022-07-15 12:00:00 GMT+0"
@@ -59,38 +67,60 @@ o.spec("user data export", function () {
 			// missing counter for second user!
 		])
 
-		const [first, second] = await loadUserExportData(entityClientMock, loginsMock, counterFacadeMock)
+		let onProgressLastComplete: number | undefined
+		let onProgressLastTotal: number | undefined
+		let onProgressCalledTimes = 0
+
+		const [first, second] = await loadUserExportData(entityClientMock, loginsMock, counterFacadeMock, (complete, total) => {
+			if (onProgressLastComplete != null) {
+				o.check(complete > onProgressLastComplete).equals(true)
+			} else {
+				o.check(complete).equals(0)
+			}
+			if (onProgressLastTotal != null) {
+				o.check(total).equals(onProgressLastTotal)
+			}
+			onProgressLastComplete = complete
+			onProgressLastTotal = total
+			onProgressCalledTimes += 1
+		})
+
+		o.check(first.name).equals("my name")
+		o.check(second.name).equals("eman ym")
 
-		o(first.name).equals("my name")
-		o(second.name).equals("eman ym")
+		o.check(first.mailAddress).equals("[email protected]")
+		o.check(second.mailAddress).equals("[email protected]")
 
-		o(first.mailAddress).equals("[email protected]")
-		o(second.mailAddress).equals("[email protected]")
+		o.check(formatDateTimeUTC(first.created)).equals("20220615T120000Z")
+		o.check(formatDateTimeUTC(second.created)).equals("20220715T120000Z")
 
-		o(formatDateTimeUTC(first.created)).equals("20220615T120000Z")
-		o(formatDateTimeUTC(second.created)).equals("20220715T120000Z")
+		o.check(formatDateTimeUTC(first.deleted!)).equals("20220617T123000Z")
+		o.check(second.deleted).equals(null)
 
-		o(formatDateTimeUTC(first.deleted!)).equals("20220617T123000Z")
-		o(second.deleted).equals(null)
+		o.check(first.usedStorage).equals(100)
+		o.check(second.usedStorage).equals(0)
 
-		o(first.usedStorage).equals(100)
-		o(second.usedStorage).equals(0)
+		o.check(first.aliases).deepEquals(["[email protected]", "[email protected]"])
+		o.check(second.aliases).deepEquals([])
 
-		o(first.aliases).deepEquals(["[email protected]", "[email protected]"])
-		o(second.aliases).deepEquals([])
+		o.check(onProgressLastComplete).equals(2)
+		o.check(onProgressLastTotal).equals(2)
+		o.check(onProgressCalledTimes).equals(2)
 	})
 
 	function addUser(name, mailAddress, created, deleted, usedStorage, aliases, userId, groupId, storageCounterId) {
-		allUserGroupInfos.push({
-			name,
-			mailAddress,
-			created,
-			deleted,
-			mailAddressAliases: aliases.map((alias) => ({ mailAddress: alias })),
-			group: groupId,
-		} as GroupInfo)
-
-		const group = { storageCounter: storageCounterId } as Group
-		when(entityClientMock.load(GroupTypeRef, groupId)).thenResolve({ storageCounter: group.storageCounter })
+		allUserGroupInfos.push(
+			createTestEntity(GroupInfoTypeRef, {
+				name,
+				mailAddress,
+				created,
+				deleted,
+				mailAddressAliases: aliases.map((alias) => ({ mailAddress: alias })),
+				group: groupId,
+			}),
+		)
+
+		const group = createTestEntity(GroupTypeRef, { storageCounter: storageCounterId, _id: groupId })
+		allGroups.push(group)
 	}
 })