BC-5629 batch deletion mechanism (#4521)

* first commit

* add some tests

* add test cases and services

* add new (almost empty for now) batch deletion app

* refactor config vars

* add optional env var for specifying delay between the API calls

* add usecases and test cases

* fix importing

* add type in uc

* fix import

* add references service that'll load all the references to the data we want to delete

* fix most of issue form review

* add deletion API client with just a single method for now that allows for sending a deletion request

* refactor the env vars for configurting the Admin API

* add exporting DeletionClientConfig

* move references service to the deletion module

* delete unused code

* add batch deletion service that makes it possible ot queue deletion for many references at once

* move some parts of the interface to the interface subdir

* add an interface for the batch deletion summary

* move some interfaces to a separate subdir

* refactor the batch deletion summary interface

* add uc for the batch deletion

* remove unused annotation

* refactor deletion client implementation

* add batch deletion service implementation

* add UC for the batch deletion

* add a console app for the deletion module and a console command to manage deletion requests queue

* remove no longer used app, add param to make it possible to define delay between the client calls for the case one would like to queue many thousands of deletion requests at once

* remove no longer used separate batch-deletion module (it became a part of the main deletion module)

* fix invalid key

* remove no longer used config vars

* remove no longer used commands

* remove no longer used Nest cli config

* remove no longer used code

* change name of the method that prepares default headers

* add builders for most of the interfaces

* add builders for the remaining interfaces

* add type in catch clause

* do some adjustments, move PushDeletionRequestsOptions interface to a separate file

* remove unused import

* rollback

* remove unnecessary indent

* remove unnecessary indents

* remove empty line

* remove repeated imports

* refactor some imports to omit calling Configuration.get() on every subpackage import

* add builder for the DeletionRequestOutput class

* add unit tests for the batch deletion service

* add unit tests for the BatchDeletionUc

* modify env keys for the Admin API client configuration, refactor the way the deletion module's console is bootstrapped

* fix invalid import, remove unused undefined arg

* add comment to ignore console.ts file for coverage

* move deletion client config interface to a separate file, refactor function that prepares current config, add unit tests for it

* fix invalid import

* add more test cases to the deletion client unit tests

* change invalid import

Co-authored-by: WojciechGrancow <[email protected]>

* fix invalid import

* add builder for the PushDeletionRequestsOptions class, add unit tests for the DeletionQueueConsole

* rename the file containing the deletion module console to deletion.console.ts, add coverage exclusion for it for the Sonar coverage analysis

* remove deletion.console.ts from the sonar.coverage.exclusions param as it doesn't seem to work anyway

* add deletion.console.ts file to the coverage exclusions (another try with different path)

* change name of the file containing the deletion console app

* fix some imports

* move default value for the ADMIN_API_CLIENT object to default.schema.json

* move default for the BASE_URL

* move Deletion module console app to the apps/ dir

* add separate functino to log error and set exit code

* add handling of the case that only CR chars are used as a line separators

* add use of the BatchDeletionSummaryBuilder in place of an anonymous object creation

* fix some imports/exports

* refactor console app flow

---------

Co-authored-by: WojciechGrancow <[email protected]>
Co-authored-by: WojciechGrancow <[email protected]>

apps/server/src/apps/deletion-console.app.ts

@@ -0,0 +1,31 @@
+/* istanbul ignore file */
+import { BootstrapConsole } from 'nestjs-console';
+import { DeletionConsoleModule } from '@modules/deletion';
+
+async function run() {
+	const bootstrap = new BootstrapConsole({
+		module: DeletionConsoleModule,
+		useDecorators: true,
+	});
+
+	const app = await bootstrap.init();
+
+	try {
+		await app.init();
+
+		// Execute console application with provided arguments.
+		await bootstrap.boot();
+	} catch (err) {
+		// eslint-disable-next-line no-console, @typescript-eslint/no-unsafe-call
+		console.error(err);
+
+		// Set the exit code to 1 to indicate a console app failure.
+		process.exitCode = 1;
+	}
+
+	// Always close the app, even if some exception
+	// has been thrown from the console app.
+	await app.close();
+}
+
+void run();

apps/server/src/modules/deletion/client/builder/deletion-request-input.builder.spec.ts

@@ -0,0 +1,33 @@
+import { ObjectId } from 'bson';
+import { DeletionRequestInput } from '../interface';
+import { DeletionRequestInputBuilder } from './deletion-request-input.builder';
+
+describe(DeletionRequestInputBuilder.name, () => {
+	describe(DeletionRequestInputBuilder.build.name, () => {
+		describe('when called with proper arguments', () => {
+			const setup = () => {
+				const targetRefDomain = 'school';
+				const targetRefId = new ObjectId().toHexString();
+				const deleteInMinutes = 43200;
+
+				const expectedOutput: DeletionRequestInput = {
+					targetRef: {
+						domain: targetRefDomain,
+						id: targetRefId,
+					},
+					deleteInMinutes,
+				};
+
+				return { targetRefDomain, targetRefId, deleteInMinutes, expectedOutput };
+			};
+
+			it('should return valid object with expected values', () => {
+				const { targetRefDomain, targetRefId, deleteInMinutes, expectedOutput } = setup();
+
+				const output = DeletionRequestInputBuilder.build(targetRefDomain, targetRefId, deleteInMinutes);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});

apps/server/src/modules/deletion/client/builder/deletion-request-input.builder.ts

@@ -0,0 +1,11 @@
+import { DeletionRequestInput } from '../interface';
+import { DeletionRequestTargetRefInputBuilder } from './deletion-request-target-ref-input.builder';
+
+export class DeletionRequestInputBuilder {
+	static build(targetRefDomain: string, targetRefId: string, deleteInMinutes?: number): DeletionRequestInput {
+		return {
+			targetRef: DeletionRequestTargetRefInputBuilder.build(targetRefDomain, targetRefId),
+			deleteInMinutes,
+		};
+	}
+}

apps/server/src/modules/deletion/client/builder/deletion-request-output.builder.spec.ts

@@ -0,0 +1,29 @@
+import { ObjectId } from 'bson';
+import { DeletionRequestOutput } from '../interface';
+import { DeletionRequestOutputBuilder } from './deletion-request-output.builder';
+
+describe(DeletionRequestOutputBuilder.name, () => {
+	describe(DeletionRequestOutputBuilder.build.name, () => {
+		describe('when called with proper arguments', () => {
+			const setup = () => {
+				const requestId = new ObjectId().toHexString();
+				const deletionPlannedAt = new Date();
+
+				const expectedOutput: DeletionRequestOutput = {
+					requestId,
+					deletionPlannedAt,
+				};
+
+				return { requestId, deletionPlannedAt, expectedOutput };
+			};
+
+			it('should return valid object with expected values', () => {
+				const { requestId, deletionPlannedAt, expectedOutput } = setup();
+
+				const output = DeletionRequestOutputBuilder.build(requestId, deletionPlannedAt);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});

apps/server/src/modules/deletion/client/builder/deletion-request-output.builder.ts

@@ -0,0 +1,10 @@
+import { DeletionRequestOutput } from '../interface';
+
+export class DeletionRequestOutputBuilder {
+	static build(requestId: string, deletionPlannedAt: Date): DeletionRequestOutput {
+		return {
+			requestId,
+			deletionPlannedAt,
+		};
+	}
+}

apps/server/src/modules/deletion/client/builder/deletion-request-target-ref-input.builder.spec.ts

@@ -0,0 +1,26 @@
+import { ObjectId } from 'bson';
+import { DeletionRequestTargetRefInput } from '../interface';
+import { DeletionRequestTargetRefInputBuilder } from './deletion-request-target-ref-input.builder';
+
+describe(DeletionRequestTargetRefInputBuilder.name, () => {
+	describe(DeletionRequestTargetRefInputBuilder.build.name, () => {
+		describe('when called with proper arguments', () => {
+			const setup = () => {
+				const domain = 'user';
+				const id = new ObjectId().toHexString();
+
+				const expectedOutput: DeletionRequestTargetRefInput = { domain, id };
+
+				return { domain, id, expectedOutput };
+			};
+
+			it('should return valid object with expected values', () => {
+				const { domain, id, expectedOutput } = setup();
+
+				const output = DeletionRequestTargetRefInputBuilder.build(domain, id);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});

apps/server/src/modules/deletion/client/builder/deletion-request-target-ref-input.builder.ts

@@ -0,0 +1,7 @@
+import { DeletionRequestTargetRefInput } from '../interface';
+
+export class DeletionRequestTargetRefInputBuilder {
+	static build(domain: string, id: string): DeletionRequestTargetRefInput {
+		return { domain, id };
+	}
+}

apps/server/src/modules/deletion/client/builder/index.ts

@@ -0,0 +1,3 @@
+export * from './deletion-request-target-ref-input.builder';
+export * from './deletion-request-input.builder';
+export * from './deletion-request-output.builder';

apps/server/src/modules/deletion/client/deletion-client.config.spec.ts

@@ -0,0 +1,41 @@
+import { IConfig } from '@hpi-schul-cloud/commons/lib/interfaces/IConfig';
+import { Configuration } from '@hpi-schul-cloud/commons/lib';
+import { DeletionClientConfig } from './interface';
+import { getDeletionClientConfig } from './deletion-client.config';
+
+describe(getDeletionClientConfig.name, () => {
+	let configBefore: IConfig;
+
+	beforeAll(() => {
+		configBefore = Configuration.toObject({ plainSecrets: true });
+	});
+
+	afterEach(() => {
+		Configuration.reset(configBefore);
+	});
+
+	describe('when called', () => {
+		const setup = () => {
+			const baseUrl = 'http://api-admin:4030';
+			const apiKey = '652559c2-93da-42ad-94e1-640e3afbaca0';
+
+			Configuration.set('ADMIN_API_CLIENT__BASE_URL', baseUrl);
+			Configuration.set('ADMIN_API_CLIENT__API_KEY', apiKey);
+
+			const expectedConfig: DeletionClientConfig = {
+				ADMIN_API_CLIENT_BASE_URL: baseUrl,
+				ADMIN_API_CLIENT_API_KEY: apiKey,
+			};
+
+			return { expectedConfig };
+		};
+
+		it('should return config with proper values', () => {
+			const { expectedConfig } = setup();
+
+			const config = getDeletionClientConfig();
+
+			expect(config).toEqual(expectedConfig);
+		});
+	});
+});

apps/server/src/modules/deletion/client/deletion-client.config.ts

@@ -0,0 +1,9 @@
+import { Configuration } from '@hpi-schul-cloud/commons/lib';
+import { DeletionClientConfig } from './interface';
+
+export const getDeletionClientConfig = (): DeletionClientConfig => {
+	return {
+		ADMIN_API_CLIENT_BASE_URL: Configuration.get('ADMIN_API_CLIENT__BASE_URL') as string,
+		ADMIN_API_CLIENT_API_KEY: Configuration.get('ADMIN_API_CLIENT__API_KEY') as string,
+	};
+};

apps/server/src/modules/deletion/client/deletion.client.spec.ts

@@ -0,0 +1,154 @@
+import { of } from 'rxjs';
+import { AxiosResponse } from 'axios';
+import { HttpService } from '@nestjs/axios';
+import { ConfigService } from '@nestjs/config';
+import { Test, TestingModule } from '@nestjs/testing';
+import { createMock, DeepMocked } from '@golevelup/ts-jest';
+import { axiosResponseFactory } from '@shared/testing';
+import { DeletionRequestInputBuilder, DeletionRequestOutputBuilder } from '.';
+import { DeletionRequestOutput } from './interface';
+import { DeletionClient } from './deletion.client';
+
+describe(DeletionClient.name, () => {
+	let module: TestingModule;
+	let client: DeletionClient;
+	let httpService: DeepMocked<HttpService>;
+
+	beforeEach(async () => {
+		module = await Test.createTestingModule({
+			providers: [
+				DeletionClient,
+				{
+					provide: ConfigService,
+					useValue: createMock<ConfigService>({
+						get: jest.fn((key: string) => {
+							if (key === 'ADMIN_API_CLIENT_BASE_URL') {
+								return 'http://localhost:4030';
+							}
+
+							// Default is for the Admin APIs API Key.
+							return '6b3df003-61e9-467c-9e6b-579634801896';
+						}),
+					}),
+				},
+				{
+					provide: HttpService,
+					useValue: createMock<HttpService>(),
+				},
+			],
+		}).compile();
+
+		client = module.get(DeletionClient);
+		httpService = module.get(HttpService);
+	});
+
+	afterEach(() => {
+		jest.resetAllMocks();
+	});
+
+	afterAll(async () => {
+		await module.close();
+	});
+
+	describe('queueDeletionRequest', () => {
+		describe('when received valid response with expected HTTP status code', () => {
+			const setup = () => {
+				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
+
+				const output: DeletionRequestOutput = DeletionRequestOutputBuilder.build(
+					'6536ce29b595d7c8e5faf200',
+					new Date('2024-10-15T12:42:50.521Z')
+				);
+
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					data: output,
+					status: 202,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+
+				return { input, output };
+			};
+
+			it('should return proper output', async () => {
+				const { input, output } = setup();
+
+				const result = await client.queueDeletionRequest(input);
+
+				expect(result).toEqual(output);
+			});
+		});
+
+		describe('when received invalid HTTP status code in a response', () => {
+			const setup = () => {
+				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
+
+				const output: DeletionRequestOutput = DeletionRequestOutputBuilder.build('', new Date());
+
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					data: output,
+					status: 200,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+
+				return { input };
+			};
+
+			it('should throw an exception', async () => {
+				const { input } = setup();
+
+				await expect(client.queueDeletionRequest(input)).rejects.toThrow(Error);
+			});
+		});
+
+		describe('when received no requestId in a response', () => {
+			const setup = () => {
+				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
+
+				const output: DeletionRequestOutput = DeletionRequestOutputBuilder.build(
+					'',
+					new Date('2024-10-15T12:42:50.521Z')
+				);
+
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					data: output,
+					status: 202,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+
+				return { input };
+			};
+
+			it('should throw an exception', async () => {
+				const { input } = setup();
+
+				await expect(client.queueDeletionRequest(input)).rejects.toThrow(Error);
+			});
+		});
+
+		describe('when received no deletionPlannedAt in a response', () => {
+			const setup = () => {
+				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
+
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					data: {
+						requestId: '6536ce29b595d7c8e5faf200',
+					},
+					status: 202,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+
+				return { input };
+			};
+
+			it('should throw an exception', async () => {
+				const { input } = setup();
+
+				await expect(client.queueDeletionRequest(input)).rejects.toThrow(Error);
+			});
+		});
+	});
+});