N21-2190 add ctl media metadata sync cronjob

ansible/roles/media-metadata-sync/defaults/main.yml

@@ -0,0 +1 @@
+SERVER_MEDIA_METADATA_SYNC_CRONJOB_SCHEDULE: "30 2 * * *"

ansible/roles/media-metadata-sync/meta/main.yml

@@ -0,0 +1,9 @@
+galaxy_info:
+  role_name: media-metadata-sync
+  author: Schul-Cloud Verbund
+  description: media-metadata-sync role for the media metadata synchronization purposes
+  company: Schul-Cloud Verbund
+  license: license (AGPLv3)
+  min_ansible_version: 2.8
+  galaxy_tags: []
+dependencies: []

ansible/roles/media-metadata-sync/tasks/main.yml

@@ -0,0 +1,17 @@
+- name: media metadata sync CronJob ConfigMap
+  kubernetes.core.k8s:
+    kubeconfig: ~/.kube/config
+    namespace: "{{ NAMESPACE }}"
+    template: media-metadata-sync-cronjob-configmap.yml.j2
+    state: "{{ 'present' if WITH_MEDIA_METADATA_SYNC is defined and WITH_MEDIA_METADATA_SYNC|bool else 'absent'}}"
+  tags:
+    - configmap
+
+- name: media metadata sync sync CronJob
+  kubernetes.core.k8s:
+    kubeconfig: ~/.kube/config
+    namespace: "{{ NAMESPACE }}"
+    template: media-metadata-sync-cronjob.yml.j2
+    state: "{{ 'present' if WITH_MEDIA_METADATA_SYNC is defined and WITH_MEDIA_METADATA_SYNC|bool else 'absent'}}"
+  tags:
+    - cronjob

ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob-configmap.yml.j2

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ NAMESPACE }}
+  name: media-metadata-sync-cronjob-configmap
+  labels:
+    app: media-metadata-sync-cronjob
+data:
+  NODE_OPTIONS: "--max-old-space-size=1536"
+  NEST_LOG_LEVEL: "info"
+  EXIT_ON_ERROR: "true"

ansible/roles/media-metadata-sync/templates/media-metadata-sync-cronjob.yml.j2

@@ -0,0 +1,97 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: media-metadata-sync-cronjob
+    app.kubernetes.io/part-of: schulcloud-verbund
+    app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+    app.kubernetes.io/name: media-metadata-sync-cronjob
+    app.kubernetes.io/component: sync
+    app.kubernetes.io/managed-by: ansible
+    git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+    git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+  name: media-metadata-sync-cronjob
+spec:
+  schedule: {{ SERVER_MEDIA_METADATA_SYNC_CRONJOB_SCHEDULE|default("30 2 * * *", true) }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: media-metadata-sync-cronjob
+            app.kubernetes.io/part-of: schulcloud-verbund
+            app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            app.kubernetes.io/name: media-metadata-sync-cronjob
+            app.kubernetes.io/component: sync
+            app.kubernetes.io/managed-by: ansible
+            git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+            git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+        spec:
+          containers:
+          - name: media-metadata-sync-cronjob
+            image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            envFrom:
+            - configMapRef:
+                name: api-configmap
+            - secretRef:
+                name: api-secret
+            - configMapRef:
+                name: media-metadata-sync-cronjob-configmap
+            command: ['/bin/sh','-c']
+            args: ['npm run nest:start:sync:media-metadata']
+            resources:
+              limits:
+                cpu: {{ MEDIA_METADATA_SYNC_CPU_LIMITS|default("2000m", true) }}
+                memory: {{ MEDIA_METADATA_SYNC_MEMORY_LIMITS|default("2Gi", true) }}
+              requests:
+                cpu: {{ MEDIA_METADATA_SYNC_CPU_REQUESTS|default("100m", true) }}
+                memory: {{ MEDIA_METADATA_SYNC_MEMORY_REQUESTS|default("150Mi", true) }}
+          restartPolicy: OnFailure
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+          affinity:
+            podAffinity:
+              preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 20
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/part-of
+                      operator: In
+                      values:
+                      - schulcloud-verbund
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+              - weight: 10
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: git.repo
+                      operator: In
+                      values:
+                      - {{ SCHULCLOUD_SERVER_REPO_NAME }}
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+              - weight: 10
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: git.branch
+                      operator: In
+                      values:
+                      - {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+              - weight: 10
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/version
+                      operator: In
+                      values:
+                      - {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+{% endif %}
+

apps/server/src/infra/sync/console/README.md

@@ -10,13 +10,14 @@ npm run nest:start:console sync run <target>
 Where `<target>` is the name of the system you want to start the synchronization for. The currently available systems are:
 - `tsp` - Synchronize Thüringer schulportal.
 - `vidis` - Synchronize Vidis Activation Data.
+- `media_metadata` - Synchronize media metadata in ctl tools in SVS with data from its media source.
 
 
 If the target is not provided, the synchronization will not start and the available targets will be displayed in an error message.
 ```bash
 {
     message: 'Either synchronization is not activated or the target entered is invalid',  
-    data: { enteredTarget: 'tsp', availableTargets: { TSP: 'tsp', VIDIS: 'vidis' }}
+    data: { enteredTarget: 'tsp', availableTargets: { TSP: 'tsp', VIDIS: 'vidis', MEDIA_METADTA: 'media_metadata' }}
 }
 ```
 

apps/server/src/infra/sync/index.ts

@@ -1,2 +1,3 @@
 export * from './tsp';
 export * from './media-licenses';
+export * from './media-metadata';

apps/server/src/infra/sync/media-metadata/index.ts

@@ -0,0 +1 @@
+export { MediaMetadataSyncStrategy } from './strategy';

apps/server/src/infra/sync/media-metadata/loggable/index.ts

@@ -0,0 +1 @@
+export { MediaMetadataSyncReportLoggable } from './media-metadata-sync-report.loggable';

apps/server/src/infra/sync/media-metadata/loggable/media-metadata-sync-report.loggable.spec.ts

@@ -0,0 +1,48 @@
+import { MediaSourceSyncOperationReport } from '@modules/media-source/domain';
+import { mediaSourceSyncReportFactory } from '@modules/media-source/testing';
+import { MediaMetadataSyncReportLoggable } from './media-metadata-sync-report.loggable';
+
+describe(MediaMetadataSyncReportLoggable.name, () => {
+	describe('getLogMessage', () => {
+		const setup = () => {
+			const report = mediaSourceSyncReportFactory.build();
+
+			const loggable = new MediaMetadataSyncReportLoggable(report);
+
+			let expectedMessage =
+				'Media metadata sync had finished\n' +
+				`Total media processed: ${report.totalCount}\n` +
+				`Total successful sync: ${report.successCount}\n` +
+				`Total failed sync: ${report.failedCount}\n` +
+				`Total undelivered media: ${report.undeliveredCount}\n`;
+
+			const operationsString = report.operations
+				.map(
+					(operation: MediaSourceSyncOperationReport): string =>
+						`${operation.operation} operation, Status: ${operation.status}, Total: ${operation.count}`
+				)
+				.join('\n');
+
+			expectedMessage += operationsString;
+
+			return {
+				loggable,
+				report,
+				expectedMessage,
+			};
+		};
+
+		it('should return the correct log message', () => {
+			const { loggable, report, expectedMessage } = setup();
+
+			const logMessage = loggable.getLogMessage();
+
+			expect(logMessage).toEqual({
+				message: expectedMessage,
+				data: {
+					report: JSON.stringify(report),
+				},
+			});
+		});
+	});
+});

apps/server/src/infra/sync/media-metadata/loggable/media-metadata-sync-report.loggable.ts

@@ -0,0 +1,40 @@
+import { ErrorLogMessage, Loggable, LogMessage, ValidationErrorLogMessage } from '@core/logger';
+import { MediaSourceSyncOperationReport, MediaSourceSyncReport } from '@modules/media-source/domain';
+
+export class MediaMetadataSyncReportLoggable implements Loggable {
+	constructor(private readonly report: MediaSourceSyncReport) {}
+
+	public getLogMessage(): LogMessage | ErrorLogMessage | ValidationErrorLogMessage {
+		const message = `Media metadata sync had finished\n${this.formatCountOverview(this.report)}${this.formatOperations(
+			this.report.operations
+		)}`;
+
+		return {
+			message,
+			data: {
+				report: JSON.stringify(this.report),
+			},
+		};
+	}
+
+	private formatCountOverview(syncReport: MediaSourceSyncReport): string {
+		const formattedString =
+			`Total media processed: ${syncReport.totalCount}\n` +
+			`Total successful sync: ${syncReport.successCount}\n` +
+			`Total failed sync: ${syncReport.failedCount}\n` +
+			`Total undelivered media: ${syncReport.undeliveredCount}\n`;
+
+		return formattedString;
+	}
+
+	private formatOperations(operations: MediaSourceSyncOperationReport[]): string {
+		const formattedString = operations
+			.map(
+				(operation: MediaSourceSyncOperationReport): string =>
+					`${operation.operation} operation, Status: ${operation.status}, Total: ${operation.count}`
+			)
+			.join('\n');
+
+		return formattedString;
+	}
+}

apps/server/src/infra/sync/media-metadata/strategy/index.ts

@@ -0,0 +1 @@
+export { MediaMetadataSyncStrategy } from './media-metadata-sync.strategy';

apps/server/src/infra/sync/media-metadata/strategy/media-metadata-sync.strategy.spec.ts

@@ -0,0 +1,74 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { Logger } from '@core/logger';
+import { createMock, DeepMocked } from '@golevelup/ts-jest';
+import { MediaSourceSyncService } from '@modules/media-source/service';
+import { mediaSourceSyncReportFactory } from '@modules/media-source/testing';
+import { SyncStrategyTarget } from '../../sync-strategy.types';
+import { MediaMetadataSyncReportLoggable } from '../loggable';
+import { MediaMetadataSyncStrategy } from './media-metadata-sync.strategy';
+
+describe(MediaMetadataSyncStrategy.name, () => {
+	let module: TestingModule;
+	let strategy: MediaMetadataSyncStrategy;
+	let syncService: DeepMocked<MediaSourceSyncService>;
+	let logger: DeepMocked<Logger>;
+
+	beforeAll(async () => {
+		module = await Test.createTestingModule({
+			providers: [
+				MediaMetadataSyncStrategy,
+				{
+					provide: MediaSourceSyncService,
+					useValue: createMock<MediaSourceSyncService>(),
+				},
+				{
+					provide: Logger,
+					useValue: createMock<Logger>(),
+				},
+			],
+		}).compile();
+
+		strategy = module.get(MediaMetadataSyncStrategy);
+		syncService = module.get(MediaSourceSyncService);
+		logger = module.get(Logger);
+	});
+
+	describe('getType', () => {
+		describe('when the method is called', () => {
+			it('should return the correct sync target', () => {
+				const result = strategy.getType();
+
+				expect(result).toEqual(SyncStrategyTarget.MEDIA_METADATA);
+			});
+		});
+	});
+
+	describe('sync', () => {
+		describe('when the method is called', () => {
+			const setup = () => {
+				const report = mediaSourceSyncReportFactory.build();
+
+				syncService.syncAllMediaMetadata.mockResolvedValue(report);
+
+				return { report };
+			};
+
+			it('should start the sync for media metadata', async () => {
+				setup();
+
+				await strategy.sync();
+
+				expect(syncService.syncAllMediaMetadata).toBeCalled();
+			});
+
+			it('should log the report after sync', async () => {
+				const { report } = setup();
+
+				await strategy.sync();
+
+				const loggable = new MediaMetadataSyncReportLoggable(report);
+				expect(logger.info).toBeCalledWith(loggable);
+			});
+		});
+	});
+});

apps/server/src/infra/sync/media-metadata/strategy/media-metadata-sync.strategy.ts

@@ -0,0 +1,28 @@
+import { Injectable } from '@nestjs/common';
+import { Logger } from '@core/logger';
+import { MediaSourceSyncService } from '@modules/media-source/service';
+import { MediaSourceSyncReport } from '@modules/media-source/domain';
+import { SyncStrategy } from '../../strategy/sync-strategy';
+import { SyncStrategyTarget } from '../../sync-strategy.types';
+import { MediaMetadataSyncReportLoggable } from '../loggable';
+
+@Injectable()
+export class MediaMetadataSyncStrategy implements SyncStrategy {
+	constructor(private readonly mediaSourceSyncService: MediaSourceSyncService, private readonly logger: Logger) {}
+
+	public getType(): SyncStrategyTarget {
+		return SyncStrategyTarget.MEDIA_METADATA;
+	}
+
+	public async sync(): Promise<void> {
+		const report: MediaSourceSyncReport = await this.mediaSourceSyncService.syncAllMediaMetadata();
+
+		this.logSyncReport(report);
+	}
+
+	private logSyncReport(report: MediaSourceSyncReport): void {
+		const loggable = new MediaMetadataSyncReportLoggable(report);
+
+		this.logger.info(loggable);
+	}
+}

apps/server/src/infra/sync/service/sync.service.ts

@@ -5,6 +5,7 @@ import { SyncStrategy } from '../strategy/sync-strategy';
 import { SyncStrategyTarget } from '../sync-strategy.types';
 import { InvalidTargetLoggable } from '../errors/invalid-target.loggable';
 import { VidisSyncStrategy } from '../media-licenses/strategy';
+import { MediaMetadataSyncStrategy } from '../media-metadata/strategy';
 
 @Injectable()
 export class SyncService {
@@ -13,11 +14,12 @@ export class SyncService {
 	constructor(
 		private readonly logger: Logger,
 		@Optional() private readonly tspSyncStrategy?: TspSyncStrategy,
-		@Optional() private readonly vidisSyncStrategy?: VidisSyncStrategy
+		@Optional() private readonly vidisSyncStrategy?: VidisSyncStrategy,
+		@Optional() private readonly mediaMetadataSyncStrategy?: MediaMetadataSyncStrategy
 	) {
 		this.logger.setContext(SyncService.name);
 		this.registerStrategy(tspSyncStrategy);
-		this.registerStrategy(vidisSyncStrategy);
+		this.registerStrategy(mediaMetadataSyncStrategy);
 	}
 
 	protected registerStrategy(strategy?: SyncStrategy) {

apps/server/src/infra/sync/sync-strategy.types.ts

@@ -1,4 +1,5 @@
 export enum SyncStrategyTarget {
 	TSP = 'tsp',
 	VIDIS = 'vidis',
+	MEDIA_METADATA = 'media-metadata',
 }