From 0d707cbba14f2e092dbd2ce1a53a9aec807da7a7 Mon Sep 17 00:00:00 2001 From: Jon Shipley Date: Sat, 8 Jun 2024 08:56:34 +0100 Subject: [PATCH] Hotfix/64795 ps report issue friday 7th june (#2808) * small refactor * Lint fix * Remove ps-report-staging-start message send function * Fold the transformer step into the ps-report-2 pupil data function to save a step * Send the staging start message from ps-report-1 rather than ps-report-2 --- .../ps-report-1-list-schools/function.json | 7 ++ .../ps-report-2-pupil-data/function.json | 4 +- .../ps-report-3-transformer/function.json | 19 ---- tslib/src/azure/service-bus-queue.names.ts | 1 + tslib/src/azure/service-bus.queue.service.ts | 16 +++- .../ps-report-1-list-schools/index.ts | 12 ++- .../ps-report-2-pupil-data/index.ts | 8 +- .../mocks/answers.ts | 2 +- .../mocks/check-config.ts | 2 +- .../mocks/check-form.ts | 2 +- .../mocks/check-incomplete.ts | 2 +- .../mocks/check.ts | 2 +- .../mocks/device.ts | 2 +- .../mocks/events.ts | 2 +- ...-complete-and-restart-available-corrupt.ts | 2 +- .../mocks/pupil-not-attending-annulled.ts | 2 +- .../mocks/pupil-not-attending-corrupt.ts | 2 +- .../mocks/pupil-not-attending.ts | 2 +- .../mocks/pupil-who-completed-a-check.ts | 2 +- .../mocks/pupil-with-incomplete-check.ts | 2 +- .../mocks/school.ts | 2 +- .../ps-report.data.service.spec.ts | 2 +- .../ps-report.data.service.ts | 2 +- .../ps-report.service.spec.ts | 9 +- .../ps-report.service.ts | 87 +++---------------- .../{models.ts => pupil-data.models.ts} | 0 .../report-line-answer.class.spec.ts | 2 +- .../report-line-answer.class.ts | 4 +- .../report-line.class.spec.ts | 4 +- .../report-line.class.ts | 4 +- .../transformer-models.ts} | 0 .../ps-report-3-transformer/index.ts | 55 ------------ .../csv-transform.spec.ts | 2 +- .../csv-transformer.ts | 2 +- .../ps-report-3b-stage-csv-file/index.ts | 2 +- .../services/redis-pupil-auth.service.spec.ts | 3 +- .../src/services/check-submit.service.spec.ts | 3 +- .../tests-integration/mock-payload.class.ts | 2 +- 38 files changed, 87 insertions(+), 191 deletions(-) delete mode 100644 func-ps-report/ps-report-3-transformer/function.json rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/answers.ts (93%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/check-config.ts (77%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/check-form.ts (70%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/check-incomplete.ts (79%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/check.ts (82%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/device.ts (70%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/events.ts (96%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-complete-and-restart-available-corrupt.ts (82%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-not-attending-annulled.ts (87%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-not-attending-corrupt.ts (83%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-not-attending.ts (83%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-who-completed-a-check.ts (82%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/pupil-with-incomplete-check.ts (82%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/mocks/school.ts (67%) rename tslib/src/functions-ps-report/ps-report-2-pupil-data/{models.ts => pupil-data.models.ts} (100%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/report-line-answer.class.spec.ts (99%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/report-line-answer.class.ts (98%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/report-line.class.spec.ts (99%) rename tslib/src/functions-ps-report/{ps-report-3-transformer => ps-report-2-pupil-data}/report-line.class.ts (99%) rename tslib/src/functions-ps-report/{ps-report-3-transformer/models.ts => ps-report-2-pupil-data/transformer-models.ts} (100%) delete mode 100644 tslib/src/functions-ps-report/ps-report-3-transformer/index.ts diff --git a/func-ps-report/ps-report-1-list-schools/function.json b/func-ps-report/ps-report-1-list-schools/function.json index 39a5ae7b3a..df22cf784b 100644 --- a/func-ps-report/ps-report-1-list-schools/function.json +++ b/func-ps-report/ps-report-1-list-schools/function.json @@ -13,6 +13,13 @@ "name": "schoolMessages", "queueName": "ps-report-schools", "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" + }, + { + "direction": "out", + "type": "serviceBus", + "name": "stagingStart", + "queueName": "ps-report-staging-start", + "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" } ], "scriptFile": "../dist/functions-ps-report/ps-report-1-list-schools/index.js" diff --git a/func-ps-report/ps-report-2-pupil-data/function.json b/func-ps-report/ps-report-2-pupil-data/function.json index 98fdd48983..2492edeabc 100644 --- a/func-ps-report/ps-report-2-pupil-data/function.json +++ b/func-ps-report/ps-report-2-pupil-data/function.json @@ -8,10 +8,10 @@ "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" }, { - "name": "psReportPupilMessage", + "name": "psReportExportOutput", "type": "serviceBus", "direction": "out", - "queueName": "ps-report-staging", + "queueName": "ps-report-export", "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" } ], diff --git a/func-ps-report/ps-report-3-transformer/function.json b/func-ps-report/ps-report-3-transformer/function.json deleted file mode 100644 index ad8e933097..0000000000 --- a/func-ps-report/ps-report-3-transformer/function.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bindings": [ - { - "direction": "in", - "type": "serviceBusTrigger", - "name": "inputData", - "queueName": "ps-report-staging", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - }, - { - "direction": "out", - "type": "serviceBus", - "name": "outputData", - "queueName": "ps-report-export", - "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING" - } - ], - "scriptFile": "../dist/functions-ps-report/ps-report-3-transformer/index.js" -} diff --git a/tslib/src/azure/service-bus-queue.names.ts b/tslib/src/azure/service-bus-queue.names.ts index 8714948f33..4ccb53c6af 100644 --- a/tslib/src/azure/service-bus-queue.names.ts +++ b/tslib/src/azure/service-bus-queue.names.ts @@ -7,6 +7,7 @@ export enum ServiceBusQueueName { checkValidation = 'check-validation', psReportSchools = 'ps-report-schools', psReportStaging = 'ps-report-staging', + psReportStagingStart = 'ps-report-staging-start', psReportExport = 'ps-report-export', pupilLogin = 'pupil-login', queueReplay = 'queue-replay', diff --git a/tslib/src/azure/service-bus.queue.service.ts b/tslib/src/azure/service-bus.queue.service.ts index 8c3dff67cb..1a28d05075 100644 --- a/tslib/src/azure/service-bus.queue.service.ts +++ b/tslib/src/azure/service-bus.queue.service.ts @@ -1,23 +1,28 @@ -import { ServiceBusClient } from '@azure/service-bus' import config from '../pupil-api/config' import { type ServiceBusQueueName } from './service-bus-queue.names' +import { ServiceBusAdministrationClient, ServiceBusClient } from '@azure/service-bus' export interface IServiceBusQueueService { dispatch (message: IServiceBusQueueMessage, queueName: string): Promise + getActiveMessageCount (queueName: ServiceBusQueueName): Promise } export interface IServiceBusQueueMessage { body: any + messageId?: string + contentType?: string } export class ServiceBusQueueService implements IServiceBusQueueService { private readonly serviceBusClient: ServiceBusClient + private readonly serviceBusAdministrationClient: ServiceBusAdministrationClient constructor () { if (config.ServiceBus.connectionString === undefined) { throw new Error('Azure Service Bus Connection String missing') } this.serviceBusClient = new ServiceBusClient(config.ServiceBus.connectionString) + this.serviceBusAdministrationClient = new ServiceBusAdministrationClient(config.ServiceBus.connectionString) } async dispatch (message: IServiceBusQueueMessage, queueName: ServiceBusQueueName): Promise { @@ -25,4 +30,13 @@ export class ServiceBusQueueService implements IServiceBusQueueService { await sender.sendMessages(message) return sender.close() } + + async getActiveMessageCount (queueName: ServiceBusQueueName): Promise { + const queueRuntimeProperties = await this.serviceBusAdministrationClient.getQueueRuntimeProperties(queueName.toString()) + const msgCount = queueRuntimeProperties.activeMessageCount + if (msgCount === undefined || msgCount === null) { + return -1 + } + return msgCount + } } diff --git a/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts b/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts index 10492254a4..1259967218 100644 --- a/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts +++ b/tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts @@ -6,7 +6,7 @@ import { PsReportSource } from '../common/ps-report-log-entry' import { JobDataService } from '../../services/data/job.data.service' import { JobStatusCode } from '../../common/job-status-code' import moment from 'moment' -import { type PsReportListSchoolsIncomingMessage } from '../common/ps-report-service-bus-messages' +import type { PsReportStagingStartMessage, PsReportListSchoolsIncomingMessage } from '../common/ps-report-service-bus-messages' const serviceBusTrigger: AzureFunction = async function (context: Context, jobInfo: PsReportListSchoolsIncomingMessage): Promise { const logger = new PsReportLogger(context, PsReportSource.SchoolGenerator) @@ -28,6 +28,16 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn const messages = await schoolListService.getSchoolMessages(messageSpec) context.bindings.schoolMessages = messages meta.processCount = messages.length + + // Send a message to start ps-report-3b-staging (the csv assembly) + // service-bus queue = stagingStart + const stagingStartMessage: PsReportStagingStartMessage = { + startTime: new Date(), + jobUuid: jobInfo.jobUuid, + filename + } + context.bindings.stagingStart = stagingStartMessage + logger.info(`staging-start message sent: ${JSON.stringify(stagingStartMessage)}`) } catch (error) { let errorMessage = 'unknown error' if (error instanceof Error) { diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts index a6bc5cbbf0..95dd02f710 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts @@ -1,11 +1,11 @@ import { type AzureFunction, type Context } from '@azure/functions' import { performance } from 'perf_hooks' import { PsReportService } from './ps-report.service' -import { type PupilResult } from './models' import { PsReportLogger } from '../common/ps-report-logger' import { PsReportSource } from '../common/ps-report-log-entry' import type { PsReportSchoolFanOutMessage } from '../common/ps-report-service-bus-messages' import config from '../../config' +import type { IPsychometricReportLine } from './transformer-models' /** * Incoming message is just the name and UUID of the school to process @@ -14,7 +14,7 @@ import config from '../../config' */ export interface IOutputBinding { - psReportPupilMessage: PupilResult[] + psReportExportOutput: IPsychometricReportLine[] } const serviceBusQueueTrigger: AzureFunction = async function (context: Context, incomingMessage: PsReportSchoolFanOutMessage): Promise { @@ -23,14 +23,14 @@ const serviceBusQueueTrigger: AzureFunction = async function (context: Context, if (config.Logging.DebugVerbosity > 1) { logger.verbose(`called for school ${incomingMessage.name}`) } - const outputBinding: IOutputBinding = { psReportPupilMessage: [] } + const outputBinding: IOutputBinding = { psReportExportOutput: [] } context.bindings = outputBinding const psReportService = new PsReportService(outputBinding, logger) await psReportService.process(incomingMessage) const end = performance.now() const durationInMilliseconds = end - start if (config.Logging.DebugVerbosity > 1) { - logger.info(`processed ${outputBinding.psReportPupilMessage.length} pupils, run took ${durationInMilliseconds} ms`) + logger.info(`processed ${outputBinding.psReportExportOutput.length} pupils, run took ${durationInMilliseconds} ms`) } } diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/answers.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/answers.ts similarity index 93% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/answers.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/answers.ts index 6600249fba..562933d53c 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/answers.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/answers.ts @@ -1,5 +1,5 @@ import moment from 'moment' -import { type Answer } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Answer } from '../pupil-data.models' export const answers: readonly Answer[] = [ { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-config.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-config.ts similarity index 77% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-config.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-config.ts index 3309d81dc9..925d113871 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-config.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-config.ts @@ -1,4 +1,4 @@ -import { type CheckConfig } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type CheckConfig } from '../pupil-data.models' export const checkConfig: CheckConfig = { audibleSounds: false, diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-form.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-form.ts similarity index 70% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-form.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-form.ts index 4554a0f173..41aeb33a71 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-form.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-form.ts @@ -1,4 +1,4 @@ -import { type CheckForm } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type CheckForm } from '../pupil-data.models' export const checkForm: CheckForm = { id: 9, diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-incomplete.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-incomplete.ts similarity index 79% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-incomplete.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-incomplete.ts index 7e9cdd9082..c3250b27a9 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-incomplete.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-incomplete.ts @@ -1,4 +1,4 @@ -import { type Check } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Check } from '../pupil-data.models' export const check: Check = { id: 31, diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check.ts similarity index 82% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check.ts index 11bdc7155c..5312bc52ce 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check.ts @@ -1,4 +1,4 @@ -import { type Check } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Check } from '../pupil-data.models' import moment from 'moment' export const check: Check = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/device.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/device.ts similarity index 70% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/device.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/device.ts index b9193c00a3..0bf890e68b 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/device.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/device.ts @@ -1,4 +1,4 @@ -import { type Device } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Device } from '../pupil-data.models' export const device: Device = { type: 'Tablet', diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/events.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/events.ts similarity index 96% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/events.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/events.ts index d60ef5c058..cf2c0e5009 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/events.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/events.ts @@ -1,4 +1,4 @@ -import type { Event } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import type { Event } from '../pupil-data.models' import moment from 'moment' export const events: Event[] = [ diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-complete-and-restart-available-corrupt.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-complete-and-restart-available-corrupt.ts similarity index 82% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-complete-and-restart-available-corrupt.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-complete-and-restart-available-corrupt.ts index 0b1cc7244b..0b54d2f87f 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-complete-and-restart-available-corrupt.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-complete-and-restart-available-corrupt.ts @@ -1,4 +1,4 @@ -import type { Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import type { Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-annulled.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-annulled.ts similarity index 87% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-annulled.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-annulled.ts index c5b974e795..168779f3ba 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-annulled.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-annulled.ts @@ -1,4 +1,4 @@ -import type { Pupil } from '../../ps-report-2-pupil-data/models' +import type { Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-corrupt.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-corrupt.ts similarity index 83% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-corrupt.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-corrupt.ts index d8c2345ca1..73a16586fa 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-corrupt.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-corrupt.ts @@ -1,4 +1,4 @@ -import type { Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import type { Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending.ts similarity index 83% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending.ts index 1d0d25d927..a622b0b38c 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending.ts @@ -1,4 +1,4 @@ -import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-who-completed-a-check.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-who-completed-a-check.ts similarity index 82% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-who-completed-a-check.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-who-completed-a-check.ts index ef0e70e10b..772909d530 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-who-completed-a-check.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-who-completed-a-check.ts @@ -1,4 +1,4 @@ -import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-with-incomplete-check.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-with-incomplete-check.ts similarity index 82% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-with-incomplete-check.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-with-incomplete-check.ts index 09b1c3da66..15823f9c51 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-with-incomplete-check.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-with-incomplete-check.ts @@ -1,4 +1,4 @@ -import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Pupil } from '../pupil-data.models' import moment from 'moment' export const pupil: Pupil = { diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/school.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/school.ts similarity index 67% rename from tslib/src/functions-ps-report/ps-report-3-transformer/mocks/school.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/school.ts index 9d622969d0..bd707069eb 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/mocks/school.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/school.ts @@ -1,4 +1,4 @@ -import { type School } from '../../../functions-ps-report/ps-report-2-pupil-data/models' +import { type School } from '../pupil-data.models' export const school: School = { estabCode: 999, diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.spec.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.spec.ts index 2df9991594..608ec464c7 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.spec.ts @@ -2,7 +2,7 @@ import { PsReportDataService } from './ps-report.data.service' import { MockLogger } from '../../common/logger' import type { ISqlService } from '../../sql/sql.service' import moment from 'moment' -import type { Pupil, School } from './models' +import type { Pupil, School } from './pupil-data.models' describe('ps-report.data.service', () => { let sut: PsReportDataService diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.ts index f1d9decd79..ff5fa5832b 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.ts @@ -18,7 +18,7 @@ import { type Pupil, type PupilResult, type RestartReasonCode, type School -} from './models' +} from './pupil-data.models' import * as R from 'ramda' import * as RA from 'ramda-adjunct' import type moment from 'moment' diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts index 04118a4479..27722382f6 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts @@ -8,9 +8,8 @@ describe.skip('PsReportService', () => { let sut: PsReportService let logger: ILogger let psReportDataService: IPsReportDataService - const serviceBusAdministrationClient: any = {} const schoolUuid = 'AAAA-BBBB-CCCC-DDDD' - const outputBindings: IOutputBinding = { psReportPupilMessage: [] } + const outputBindings: IOutputBinding = { psReportExportOutput: [] } const mockPupils = [{ id: 1, schoolId: 99 }, { id: 2, schoolId: 99 }, { id: 3, schoolId: 99 }] const mockSchool = { id: 99, name: 'test school' } const psReportSchoolFanOutMessage: PsReportSchoolFanOutMessage = { @@ -28,9 +27,9 @@ describe.skip('PsReportService', () => { getPupils: jest.fn(), getSchool: jest.fn() } - outputBindings.psReportPupilMessage = [] + outputBindings.psReportExportOutput = [] - sut = new PsReportService(outputBindings, logger, psReportDataService, serviceBusAdministrationClient) + sut = new PsReportService(outputBindings, logger, psReportDataService) }) test('it is defined', () => { @@ -71,6 +70,6 @@ describe.skip('PsReportService', () => { .mockResolvedValueOnce({ data: 2 }) .mockResolvedValueOnce({ data: 3 }) await sut.process(psReportSchoolFanOutMessage) - expect(outputBindings.psReportPupilMessage).toHaveLength(3) + expect(outputBindings.psReportExportOutput).toHaveLength(3) }) }) diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts index b1de18fbd7..a63d2d9caf 100644 --- a/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.ts @@ -1,28 +1,22 @@ import { type IPsReportDataService, PsReportDataService } from './ps-report.data.service' -import { type Pupil, type PupilResult, type School } from './models' +import { type Pupil, type PupilResult, type School } from './pupil-data.models' import { type ILogger } from '../../common/logger' import { type IOutputBinding } from '.' -import type { PsReportSchoolFanOutMessage, PsReportStagingStartMessage } from '../common/ps-report-service-bus-messages' -import { ServiceBusAdministrationClient, ServiceBusClient, type ServiceBusMessage } from '@azure/service-bus' +import type { PsReportSchoolFanOutMessage } from '../common/ps-report-service-bus-messages' import config from '../../config' +import { ReportLine } from './report-line.class' const logName = 'ps-report-2-pupil-data: PsReportService' -const outputQueueName = 'ps-report-staging-start' export class PsReportService { private readonly dataService: IPsReportDataService private readonly outputBinding: IOutputBinding private readonly logger: ILogger - private readonly serviceBusAdministrationClient: ServiceBusAdministrationClient - constructor (outputBinding: IOutputBinding, logger: ILogger, dataService?: IPsReportDataService, serviceBusAdminClient?: ServiceBusAdministrationClient) { + constructor (outputBinding: IOutputBinding, logger: ILogger, dataService?: IPsReportDataService) { this.outputBinding = outputBinding this.logger = logger this.dataService = dataService ?? new PsReportDataService(this.logger) - if (config.ServiceBus.ConnectionString === undefined || config.ServiceBus.ConnectionString === null) { - throw new Error('Unable to connect to service bus. Please check the config.') - } - this.serviceBusAdministrationClient = serviceBusAdminClient ?? new ServiceBusAdministrationClient(config.ServiceBus.ConnectionString) } async process (incomingMessage: PsReportSchoolFanOutMessage): Promise { @@ -52,7 +46,7 @@ export class PsReportService { } try { const result: PupilResult = await this.dataService.getPupilData(pupil, school) - const output: PupilResult = { + const pupilResult: PupilResult = { answers: result.answers, check: result.check, checkConfig: result.checkConfig, @@ -62,75 +56,18 @@ export class PsReportService { pupil: result.pupil, school: result.school } - this.outputBinding.psReportPupilMessage.push(output) + + // Now we have the pupil data we can transform it into the report format + const reportLine = new ReportLine(pupilResult.answers, pupilResult.check, pupilResult.checkConfig, pupilResult.checkForm, pupilResult.device, pupilResult.events, pupilResult.pupil, pupilResult.school) + const outputData = reportLine.transform() + + // Send the transformed pupil data onto the ps-report-export queue using the output bindings. + this.outputBinding.psReportExportOutput.push(outputData) } catch (error: any) { // Ignore the error on the particular pupil and carry on so it reports on the rest of the school this.logger.error(`${logName}: ERROR: Failed to retrieve pupil data for pupil ${pupil.slug} in school ${incomingMessage.uuid} Error was ${error.message}`) } } - - const shouldStartStaging = await this.shouldStartStaging(incomingMessage) - if (shouldStartStaging) { - this.logger.verbose(`${logName}: sending staging start message`) - // send a message to the ps-report-3b-staging function to start up and start creating the csv file in blob storage. - const msg: PsReportStagingStartMessage = { - startTime: new Date(), - jobUuid: incomingMessage.jobUuid, - filename: incomingMessage.filename - } - await this.sendStagingStartMessage(msg) - } - } - - /** - * Determine if the staging start message should be sent to the `ps-report-3-staging` function - * which is listening for message on the sb queue `ps-report-staging-start` - * - * By default it will send the message when there is 1 remaining message. - * - * - * Similar logic works for the test environment where ps reports will be generated for a single school. - * - * The ps-report-3-staging function will then start assembling the CSV file for bulk upload. - * - * TODO: refactror the below two function into somewhere suitable for data services. - */ - private async shouldStartStaging (incomingMessage: PsReportSchoolFanOutMessage): Promise { - // See how many message are left on the "schools" sb queue - const inputQueueName = 'ps-report-schools' - const queueRuntimeProperties = await this.serviceBusAdministrationClient.getQueueRuntimeProperties(inputQueueName) - const msgCount = queueRuntimeProperties.activeMessageCount - if (msgCount === undefined || msgCount === null) { - return false - } - - // Look for the last message on the queue. This _could_ also match the first message on the queue if a school had a single pupil in Y4. - if (msgCount <= 1) { - this.logger.verbose(`shouldStartStaging() returning true as there are ${msgCount} messages left from a total of ${incomingMessage.totalNumberOfSchools}`) - return true - } - - return false - } - - private async sendStagingStartMessage (msg: PsReportStagingStartMessage): Promise { - if (config.ServiceBus.ConnectionString === undefined) { - throw new Error('Can\'t connect to service bus. Missing ConnectionString.') - } - const sbClient = new ServiceBusClient(config.ServiceBus.ConnectionString) - const sender = sbClient.createSender(outputQueueName) - try { - const message: ServiceBusMessage = { - body: msg, - messageId: msg.jobUuid, // for duplicate detection - contentType: 'application/json' - } - await sender.sendMessages(message) - await sender.close() - await sbClient.close() - } finally { - await sbClient.close() - } } } diff --git a/tslib/src/functions-ps-report/ps-report-2-pupil-data/models.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/pupil-data.models.ts similarity index 100% rename from tslib/src/functions-ps-report/ps-report-2-pupil-data/models.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/pupil-data.models.ts diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.spec.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.spec.ts similarity index 99% rename from tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.spec.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.spec.ts index 056bc10db3..4ceb1fa722 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.spec.ts @@ -1,4 +1,4 @@ -import { type Input } from '../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Input } from './pupil-data.models' import moment from 'moment/moment' import { ReportLineAnswer } from './report-line-answer.class' diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.ts similarity index 98% rename from tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.ts index 6825ac1a1a..267bbdc34c 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line-answer.class.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line-answer.class.ts @@ -1,6 +1,6 @@ -import { type Input } from '../../functions-ps-report/ps-report-2-pupil-data/models' +import { type Input } from './pupil-data.models' import * as R from 'ramda' -import { type IReportLineAnswer } from './models' +import { type IReportLineAnswer } from './transformer-models' export class ReportLineAnswer { private _questionNumber: number | null = null diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.spec.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.spec.ts similarity index 99% rename from tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.spec.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.spec.ts index 045aeedc76..1e1bb14c76 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.spec.ts @@ -13,8 +13,8 @@ import { checkConfig } from './mocks/check-config' import { checkForm } from './mocks/check-form' import { device } from './mocks/device' import { events } from './mocks/events' -import { type NotTakingCheckCode, type RestartReasonCode } from '../../functions-ps-report/ps-report-2-pupil-data/models' -import { type DfEAbsenceCode } from './models' +import { type NotTakingCheckCode, type RestartReasonCode } from './pupil-data.models' +import { type DfEAbsenceCode } from './transformer-models' class ReportLineTest extends ReportLine { public getReasonNotTakingCheck (code: NotTakingCheckCode): string diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.ts similarity index 99% rename from tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.ts index b1d010cea1..3841cb6fb6 100644 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/report-line.class.ts +++ b/tslib/src/functions-ps-report/ps-report-2-pupil-data/report-line.class.ts @@ -7,10 +7,10 @@ import { type Event, type Pupil, type School, type Answer, type NotTakingCheckCode, type RestartReasonCode -} from '../../functions-ps-report/ps-report-2-pupil-data/models' +} from './pupil-data.models' import { deepFreeze } from '../../common/deep-freeze' import { ReportLineAnswer } from './report-line-answer.class' -import { type DfEAbsenceCode, type IPsychometricReportLine, type WorkingReportLine } from './models' +import { type DfEAbsenceCode, type IPsychometricReportLine, type WorkingReportLine } from './transformer-models' export class ReportLine { private readonly _answers: AnswersOrNull diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/models.ts b/tslib/src/functions-ps-report/ps-report-2-pupil-data/transformer-models.ts similarity index 100% rename from tslib/src/functions-ps-report/ps-report-3-transformer/models.ts rename to tslib/src/functions-ps-report/ps-report-2-pupil-data/transformer-models.ts diff --git a/tslib/src/functions-ps-report/ps-report-3-transformer/index.ts b/tslib/src/functions-ps-report/ps-report-3-transformer/index.ts deleted file mode 100644 index 075eaa1c91..0000000000 --- a/tslib/src/functions-ps-report/ps-report-3-transformer/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { type AzureFunction, type Context } from '@azure/functions' -import { ReportLine } from './report-line.class' -import { jsonReviver } from '../../common/json-reviver' -import { type PupilResult } from '../../functions-ps-report/ps-report-2-pupil-data/models' -import { PsReportLogger } from '../common/ps-report-logger' -import { PsReportSource } from '../common/ps-report-log-entry' - -/** - * This functions receives a message from a sb queue and outputs a message to a sb queue that will later be written - * as one line of the psychometric report. - * - * Input: service bus queue message containing result details for a single pupil - * Output: service bus queue message containing a single line of the psychometric report in JSON format - * - * @param context - * @param pupilResult - */ - -const serviceBusQueueTrigger: AzureFunction = async function (context: Context, inputData: PupilResult): Promise { - const logger = new PsReportLogger(context, PsReportSource.Transformer) - try { - /** - * The inputData type is not absolutely correctly typed. The Moment datetime's are still strings as the JSON parsing happens - * outside of this context, so we can't pass in a custom reviver function. - * - * We could access context.bindings.inputData (and parse that rather than the function parameter) but this leads to some brittleness - * as the parameter name may change. - * - * The upshot is that the below `revive` function does what we need, and we can use the correctly typed pupilResult const that is - * returned. - */ - const pupilResult: PupilResult = revive(inputData) - const { answers, check, checkConfig, checkForm, device, events, pupil, school } = pupilResult - const reportLine = new ReportLine(answers, check, checkConfig, checkForm, device, events, pupil, school) - const outputData = reportLine.transform() - context.bindings.outputData = outputData - } catch (error: any) { - let errorMessage = 'unknown error' - if (error instanceof Error) { - errorMessage = error.message - } - logger.error(`ERROR: ${errorMessage}`) - throw error - } -} - -/** - * JSON reviver for Date instantiation - * @param pupilResult - */ -function revive (pupilResult: PupilResult): PupilResult { - return JSON.parse(JSON.stringify(pupilResult), jsonReviver) as PupilResult -} - -export default serviceBusQueueTrigger diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transform.spec.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transform.spec.ts index 1ca373bedf..ce643fadca 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transform.spec.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transform.spec.ts @@ -2,7 +2,7 @@ import moment from 'moment' import { CsvTransformer } from './csv-transformer' import * as CSV from 'csv-string' import { ConsoleLogger } from '../../common/logger' -import { type IPsychometricReportLine } from '../ps-report-3-transformer/models' +import { type IPsychometricReportLine } from '../ps-report-2-pupil-data/transformer-models' let sut: CsvTransformer diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts index 568b03bdae..a0b03b43fe 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/csv-transformer.ts @@ -1,4 +1,4 @@ -import type { IReportLineAnswer, IPsychometricReportLine } from '../ps-report-3-transformer/models' +import type { IReportLineAnswer, IPsychometricReportLine } from '../ps-report-2-pupil-data/transformer-models' import * as CSV from 'csv-string' import moment from 'moment' import { type ILogger } from '../../common/logger' diff --git a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts index dff8464549..8ad9770abc 100644 --- a/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts +++ b/tslib/src/functions-ps-report/ps-report-3b-stage-csv-file/index.ts @@ -3,7 +3,7 @@ import { performance } from 'perf_hooks' import * as sb from '@azure/service-bus' import config from '../../config' import * as RA from 'ramda-adjunct' -import { type IPsychometricReportLine } from '../ps-report-3-transformer/models' +import { type IPsychometricReportLine } from '../ps-report-2-pupil-data/transformer-models' import { jsonReviver } from '../../common/json-reviver' import { PsReportStagingDataService } from './ps-report-staging.data.service' import { CsvTransformer } from './csv-transformer' diff --git a/tslib/src/pupil-api/services/redis-pupil-auth.service.spec.ts b/tslib/src/pupil-api/services/redis-pupil-auth.service.spec.ts index 40f935e74c..8226b34560 100644 --- a/tslib/src/pupil-api/services/redis-pupil-auth.service.spec.ts +++ b/tslib/src/pupil-api/services/redis-pupil-auth.service.spec.ts @@ -18,7 +18,8 @@ const RedisServiceMock = jest.fn(() => ({ })) const MessageDispatchMock = jest.fn(() => ({ - dispatch: jest.fn() + dispatch: jest.fn(), + getActiveMessageCount: jest.fn() })) describe('redis-pupil-auth.service', () => { diff --git a/tslib/src/services/check-submit.service.spec.ts b/tslib/src/services/check-submit.service.spec.ts index d3eef49f79..f50f0c2083 100644 --- a/tslib/src/services/check-submit.service.spec.ts +++ b/tslib/src/services/check-submit.service.spec.ts @@ -3,7 +3,8 @@ import { type IServiceBusQueueService } from '../azure/service-bus.queue.service import { ServiceBusQueueName } from '../azure/service-bus-queue.names' const SbQueueServiceMock = jest.fn(() => ({ - dispatch: jest.fn() + dispatch: jest.fn(), + getActiveMessageCount: jest.fn() })) let sut: CheckSubmitService diff --git a/tslib/src/tests-integration/mock-payload.class.ts b/tslib/src/tests-integration/mock-payload.class.ts index db8439d95d..8d8440112e 100644 --- a/tslib/src/tests-integration/mock-payload.class.ts +++ b/tslib/src/tests-integration/mock-payload.class.ts @@ -1,6 +1,6 @@ import moment from 'moment' import { faker } from '@faker-js/faker' -import type { DfEAbsenceCode, IPsychometricReportLine, IReportLineAnswer } from '../functions-ps-report/ps-report-3-transformer/models' +import type { DfEAbsenceCode, IPsychometricReportLine, IReportLineAnswer } from '../functions-ps-report/ps-report-2-pupil-data/transformer-models' const schools = [ 'The New Learning Centre', 'Shawcroft School',