Hotfix/64795 ps report issue friday 7th june (#2808)

* small refactor

* Lint fix

* Remove ps-report-staging-start message send function

* Fold the transformer step into the ps-report-2 pupil data function to save a step

* Send the staging start message from ps-report-1 rather than ps-report-2

func-ps-report/ps-report-1-list-schools/function.json

@@ -13,6 +13,13 @@
       "name": "schoolMessages",
       "queueName": "ps-report-schools",
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
+    },
+    {
+      "direction": "out",
+      "type": "serviceBus",
+      "name": "stagingStart",
+      "queueName": "ps-report-staging-start",
+      "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],
   "scriptFile": "../dist/functions-ps-report/ps-report-1-list-schools/index.js"

func-ps-report/ps-report-2-pupil-data/function.json

@@ -8,10 +8,10 @@
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     },
     {
-      "name": "psReportPupilMessage",
+      "name": "psReportExportOutput",
       "type": "serviceBus",
       "direction": "out",
-      "queueName": "ps-report-staging",
+      "queueName": "ps-report-export",
       "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
     }
   ],

tslib/src/azure/service-bus-queue.names.ts

@@ -7,6 +7,7 @@ export enum ServiceBusQueueName {
   checkValidation = 'check-validation',
   psReportSchools = 'ps-report-schools',
   psReportStaging = 'ps-report-staging',
+  psReportStagingStart = 'ps-report-staging-start',
   psReportExport = 'ps-report-export',
   pupilLogin = 'pupil-login',
   queueReplay = 'queue-replay',

tslib/src/azure/service-bus.queue.service.ts

@@ -1,28 +1,42 @@
-import { ServiceBusClient } from '@azure/service-bus'
 import config from '../pupil-api/config'
 import { type ServiceBusQueueName } from './service-bus-queue.names'
+import { ServiceBusAdministrationClient, ServiceBusClient } from '@azure/service-bus'
 
 export interface IServiceBusQueueService {
   dispatch (message: IServiceBusQueueMessage, queueName: string): Promise<void>
+  getActiveMessageCount (queueName: ServiceBusQueueName): Promise<number>
 }
 
 export interface IServiceBusQueueMessage {
   body: any
+  messageId?: string
+  contentType?: string
 }
 
 export class ServiceBusQueueService implements IServiceBusQueueService {
   private readonly serviceBusClient: ServiceBusClient
+  private readonly serviceBusAdministrationClient: ServiceBusAdministrationClient
 
   constructor () {
     if (config.ServiceBus.connectionString === undefined) {
       throw new Error('Azure Service Bus Connection String missing')
     }
     this.serviceBusClient = new ServiceBusClient(config.ServiceBus.connectionString)
+    this.serviceBusAdministrationClient = new ServiceBusAdministrationClient(config.ServiceBus.connectionString)
   }
 
   async dispatch (message: IServiceBusQueueMessage, queueName: ServiceBusQueueName): Promise<void> {
     const sender = this.serviceBusClient.createSender(queueName.toString())
     await sender.sendMessages(message)
     return sender.close()
   }
+
+  async getActiveMessageCount (queueName: ServiceBusQueueName): Promise<number> {
+    const queueRuntimeProperties = await this.serviceBusAdministrationClient.getQueueRuntimeProperties(queueName.toString())
+    const msgCount = queueRuntimeProperties.activeMessageCount
+    if (msgCount === undefined || msgCount === null) {
+      return -1
+    }
+    return msgCount
+  }
 }

tslib/src/functions-ps-report/ps-report-1-list-schools/index.ts

@@ -6,7 +6,7 @@ import { PsReportSource } from '../common/ps-report-log-entry'
 import { JobDataService } from '../../services/data/job.data.service'
 import { JobStatusCode } from '../../common/job-status-code'
 import moment from 'moment'
-import { type PsReportListSchoolsIncomingMessage } from '../common/ps-report-service-bus-messages'
+import type { PsReportStagingStartMessage, PsReportListSchoolsIncomingMessage } from '../common/ps-report-service-bus-messages'
 
 const serviceBusTrigger: AzureFunction = async function (context: Context, jobInfo: PsReportListSchoolsIncomingMessage): Promise<void> {
   const logger = new PsReportLogger(context, PsReportSource.SchoolGenerator)
@@ -28,6 +28,16 @@ const serviceBusTrigger: AzureFunction = async function (context: Context, jobIn
     const messages = await schoolListService.getSchoolMessages(messageSpec)
     context.bindings.schoolMessages = messages
     meta.processCount = messages.length
+
+    // Send a message to start ps-report-3b-staging (the csv assembly)
+    // service-bus queue = stagingStart
+    const stagingStartMessage: PsReportStagingStartMessage = {
+      startTime: new Date(),
+      jobUuid: jobInfo.jobUuid,
+      filename
+    }
+    context.bindings.stagingStart = stagingStartMessage
+    logger.info(`staging-start message sent: ${JSON.stringify(stagingStartMessage)}`)
   } catch (error) {
     let errorMessage = 'unknown error'
     if (error instanceof Error) {

tslib/src/functions-ps-report/ps-report-2-pupil-data/index.ts

@@ -1,11 +1,11 @@
 import { type AzureFunction, type Context } from '@azure/functions'
 import { performance } from 'perf_hooks'
 import { PsReportService } from './ps-report.service'
-import { type PupilResult } from './models'
 import { PsReportLogger } from '../common/ps-report-logger'
 import { PsReportSource } from '../common/ps-report-log-entry'
 import type { PsReportSchoolFanOutMessage } from '../common/ps-report-service-bus-messages'
 import config from '../../config'
+import type { IPsychometricReportLine } from './transformer-models'
 
 /**
  * Incoming message is just the name and UUID of the school to process
@@ -14,7 +14,7 @@ import config from '../../config'
  */
 
 export interface IOutputBinding {
-  psReportPupilMessage: PupilResult[]
+  psReportExportOutput: IPsychometricReportLine[]
 }
 
 const serviceBusQueueTrigger: AzureFunction = async function (context: Context, incomingMessage: PsReportSchoolFanOutMessage): Promise<void> {
@@ -23,14 +23,14 @@ const serviceBusQueueTrigger: AzureFunction = async function (context: Context,
   if (config.Logging.DebugVerbosity > 1) {
     logger.verbose(`called for school ${incomingMessage.name}`)
   }
-  const outputBinding: IOutputBinding = { psReportPupilMessage: [] }
+  const outputBinding: IOutputBinding = { psReportExportOutput: [] }
   context.bindings = outputBinding
   const psReportService = new PsReportService(outputBinding, logger)
   await psReportService.process(incomingMessage)
   const end = performance.now()
   const durationInMilliseconds = end - start
   if (config.Logging.DebugVerbosity > 1) {
-    logger.info(`processed ${outputBinding.psReportPupilMessage.length} pupils, run took ${durationInMilliseconds} ms`)
+    logger.info(`processed ${outputBinding.psReportExportOutput.length} pupils, run took ${durationInMilliseconds} ms`)
   }
 }
 

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/answers.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/answers.ts

@@ -1,5 +1,5 @@
 import moment from 'moment'
-import { type Answer } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Answer } from '../pupil-data.models'
 
 export const answers: readonly Answer[] = [
   {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-config.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-config.ts

@@ -1,4 +1,4 @@
-import { type CheckConfig } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type CheckConfig } from '../pupil-data.models'
 
 export const checkConfig: CheckConfig = {
   audibleSounds: false,

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-form.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-form.ts

@@ -1,4 +1,4 @@
-import { type CheckForm } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type CheckForm } from '../pupil-data.models'
 
 export const checkForm: CheckForm = {
   id: 9,

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check-incomplete.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check-incomplete.ts

@@ -1,4 +1,4 @@
-import { type Check } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Check } from '../pupil-data.models'
 
 export const check: Check = {
   id: 31,

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/check.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/check.ts

@@ -1,4 +1,4 @@
-import { type Check } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Check } from '../pupil-data.models'
 import moment from 'moment'
 
 export const check: Check = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/device.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/device.ts

@@ -1,4 +1,4 @@
-import { type Device } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Device } from '../pupil-data.models'
 
 export const device: Device = {
   type: 'Tablet',

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/events.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/events.ts

@@ -1,4 +1,4 @@
-import type { Event } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import type { Event } from '../pupil-data.models'
 import moment from 'moment'
 
 export const events: Event[] = [

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-complete-and-restart-available-corrupt.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-complete-and-restart-available-corrupt.ts

@@ -1,4 +1,4 @@
-import type { Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import type { Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-annulled.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-annulled.ts

@@ -1,4 +1,4 @@
-import type { Pupil } from '../../ps-report-2-pupil-data/models'
+import type { Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending-corrupt.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending-corrupt.ts

@@ -1,4 +1,4 @@
-import type { Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import type { Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-not-attending.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-not-attending.ts

@@ -1,4 +1,4 @@
-import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-who-completed-a-check.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-who-completed-a-check.ts

@@ -1,4 +1,4 @@
-import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/pupil-with-incomplete-check.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/pupil-with-incomplete-check.ts

@@ -1,4 +1,4 @@
-import { type Pupil } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type Pupil } from '../pupil-data.models'
 import moment from 'moment'
 
 export const pupil: Pupil = {

tslib/src/functions-ps-report/ps-report-3-transformer/mocks/school.ts → tslib/src/functions-ps-report/ps-report-2-pupil-data/mocks/school.ts

@@ -1,4 +1,4 @@
-import { type School } from '../../../functions-ps-report/ps-report-2-pupil-data/models'
+import { type School } from '../pupil-data.models'
 
 export const school: School = {
   estabCode: 999,

tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.spec.ts

@@ -2,7 +2,7 @@ import { PsReportDataService } from './ps-report.data.service'
 import { MockLogger } from '../../common/logger'
 import type { ISqlService } from '../../sql/sql.service'
 import moment from 'moment'
-import type { Pupil, School } from './models'
+import type { Pupil, School } from './pupil-data.models'
 
 describe('ps-report.data.service', () => {
   let sut: PsReportDataService

tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.data.service.ts

@@ -18,7 +18,7 @@ import {
   type Pupil,
   type PupilResult, type RestartReasonCode,
   type School
-} from './models'
+} from './pupil-data.models'
 import * as R from 'ramda'
 import * as RA from 'ramda-adjunct'
 import type moment from 'moment'

tslib/src/functions-ps-report/ps-report-2-pupil-data/ps-report.service.spec.ts

@@ -8,9 +8,8 @@ describe.skip('PsReportService', () => {
   let sut: PsReportService
   let logger: ILogger
   let psReportDataService: IPsReportDataService
-  const serviceBusAdministrationClient: any = {}
   const schoolUuid = 'AAAA-BBBB-CCCC-DDDD'
-  const outputBindings: IOutputBinding = { psReportPupilMessage: [] }
+  const outputBindings: IOutputBinding = { psReportExportOutput: [] }
   const mockPupils = [{ id: 1, schoolId: 99 }, { id: 2, schoolId: 99 }, { id: 3, schoolId: 99 }]
   const mockSchool = { id: 99, name: 'test school' }
   const psReportSchoolFanOutMessage: PsReportSchoolFanOutMessage = {
@@ -28,9 +27,9 @@ describe.skip('PsReportService', () => {
       getPupils: jest.fn(),
       getSchool: jest.fn()
     }
-    outputBindings.psReportPupilMessage = []
+    outputBindings.psReportExportOutput = []
 
-    sut = new PsReportService(outputBindings, logger, psReportDataService, serviceBusAdministrationClient)
+    sut = new PsReportService(outputBindings, logger, psReportDataService)
   })
 
   test('it is defined', () => {
@@ -71,6 +70,6 @@ describe.skip('PsReportService', () => {
       .mockResolvedValueOnce({ data: 2 })
       .mockResolvedValueOnce({ data: 3 })
     await sut.process(psReportSchoolFanOutMessage)
-    expect(outputBindings.psReportPupilMessage).toHaveLength(3)
+    expect(outputBindings.psReportExportOutput).toHaveLength(3)
   })
 })