Feature/43328 ps report transformation (#1802)

* [tslib] add new function skel * Add pupil fields to report * Fix typos * fix timestamp typo * [tslib] add tests test constructor / add immutability * [admin] fix flaky test * Add fields * Add fields * Add check fields * refactor * add incomplete check tests * Add missing mock * Add device fields * Change date of birth to be an actual date type * Add start of question fields * refactor * refactor part 2 * add tests * Fix the check form service integration test * Add first and last key press calculations * [tslib] Add tests for response time * Fix test, broken by recent change to the unit test * [tslib] add question timeout * [tslib] add timeoutResponse * [tslib] add timeout score * [tslib] add question load time * [tslib] add overall time * [tslib] add recall time * [tslib] type update * [tslib] Drop obsolete table mtc_admin.answer was migrated to mtc_results.answer * [db] Drop obsolete table * [db] drop obsolete table * [tslib] clean up Co-authored-by: Guy Harwood <[email protected]> Co-authored-by: Mohsen Qureshi <[email protected]>
DFEAGILEDEVOPS · Jan 29, 2021 · 45783a0 · 45783a0
1 parent a6a9c31
commit 45783a0
Show file tree

Hide file tree

Showing 36 changed files with 2,510 additions and 52 deletions.
diff --git a/admin/assets/javascripts/session-expiry.js b/admin/assets/javascripts/session-expiry.js
@@ -11,13 +11,18 @@ $(function () {
   window.GOVUK.sessionExpiry = {
     /**
      * Set the formatted text on minutesCountdown to the count of `minutes`
+     * @param {jQuery} minutesCountdown
+     * @param {number} minutes
+     *
      */
     setCountdownText: function (minutesCountdown, minutes) {
       var formattedText = minutes === 1 ? '1 minute' : minutes + ' minutes'
       minutesCountdown.text(formattedText)
     },
     /**
      * Start a setInterval to update the countdown every `tickMs`
+     * @param {jQuery} minutesCountdown
+     * @param {number} tickMs
      */
     startTimer: function (minutesCountdown, tickMs) {
       var remainingMinutes = Math.ceil((SESSION_EXPIRATION_TIME - SESSION_DISPLAY_NOTICE_TIME) / 60)
@@ -33,6 +38,9 @@ $(function () {
     },
     /**
      * Unhide the expiration banner, register the button click handler and start the countdown
+     * @param {jQuery} sessionExpirationError
+     * @param {jQuery} minutesCountdown
+     * @param {jQuery} continueSessionButton
      */
     displayExpiryBanner: function (sessionExpirationError, minutesCountdown, continueSessionButton) {
       sessionExpirationError.removeClass('error-session-expiration')
@@ -48,6 +56,7 @@ $(function () {
 
     /**
      * Hide the expiration banner
+     * @param {jQuery} sessionExpirationError
      */
     hideExpiryBanner: function (sessionExpirationError) {
       sessionExpirationError.removeClass('error-about-to-expire-session')
@@ -56,6 +65,8 @@ $(function () {
 
     /**
      * Display the banner with the expired content
+     * @param {jQuery} sessionExpirationError
+     * @param {jQuery} sessionExpirationErrorBody
      */
     displayExpiredBanner: function (sessionExpirationError, sessionExpirationErrorBody) {
       // Replace the content of the session expiration body

diff --git a/admin/spec/front-end/session-expiry.spec.js b/admin/spec/front-end/session-expiry.spec.js
@@ -61,21 +61,30 @@ describe('sessionExpiry', function () {
       fixtureMinutesCountdown = $('<span class="session-expiration-countdown"></span>')
       fixtureButton = $('<button id="continue-session-expiration"></button>')
     })
-    it('should make the container visible by changing classes', function () {
-      window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
-      expect(fixtureContainer.hasClass('error-session-expiration')).toBe(false)
-      expect(fixtureContainer.hasClass('error-about-to-expire-session')).toBe(true)
+    it('should make the container visible by changing classes', function (done) {
+      $(function () {
+        window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
+        expect(fixtureContainer.hasClass('error-session-expiration')).toBe(false)
+        expect(fixtureContainer.hasClass('error-about-to-expire-session')).toBe(true)
+        done()
+      })
     })
-    it('should add a reload click handler on the continue button', function () {
-      spyOn(window.GOVUK.sessionExpiry, 'hideExpiryBanner')
-      window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
-      fixtureButton.click()
-      expect(window.GOVUK.sessionExpiry.hideExpiryBanner).toHaveBeenCalled()
+    it('should add a reload click handler on the continue button', function (done) {
+      $(function () {
+        spyOn(window.GOVUK.sessionExpiry, 'hideExpiryBanner')
+        window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
+        fixtureButton.click()
+        expect(window.GOVUK.sessionExpiry.hideExpiryBanner).toHaveBeenCalled()
+        done()
+      })
     })
-    it('should start the timer with a minute interval', function () {
-      spyOn(window.GOVUK.sessionExpiry, 'startTimer')
-      window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
-      expect(window.GOVUK.sessionExpiry.startTimer).toHaveBeenCalledWith(fixtureMinutesCountdown, 60 * 1000)
+    it('should start the timer with a minute interval', function (done) {
+      $(function () {
+        spyOn(window.GOVUK.sessionExpiry, 'startTimer')
+        window.GOVUK.sessionExpiry.displayExpiryBanner(fixtureContainer, fixtureMinutesCountdown, fixtureButton)
+        expect(window.GOVUK.sessionExpiry.startTimer).toHaveBeenCalledWith(fixtureMinutesCountdown, 60 * 1000)
+        done()
+      })
     })
   })
 

diff --git a/admin/tests-integration/check-form.service.spec.js b/admin/tests-integration/check-form.service.spec.js
@@ -4,7 +4,7 @@
  * @file Integration Tests for Check Form Service
  */
 
-/* global describe test expect it beforeAll jasmine */
+/* global describe test expect beforeAll jasmine */
 
 // This test may take some time to complete
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 120000
@@ -14,29 +14,28 @@ const moment = require('moment')
 
 const checkFormService = require('../services/check-form.service')
 
-describe('check-form.service', () => {
+describe.skip('check-form.service', () => {
   const availableForms = []
   const seenForms = []
 
-  test.skip('in-memory tests of check-form.service.allocateCheckForm', () => {
+  describe('in-memory tests of check-form.service.allocateCheckForm', () => {
     beforeAll(async () => {
-      const form1 = await checkFormService.getCheckForm(1)
-      for (let i = 1; i < 21; i++) {
+      for (let i = 1; i < 10; i++) {
         const form = {
           id: i,
           name: `Integration Test Form ${i}`,
           isDeleted: false,
-          formData: form1.formData
+          formData: []
         }
         availableForms.push(form)
       }
     })
 
-    it('has enough forms to complete a random sample', () => {
-      expect(availableForms.length).toBe(20)
+    test('has enough forms to complete a random sample', () => {
+      expect(availableForms.length).toBe(9)
     })
 
-    it('allocates 20 check forms equally when there are no seen forms', async () => {
+    test('allocates 20 check forms equally when there are no seen forms', async () => {
       const formsAllocated = []
       const runs = 628718
       for (let i = 0; i < runs; i++) {

diff --git a/db/migrations/schema-objects/20210128093754.do.drop-mtc-admin-answer.sql b/db/migrations/schema-objects/20210128093754.do.drop-mtc-admin-answer.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS [mtc_admin].[answer];
diff --git a/db/migrations/schema-objects/20210128093754.undo.drop-mtc-admin-answer.sql b/db/migrations/schema-objects/20210128093754.undo.drop-mtc-admin-answer.sql
@@ -0,0 +1,30 @@
+CREATE TABLE [mtc_admin].[answer]
+([id]             [int] IDENTITY (1,1) NOT NULL,
+ [createdAt]      [datetimeoffset](3)  NOT NULL,
+ [updatedAt]      [datetimeoffset](3)  NOT NULL,
+ [version]        [timestamp]          NOT NULL,
+ [check_id]       [int]                NOT NULL,
+ [questionNumber] [smallint]           NOT NULL,
+ [answer]         [nvarchar](60)       NOT NULL,
+ [isCorrect]      [bit]                NOT NULL,
+ [question_id]    [int]                NOT NULL,
+ CONSTRAINT [PK_answers] PRIMARY KEY CLUSTERED ([id] ASC) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+)
+ON [PRIMARY]
+GO
+
+/****** Object:  Index [answer_check_id_questionNumber_uindex]   ******/
+CREATE UNIQUE NONCLUSTERED INDEX [answer_check_id_questionNumber_uindex] ON [mtc_admin].[answer] ([check_id] ASC, [questionNumber] ASC) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+
+/****** Object:  Index [ix_answer_question_id]  ******/
+CREATE NONCLUSTERED INDEX [ix_answer_question_id] ON [mtc_admin].[answer] ([question_id] ASC) WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+GO
+
+
+CREATE TRIGGER [mtc_admin].[answerUpdatedAtTrigger]
+    ON [mtc_admin].[answer]
+    FOR UPDATE AS
+BEGIN
+    UPDATE [mtc_admin].[answer] SET updatedAt = GETUTCDATE() FROM inserted WHERE [answer].id = inserted.id
+END
diff --git a/...ns/schema-objects/20210128112413.do.drop-table-mtc-admin-psychometrician-report-cache.sql b/...ns/schema-objects/20210128112413.do.drop-table-mtc-admin-psychometrician-report-cache.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS [mtc_admin].[psychometricianReportCache];
diff --git a/.../schema-objects/20210128112413.undo.drop-table-mtc-admin-psychometrician-report-cache.sql b/.../schema-objects/20210128112413.undo.drop-table-mtc-admin-psychometrician-report-cache.sql
@@ -0,0 +1,31 @@
+CREATE TABLE [mtc_admin].[psychometricianReportCache]
+(
+    id INT IDENTITY
+        CONSTRAINT PK_psychometricianReportCache
+            PRIMARY KEY,
+    check_id INT NOT NULL
+        REFERENCES mtc_admin.[check],
+    jsonData NVARCHAR(max) NOT NULL,
+    version TIMESTAMP NOT NULL,
+    createdAt DATETIMEOFFSET(3) DEFAULT getutcdate() NOT NULL,
+    updatedAt DATETIMEOFFSET(3) DEFAULT getutcdate() NOT NULL
+)
+GO
+
+GRANT DELETE ON mtc_admin.psychometricianReportCache to mtcAdminUser
+GO
+
+CREATE UNIQUE INDEX psychometricianReportCache_check_id_uindex
+    on mtc_admin.psychometricianReportCache (check_id)
+GO
+
+CREATE TRIGGER [mtc_admin].[psychometricianReportCacheUpdatedAtTrigger]
+    ON [mtc_admin].[psychometricianReportCache]
+    FOR UPDATE
+    AS
+BEGIN
+    UPDATE [mtc_admin].[psychometricianReportCache]
+       SET updatedAt = GETUTCDATE()
+      FROM inserted
+     WHERE [psychometricianReportCache].id = inserted.id
+END
diff --git a/db/migrations/schema-objects/20210128113040.do.drop-table-mtc-admin-anomaly-report-cache.sql b/db/migrations/schema-objects/20210128113040.do.drop-table-mtc-admin-anomaly-report-cache.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS [mtc_admin].[anomalyReportCache];
diff --git a/...grations/schema-objects/20210128113040.undo.drop-table-mtc-admin-anomaly-report-cache.sql b/...grations/schema-objects/20210128113040.undo.drop-table-mtc-admin-anomaly-report-cache.sql
@@ -0,0 +1,24 @@
+CREATE TABLE mtc_admin.anomalyReportCache
+(id        INT IDENTITY
+     CONSTRAINT PK_anomalyReportCache PRIMARY KEY,
+ check_id  INT                                    NOT NULL REFERENCES mtc_admin.[check],
+ jsonData  NVARCHAR(MAX)                          NOT NULL,
+ version   TIMESTAMP                              NOT NULL,
+ createdAt DATETIMEOFFSET(3) DEFAULT getutcdate() NOT NULL,
+ updatedAt DATETIMEOFFSET(3) DEFAULT getutcdate() NOT NULL
+)
+GO
+
+GRANT DELETE ON mtc_admin.anomalyReportCache TO mtcAdminUser
+GO
+
+
+CREATE TRIGGER [mtc_admin].[anomalyReportCacheUpdatedAtTrigger]
+    ON [mtc_admin].[anomalyReportCache]
+    FOR UPDATE AS
+BEGIN
+    UPDATE [mtc_admin].[anomalyReportCache]
+       SET updatedAt = GETUTCDATE()
+      FROM inserted
+     WHERE [anomalyReportCache].id = inserted.id
+END
diff --git a/deploy/service-bus/queues-topics.json b/deploy/service-bus/queues-topics.json
@@ -7,6 +7,7 @@
     "check-validation",
     "ps-report-schools",
     "ps-report-staging",
+    "ps-report-export",
     "pupil-login",
     "school-results-cache",
     "sync-results-to-db-complete"

diff --git a/docs/diagrams/psychometric-report-generation.graffle b/docs/diagrams/psychometric-report-generation.graffle
diff --git a/docs/psychometric-report-data-sourcing.md b/docs/psychometric-report-data-sourcing.md
@@ -44,7 +44,7 @@
 | FormID             | mtc_admin.checkForm.name                   |                                                              |
 | TestDate           | mtc_admin.check.pupilLoginDate             |                                                              |
 | TimeStart          | mtc_results.event                          | Where eventType = "CheckStarted"                             |
-| TimeComplete       | mtc_results.userInput OR mtc_results.event | Time the check was completed - From last key (enter) is pressed or timeout.  userInput is user is the user pressed 'Enter' but the event is used if we need to use the timeout |
+| TimeComplete       | mtc_results.userInput OR mtc_results.event | Time the check was completed - From last key (enter) is pressed or timeout.  userInput is user is the user pressed 'Enter' but the event is used if we need to use the timeout.  This is the same as the timestamp on the answer. |
 | TimeTaken          | Calculated                                 | TimeComplete - TimeStart expressed as a duration in hh::mm::ss |
 | RestartNumber      | mtc_admin.pupilRestart                     | Values 0-2                                                   |
 | FormMark           | mtc_results.checkResult.mark               | As pupils could have taken many checks the check used by the report is determined by the the check ID stored in mtc_admin.pupil.currentCheckId which is a FK to mtc_admin.check |
@@ -69,17 +69,17 @@ where *n* is the question number (from 1 to 25)
 | Psychometric field | Source                      | Comment                                                      |
 | ------------------ | --------------------------- | ------------------------------------------------------------ |
 | QnID               | mtc_admin.question          | e.g. '6x7'                                                   |
-| QnResponse         | mtc_results.answer          |                                                              |
-| QnInputMethods     | mtc_results.userInputLookup |                                                              |
-| QnK                | mtc_results.userInput       |                                                              |
-| QnScore            | mtc_results.answer          |                                                              |
+| QnResponse         | mtc_results.answer          | the answer provided by the pupil                             |
+| QnInputMethods     | mtc_results.userInputLookup | Single character string: `k` - when using keyboard `t` - when using a touchscreen `m` - when using a mouse `x` - when combination blank - when there is no input |
+| QnK                | mtc_results.userInput       | Lists each individual key stroke during the time limit separated by square brackets and preceded by: `k` - when using keyboard `t` - when using a touchscreen `m` - when using a mouse `x` - when combination blank - when there is no input |
+| QnSco              | mtc_results.answer          | Question answer (1 = correct, 0 = incorrect)                 |
+| QntFirstKey        | mtc_results.userInput       | Timestamp of the first key pressed (whether using key, mouse or touchscreen) |
+| QntLastKey         | mtc_results.userInput       | Timestamp of the last key that is not "enter" (whether using key, mouse or touchscreen) QntLastKey excludes everything that is not 0-9 |
 | QnResponseTime     | mtc_results.userInput       | QntLastkey - QntFirstKey                                     |
 | QnTimeOut          | mtc_results.userInput       | If the user pressed the Enter key as the last input it did not timeout |
 | QnTimeOutResponse  | QnTimeout, QnResponse       | Timeout with no response (1 = with response, 0 = no response, empty = didn't time out) |
 | QnTimeOutSco       | QnTimeout and QnScore       | Timeout with correct answer (1 = correct, 0 = incorrect, empty = didn't time out) |
 | QntLoad            | mtc_results.event           | Timestamp when the question loads.  Where eventType = QuestionTimerStarted |
-| QntFirstKey        | mtc_results.userInput       | Timestamp of the first key pressed (whether using key, mouse or touchscreen) |
-| QntLastKey         | mtc_results.userInput       | Timestamp of the last key that is not "enter" (whether using key, mouse or touchscreen) QntLastKey excludes everything that is not 0-9 |
 | QnOverallTime      | QntLastKey, QntLoad         | QntLastKey - QntLoad                                         |
 | QnRecallTime       | QntFirstKey, QntLoad        | Time between question appearing and the first key being pressed [QntFirstKey - QntLoad] (whether using key, mouse or touchscreen) |
 | QnReaderStart      | mtc_results.event           | Where eventType = QuestionReadingStarted                     |

diff --git a/func-consumption/disable-functions.env b/func-consumption/disable-functions.env
@@ -1,20 +1,21 @@
-AzureWebJobs.check-marker.Disabled=false
-AzureWebJobs.check-notifier-batch.Disabled=false
-AzureWebJobs.check-notifier.Disabled=false
+AzureWebJobs.check-marker.Disabled=true
+AzureWebJobs.check-notifier-batch.Disabled=true
+AzureWebJobs.check-notifier.Disabled=true
 AzureWebJobs.check-pin-expiry.Disabled=true
-AzureWebJobs.check-receiver.Disabled=false
+AzureWebJobs.check-receiver.Disabled=true
 AzureWebJobs.check-started.Disabled=false
 AzureWebJobs.check-sync.Disabled=true
-AzureWebJobs.check-validator.Disabled=false
+AzureWebJobs.check-validator.Disabled=true
 AzureWebJobs.gias-sync.Disabled=true
-AzureWebJobs.pupil-auth.Disabled=false
-AzureWebJobs.pupil-feedback.Disabled=false
-AzureWebJobs.pupil-login.Disabled=false
-AzureWebJobs.pupil-prefs.Disabled=false
-AzureWebJobs.school-pin-generator.Disabled=false
-AzureWebJobs.school-pin-http-service.Disabled=false
-AzureWebJobs.school-results-cache-determiner.Disabled=false
-AzureWebJobs.school-results-cache.Disabled=false
+AzureWebJobs.ps-report-transformer.Disabled=false
+AzureWebJobs.pupil-auth.Disabled=true
+AzureWebJobs.pupil-feedback.Disabled=true
+AzureWebJobs.pupil-login.Disabled=true
+AzureWebJobs.pupil-prefs.Disabled=true
+AzureWebJobs.school-pin-generator.Disabled=true
+AzureWebJobs.school-pin-http-service.Disabled=true
+AzureWebJobs.school-results-cache-determiner.Disabled=true
+AzureWebJobs.school-results-cache.Disabled=true
 AzureWebJobs.util-diag.Disabled=true
 AzureWebJobs.util-school-pin-sampler.Disabled=true
 AzureWebJobs.util-submit-check.Disabled=true
diff --git a/func-consumption/ps-report-transformer/function.json b/func-consumption/ps-report-transformer/function.json
@@ -0,0 +1,19 @@
+{
+  "bindings": [
+    {
+      "direction": "in",
+      "type": "serviceBusTrigger",
+      "name": "inputData",
+      "queueName": "ps-report-staging",
+      "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
+    },
+    {
+      "direction": "out",
+      "type": "serviceBus",
+      "name": "outputData",
+      "queueName": "ps-report-export",
+      "connection": "AZURE_SERVICE_BUS_CONNECTION_STRING"
+    }
+  ],
+  "scriptFile": "../dist/functions/ps-report-transformer/index.js"
+}
diff --git a/tslib/src/common/deep-freeze.ts b/tslib/src/common/deep-freeze.ts
@@ -0,0 +1,17 @@
+export function deepFreeze <T> (obj: T): T {
+  if (typeof obj !== 'object') {
+    return obj
+  }
+  if (obj === null) {
+    return obj
+  }
+  const properties = Object.getOwnPropertyNames(obj)
+  for (const name of properties) {
+    // @ts-ignore - ignore any return type
+    const value: any = obj[name]
+    if (value !== null && value !== undefined && typeof value === 'object') {
+      deepFreeze(value)
+    }
+  }
+  return Object.freeze(obj)
+}
diff --git a/tslib/src/common/json-reviver.ts b/tslib/src/common/json-reviver.ts
@@ -0,0 +1,21 @@
+import moment from 'moment'
+const simpleIso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
+
+/**
+ * Reviver function for use with JSON.parse() to instantiate DateTime strings as Moment.moment objects
+ * @param key
+ * @param value
+ */
+export function jsonReviver (key: any, value: any): any {
+  if (value !== null && value !== undefined && typeof value === 'string') {
+    if (simpleIso8601Regex.test(value)) {
+      try {
+        const d = moment(value)
+        if (d?.isValid()) {
+          return d
+        }
+      } catch (ignored) {}
+    }
+  }
+  return value
+}
diff --git a/tslib/src/functions-throttled/ps-report-2-pupil-data/models.ts b/tslib/src/functions-throttled/ps-report-2-pupil-data/models.ts
@@ -2,7 +2,7 @@ import moment from 'moment'
 
 export interface Pupil {
   attendanceId: number | null
-  checkComplete: boolean
+  checkComplete: boolean | null
   currentCheckId: number | null
   dateOfBirth: moment.Moment
   forename: string
@@ -57,12 +57,12 @@ export interface Check {
   checkFormId: number
   checkWindowId: number
   complete: boolean
-  completedAt: moment.Moment
+  completedAt: moment.Moment | null
   inputAssistantAddedRetrospectively: boolean
   isLiveCheck: boolean
-  mark: number
+  mark: number | null
   processingFailed: boolean
-  pupilLoginDate: moment.Moment
+  pupilLoginDate: moment.Moment | null
   received: boolean
   restartNumber: number
 }
@@ -93,6 +93,7 @@ export interface Answer {
   isCorrect: boolean
   question: string
   questionCode: string
+  questionNumber: number
   response: string
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DROP TABLE IF EXISTS [mtc_admin].[psychometricianReportCache];
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		DROP TABLE IF EXISTS [mtc_admin].[anomalyReportCache];