Bug/53880 user inputs (#2376)

* Trim inputs during data sync * Add tests * Remove debug * Add test, remove debug, mod service * Ordered timestamps for answer, inputs and events are cohesive * add logger * Remove unused code * Remove unused code * fix lint * Cleanup and fix tests * update assets version * Remove debug * Fix flaky test * adding test for duplicate questions --------- Co-authored-by: Mohsen Qureshi <[email protected]> Co-authored-by: Mohsen Qureshi <[email protected]>
DFEAGILEDEVOPS · Feb 2, 2023 · 38372c4 · 38372c4
1 parent 439eb22
commit 38372c4
Show file tree

Hide file tree

Showing 24 changed files with 577 additions and 280 deletions.
diff --git a/docs/support-utils/util-submit-check.md b/docs/support-utils/util-submit-check.md
@@ -56,3 +56,4 @@ If submitting a [check-started message](../messaging/message-schemas.md) is part
 
 ## Roadmap
 Invalid payload generation and structures that mimic real world examples of faulty payloads seen in live check periods.
+
diff --git a/func-consumption/package.json b/func-consumption/package.json
@@ -23,6 +23,7 @@
   },
   "devDependencies": {
     "@azure/functions": "^1.0.3",
+    "@faker-js/faker": "^7.6.0",
     "@types/bcryptjs": "^2.4.2",
     "@types/bluebird": "^3.5.27",
     "@types/ioredis": "^4.0.18",
@@ -53,7 +54,6 @@
     "bluebird": "^3.7.0",
     "csv-string": "^4.1.0",
     "dotenv": "^10.0.0",
-    "faker": "^5.5.3",
     "fast-xml-parser": "^3.20.0",
     "ioredis": "^4.27.9",
     "lz-string": "^1.4.4",

diff --git a/func-consumption/yarn.lock b/func-consumption/yarn.lock
@@ -298,6 +298,11 @@
     "@azure/logger" "^1.0.0"
     tslib "^2.2.0"
 
+"@faker-js/faker@^7.6.0":
+  version "7.6.0"
+  resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-7.6.0.tgz#9ea331766084288634a9247fcd8b84f16ff4ba07"
+  integrity sha512-XK6BTq1NDMo9Xqw/YkYyGjSsg44fbNwYRx7QK2CuoQgyy+f1rrTDHoExVM5PsyXCtfl2vs2vVJ0MN0yN6LppRw==
+
 "@js-joda/core@^5.2.0":
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/@js-joda/core/-/core-5.3.1.tgz#998557969f357b4494649d4a63164d02645295af"
@@ -1429,11 +1434,6 @@ extglob@^2.0.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-faker@^5.5.3:
-  version "5.5.3"
-  resolved "https://registry.yarnpkg.com/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e"
-  integrity sha512-wLTv2a28wjUyWkbnX7u/ABZBkUkIF2fCd73V6P2oFqEGEktDfzWx4UxrSqtPRw0xPRAcjeAOIiJWqZm3pP4u3g==
-
 fancy-log@^1.3.2:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.3.tgz#dbc19154f558690150a23953a0adbd035be45fc7"

diff --git a/func-ps-report/package.json b/func-ps-report/package.json
@@ -43,7 +43,6 @@
     "bluebird": "^3.7.0",
     "csv-string": "^4.1.0",
     "dotenv": "^10.0.0",
-    "faker": "^5.5.3",
     "fast-xml-parser": "^3.20.0",
     "ioredis": "^4.27.9",
     "lz-string": "^1.4.4",

diff --git a/func-ps-report/yarn.lock b/func-ps-report/yarn.lock
@@ -1364,11 +1364,6 @@ extglob@^2.0.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-faker@^5.5.3:
-  version "5.5.3"
-  resolved "https://registry.yarnpkg.com/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e"
-  integrity sha512-wLTv2a28wjUyWkbnX7u/ABZBkUkIF2fCd73V6P2oFqEGEktDfzWx4UxrSqtPRw0xPRAcjeAOIiJWqZm3pP4u3g==
-
 fancy-log@^1.3.2:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.3.tgz#dbc19154f558690150a23953a0adbd035be45fc7"

diff --git a/func-throttled/package.json b/func-throttled/package.json
@@ -43,7 +43,6 @@
     "bluebird": "^3.7.0",
     "csv-string": "^4.1.0",
     "dotenv": "^10.0.0",
-    "faker": "^5.5.3",
     "fast-xml-parser": "^3.20.0",
     "ioredis": "^4.27.9",
     "lz-string": "^1.4.4",

diff --git a/func-throttled/yarn.lock b/func-throttled/yarn.lock
@@ -1364,11 +1364,6 @@ extglob@^2.0.4:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-faker@^5.5.3:
-  version "5.5.3"
-  resolved "https://registry.yarnpkg.com/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e"
-  integrity sha512-wLTv2a28wjUyWkbnX7u/ABZBkUkIF2fCd73V6P2oFqEGEktDfzWx4UxrSqtPRw0xPRAcjeAOIiJWqZm3pP4u3g==
-
 fancy-log@^1.3.2:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.3.tgz#dbc19154f558690150a23953a0adbd035be45fc7"

diff --git a/test/pupil-hpa/features/a_ps_report.feature b/test/pupil-hpa/features/a_ps_report.feature
@@ -83,3 +83,8 @@ Feature:
     When I generate a new pin and complete the check
     And the data sync and ps report function has run
     Then I should see the restart reason in the ps report record
+
+  Scenario: Only first input used when a duplicate question is shown
+    Given I have completed a check with duplicate questions
+    And the data sync and ps report function has run
+    Then I should see the ps report showing the first input
diff --git a/test/pupil-hpa/features/step_definitions/ps_report_steps.rb b/test/pupil-hpa/features/step_definitions/ps_report_steps.rb
@@ -235,17 +235,14 @@
   p @check_code
 end
 
-
 Then(/^the ps report record should be updated with all the check details$/) do
   step 'I should see a record for the pupil in the ps report table'
 end
 
-
 And(/^the latest check is recorded$/) do
   expect(@check_details['checkCode']).eql? @second_check_code
 end
 
-
 When(/^I add an AA arrangement$/) do
   access_arrangments_type = "Audible time alert"
   visit ENV["ADMIN_BASE_URL"] + access_arrangements_page.url
@@ -281,7 +278,6 @@
   p @check_code
 end
 
-
 Then(/^the PS report should include the AA for the pupil$/) do
   step 'the data sync and ps report function has run'
   step 'I should see a record for the pupil in the ps report table'
@@ -290,7 +286,6 @@
   expect(pupil_aa).to eql '[1]'
 end
 
-
 Given(/^I generated a pin after applying a restart$/) do
   step "I have completed the check"
   step 'I login to the admin app'
@@ -315,7 +310,6 @@
   SqlDbHelper.delete_check_pin(@check_details["id"])
 end
 
-
 When(/^I generate a new pin and complete the check$/) do
   navigate_to_pupil_list_for_pin_gen('live')
   generate_pins_overview_page.generate_pin_using_name(@details_hash[:last_name] + ', ' + @details_hash[:first_name])
@@ -345,24 +339,20 @@
   p @check_code
 end
 
-
 Then(/^I should see the restart reason in the ps report record$/) do
   ps_report_record = SqlDbHelper.get_ps_record_for_pupil(@pupil_details['id'])
   expect(ps_report_record['RestartReason']).to eql 2
 end
 
-
 Given(/^I have completed the check for a pupil attending a test school$/) do
   SqlDbHelper.set_school_as_test_school(@school['entity']['dfeNumber'])
   step 'I have completed the check'
 end
 
-
 Then(/^I should not see any records for the test school$/) do
   expect(SqlDbHelper.count_all_ps_records_for_school(@school_id)).to eql 0
 end
 
-
 When(/^the data sync and ps report function has run for the test school$/) do
   step 'the data sync function has run'
   sleep ENV['PS_REPORT_WAIT_TIME'].to_i
@@ -371,3 +361,35 @@
   expect(response.code).to eql 202
   sleep ENV['PS_REPORT_WAIT_TIME'].to_i
 end
+
+Given(/^I have completed a check with duplicate questions$/) do
+  name = (0...8).map {(65 + rand(26)).chr}.join
+  step 'I add a pupil'
+  step 'I login to the admin app'
+  navigate_to_pupil_list_for_pin_gen('live')
+  generate_pins_overview_page.generate_pin_using_name(@details_hash[:last_name] + ', ' + @details_hash[:first_name])
+  pupil_pin_row = view_and_custom_print_live_check_page.pupil_list.rows.find {|row| row.name.text == @details_hash[:last_name] + ', ' + @details_hash[:first_name]}
+  @pupil_credentials = {:school_password => pupil_pin_row.school_password.text, :pin => pupil_pin_row.pin.text}
+  p @pupil_credentials
+  AzureTableHelper.wait_for_prepared_check(@pupil_credentials[:school_password], @pupil_credentials[:pin])
+  @check_code = SqlDbHelper.check_details(@stored_pupil_details['id'])['checkCode']
+  @pupil_id = @stored_pupil_details['id']
+  check_entry = SqlDbHelper.check_details(@pupil_id)
+  Timeout.timeout(ENV['WAIT_TIME'].to_i) {sleep 1 until RequestHelper.auth(@pupil_credentials[:school_password], @pupil_credentials[:pin]).code == 200}
+  RequestHelper.auth(@pupil_credentials[:school_password], @pupil_credentials[:pin])
+  @check_code = check_entry['checkCode']
+  FunctionsHelper.complete_check_with_duplicates([@check_code], 25, 0, rand(25)) if check_entry["isLiveCheck"]
+  @recieved_check = AzureTableHelper.wait_for_received_check(@school['entity']['urlSlug'], @check_code) if check_entry["isLiveCheck"]
+  p @check_code
+end
+
+Then(/^I should see the ps report showing the first input$/) do
+  @answers = JSON.parse(LZString::UTF16.decompress(@recieved_check['archive']))['answers']
+  grouped = @answers.group_by {|row| [row['sequenceNumber'], row['question']]}
+  duplicates = grouped.values.select {|a| a.size > 1}
+  expected_answers = duplicates.map {|d| d.first['answer']}
+  expected_questions = duplicates.map {|d| d.first['sequenceNumber'].to_s}
+  ps_report_record = SqlDbHelper.get_ps_record_for_pupil(@pupil_id)
+  answers = expected_questions.map {|question| ps_report_record["Q#{question}Response"]}
+  expect(expected_answers).to eql answers
+end
diff --git a/test/pupil-hpa/features/support/functions_helper.rb b/test/pupil-hpa/features/support/functions_helper.rb
@@ -25,4 +25,8 @@ def self.complete_check_via_check_code(check_code_array)
     HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-submit-check", :body => {'checkCodes': check_code_array}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']})
   end
 
+  def self.complete_check_with_duplicates(check_code_array, correct_answers,incorrect_answers,duplicate_answers)
+    HTTParty.post(ENV['FUNC_CONSUMP_BASE_URL'] + "/api/util-submit-check", :body => {'checkCodes': check_code_array, 'answerNumberFromCorrectCheckForm': correct_answers, 'answerNumberFromIncorrectCheckForm': incorrect_answers, 'answerNumberOfDuplicates': duplicate_answers}.to_json, headers: {'Content-Type' => 'application/json', 'x-functions-key' => ENV['FUNC_CONSUMP_MASTER_KEY']})
+  end
+
 end
diff --git a/tslib/package.json b/tslib/package.json
@@ -29,10 +29,10 @@
   },
   "devDependencies": {
     "@azure/functions": "^1.2.3",
+    "@faker-js/faker": "^7.6.0",
     "@types/adm-zip": "^0.4.34",
     "@types/async": "^3.2.7",
     "@types/bluebird": "^3.5.36",
-    "@types/faker": "^5.5.8",
     "@types/he": "^1.1.2",
     "@types/ioredis": "^4.27.4",
     "@types/jest": "^26.0.15",
@@ -70,7 +70,6 @@
     "bluebird": "^3.7.0",
     "csv-string": "^4.1.0",
     "dotenv": "^10.0.0",
-    "faker": "^5.5.3",
     "fast-xml-parser": "^3.20.0",
     "ioredis": "^4.27.9",
     "lz-string": "^1.4.4",

diff --git a/tslib/src/caching/prepared-check.service.spec.ts b/tslib/src/caching/prepared-check.service.spec.ts
@@ -1,6 +1,6 @@
 import { IRedisService } from './redis-service'
 import { RedisServiceMock } from './redis-service.mock'
-import * as faker from 'faker'
+import { faker } from '@faker-js/faker'
 import redisKeyService from './redis-key.service'
 import { PreparedCheckService } from './prepared-check.service'
 

diff --git a/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.spec.ts b/tslib/src/functions-ps-report/ps-report-1-list-schools/list-schools-service.spec.ts
@@ -28,7 +28,6 @@ describe('ListSchoolsService', () => {
 
   test('getSchoolMessages returns messages', async () => {
     const resp = await sut.getSchoolMessages()
-    console.log(resp)
     expect(resp).toHaveLength(2)
     expect(resp[0].name).toBe('School One')
     expect(resp[0].uuid).toBe('uuid1')

diff --git a/tslib/src/functions-throttled/sync-results-to-sql/models.ts b/tslib/src/functions-throttled/sync-results-to-sql/models.ts
@@ -44,6 +44,7 @@ export interface Audit {
   type: string
   clientTimestamp: string
   data?: Data
+  monotonicTime?: IMonotonicTimeDto
 }
 
 export interface Data {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -56,3 +56,4 @@ If submitting a [check-started message](../messaging/message-schemas.md) is part

		## Roadmap
		Invalid payload generation and structures that mimic real world examples of faulty payloads seen in live check periods.