Bug/65067 ps report utf8 chars (#2903)

* Update ramda 0.28 => 0.30

* [tslib] reverse @azure/storage-blob to v12.18.0 as v12.24.0 doesn't upload blobs

* refactor

* Add cleanbuild target

* Working utf16 upload

* Fix test

* whitespace change

* Fix BOM write, add metadata

* Refactor ps report view creation to happen after loading, not before.

* Appease the linter

* [admin] upgrade minor deps

* [tslib] bump minor deps

* [func-comp] bump minor versions

* [ps report] bump minor deps

* [func-throttledf] Bump minor deps

* Whitespace change

* [admin] upgrade connect-redis from v6 to v7 [major]

* Fix integration test issue with connect-redis picking up ejs not cjs

* add TODO

* Add short-circuit logic

* Remove trace logs - no longer needed

* yarn updates

* whitespace changes from yarn

* ps report non utf char test

* Add format option back in to deal with quotes

---------

Co-authored-by: Mohsen Qureshi <[email protected]>

admin/app.js

@@ -50,6 +50,7 @@ const logger = require('./services/log.service').getLogger()
 const sqlService = require('./services/data-access/sql.service')
 const { formAlreadySubmittedErrorCode } = require('./error-types/form-already-submitted-error')
 const { dfeSignInErrorConsts } = require('./error-types/dfe-signin-error')
+const RedisStore = require('connect-redis').default
 
 const app = express()
 setupLogging(app)
@@ -191,7 +192,6 @@ if (!config.Redis.useTLS) {
   logger.warn('redis running in non-secure mode')
 }
 
-const RedisStore = require('connect-redis')(session)
 const sessionStore = new RedisStore({ client: redisClient })
 
 const sessionOptions = {

admin/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "build": "gulp build",
     "build:watch": "yarn tsc --watch",
+    "cleanbuild": "yarn clean && yarn build",
     "dev": "yarn start:dev",
     "dev-build": "gulp dev-build",
     "shell-build": "./build.sh",
@@ -50,7 +51,7 @@
     "bcryptjs": "^2.4.3",
     "bluebird": "^3.5.1",
     "connect-flash": "^0.1.1",
-    "connect-redis": "^6.1.3",
+    "connect-redis": "^7.1.1",
     "dompurify": "^2.4.2",
     "dotenv": "^16.0.3",
     "edit-json-file": "^1.4.0",
@@ -85,8 +86,8 @@
     "passport": "^0.6.0",
     "passport-local": "^1.0.0",
     "qrcode": "^1.5.1",
-    "ramda": "^0.28.0",
-    "ramda-adjunct": "^3.4.0",
+    "ramda": "^0.30.1",
+    "ramda-adjunct": "^5.1.0",
     "random-number-csprng": "^1.0.2",
     "sass": "^1.69.5",
     "simplemde": "^1.11.2",
@@ -110,7 +111,7 @@
     "@types/jsdom": "^16.2.13",
     "@types/jsonwebtoken": "^9.0.2",
     "@types/node": "18.11.9",
-    "@types/ramda": "^0.27.6",
+    "@types/ramda": "^0.30.2",
     "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^5.37.0",
     "@typescript-eslint/parser": "^5.37.0",
@@ -136,6 +137,7 @@
     "gulp-sourcemaps": "^3.0.0",
     "gulp-typescript": "^6.0.0-alpha.1",
     "gulp-uglify": "^3.0.0",
+    "iconv-lite": "^0.6.3",
     "jest": "^27.0.6",
     "karma": "~6.3.4",
     "karma-chrome-launcher": "~3.1.0",
@@ -195,7 +197,8 @@
     "verbose": false,
     "testRunner": "jest-circus",
     "moduleNameMapper": {
-      "axios": "axios/dist/node/axios.cjs"
+      "axios": "axios/dist/node/axios.cjs",
+      "connect-redis": "connect-redis/dist/cjs/index.js"
     }
   }
-}
+}

admin/services/check-window-v2.service.js

@@ -86,7 +86,7 @@ checkWindowV2Service.getCheckWindows = async () => {
  */
 checkWindowV2Service.getPresentAndFutureCheckWindows = async () => {
   const checkWindows = await checkWindowDataService.sqlFindCheckWindowsWithStatusAndFormCountByType()
-  return R.reject(R.propEq('status', 'Past'), checkWindows)
+  return R.reject(R.propEq('Past', 'status'), checkWindows)
 }
 
 /**

admin/services/csv-file.service.js

@@ -1,6 +1,6 @@
 'use strict'
-
 const blobDataService = require('./data-access/azure-blob.data.service')
+const R = require('ramda')
 
 const csvBlobContainerName = 'csvuploads'
 
@@ -11,6 +11,9 @@ const service = {
    * @returns {Promise<Buffer>}
    */
   getCsvFileAsBuffer: function getCsvFileAsBuffer (remoteFileName) {
+    if (R.isNil(remoteFileName)) {
+      throw new Error('Missing remoteFilename param')
+    }
     return blobDataService.getBlobDataAsBuffer(csvBlobContainerName, remoteFileName)
   }
 }

admin/services/data-access/sql.service.js

@@ -213,8 +213,8 @@ function addParamsToRequest (params, request) {
         throw new Error('parameter type invalid')
       }
       const options = {}
-      if (R.pathEq(['type', 'name'], 'Decimal', param) ||
-        R.pathEq(['type', 'name'], 'Numeric', param)) {
+      if (R.pathEq('Decimal', ['type', 'name'], param) ||
+        R.pathEq('Numeric', ['type', 'name'], param)) {
         options.precision = param.precision || 28
         options.scale = param.scale || 5
       }

admin/services/type-of-establishment/type-of-establishment-service.ts

@@ -4,7 +4,8 @@ import * as sortService from '../../helpers/table-sorting'
 export class TypeOfEstablishmentService {
   public static async getEstablishmentDataSortedByName (): Promise<TypeOfEstablishmentData[]> {
     const data = await TypeOfEstablishmentDataService.sqlGetEstablishmentData()
-    const sorted = sortService.sortByProps(['name'], data)
+    // @ts-ignore: ts gets this wrong
+    const sorted: TypeOfEstablishmentData[] = sortService.sortByProps(['name'], data)
     return sorted
   }
 }

admin/tests-integration/jest.integration.config.js

@@ -3,6 +3,7 @@ module.exports = {
   testMatch: ['./**/(*.)+(spec|test).[jt]s?(x)'],
   testRunner: 'jest-circus', // As of jest 27 this is the legacy mode (slow)
   moduleNameMapper: {
-    axios: 'axios/dist/node/axios.cjs'
+    axios: 'axios/dist/node/axios.cjs',
+    'connect-redis': 'connect-redis/dist/cjs/index.js'
   }
 }