-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature/45933 bulk upload schools (#2002)
* [admin] add nodemon package back in * [admin] add view page to upload the organisations file * [admin] - util update clean up local files when destroying the dev local env * [admin] bugfix auto-upload files Clarify the tmp folder for uploaded files using full path * [admin] comment out azure-upload as probably not needed * [load-test] update reset script to drop results schema tables * [admin] add initial upload work * [admin] debounce form submit * [load-test] update reset script to clear out new schools * [admin] improve form * [admin] add validation error reporting * [db] add new job type * [admin] add initial job life-cycle tracking on the school upload * [load-test] add back in removed line for users * [load-test] security upgrade relock lockfie to upgrade [email protected] to [email protected] * [admin] refactor; add test * [admin] add tests * [tslib] add test * remove debug line * [tslib] add test * [tslib/school-import] refactor to clean code and add tests * Simplify * [admin] add feature to download organisation job results * bump minor versions * bump versions * [admin/pupil] remove sinon * convert to jest * [admin] build fix * [admin] optimise sql as per review comment * Update upload-pupil-census.ejs Add csrf token back in Co-authored-by: Mohsen Qureshi <[email protected]>
- Loading branch information
1 parent
dc4f0da
commit a90b87f
Showing
27 changed files
with
2,098 additions
and
1,167 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
71 changes: 71 additions & 0 deletions
71
admin/services/data-access/organisation-bulk-upload.data.service.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| const sqlService = require('./sql.service') | ||
| const uuid = require('uuid') | ||
| const R = require('ramda') | ||
|
|
||
| const organisationBulkUploadDataService = { | ||
| /** | ||
| * Determine if an existing organisation upload job is running. | ||
| * @returns {Promise<boolean>} | ||
| */ | ||
| isExistingJob: async function isExistingJob () { | ||
| const sql = `SELECT COUNT(*) as count | ||
| FROM mtc_admin.job j | ||
| join mtc_admin.jobType jt on (j.jobType_id = jt.id) | ||
| JOIN mtc_admin.jobStatus js on (j.jobStatus_id = js.id) | ||
| WHERE js.jobStatusCode IN ('SUB', 'PRC') | ||
| AND jt.jobTypeCode = 'ORG'` | ||
| const res = await sqlService.query(sql) | ||
| const count = R.propOr(0, 'count', res[0]) | ||
| return count > 0 | ||
| }, | ||
|
|
||
| /** | ||
| * Create a new job record for Organisation file upload | ||
| * @returns {Promise<undefined | string>} The newly inserted ID of the job record | ||
| */ | ||
| createJobRecord: async function createJobRecord () { | ||
| if (await this.isExistingJob()) { | ||
| throw new Error('A job is already running') | ||
| } | ||
|
|
||
| const sql = ` | ||
| INSERT INTO mtc_admin.job (jobStatus_id, jobType_id, jobInput) | ||
| OUTPUT INSERTED.urlSlug | ||
| VALUES ((SELECT id from mtc_admin.jobStatus where jobStatusCode = 'SUB'), | ||
| (SELECT id from mtc_admin.jobType where jobTypeCode = 'ORG'), | ||
| 'File upload') | ||
| ` | ||
| const retVal = await sqlService.query(sql) | ||
| return retVal[0].urlSlug | ||
| }, | ||
|
|
||
| /** | ||
| * Return the job record by urlSlug | ||
| * @param jobSlug | ||
| * @returns {Promise<Object>} | ||
| */ | ||
| sqlGetJobData: async function sqlGetJobData (jobSlug) { | ||
| if (!uuid.validate(jobSlug)) { | ||
| throw new Error(`Invalid UUID for jobSlug: ${jobSlug}`) | ||
| } | ||
| const sql = ` | ||
| SELECT j.*, | ||
| jt.description as jobTypeDescription, | ||
| jt.jobTypeCode, | ||
| js.description as jobStatusDescription, | ||
| js.jobStatusCode | ||
| FROM mtc_admin.job j | ||
| JOIN mtc_admin.jobType jt on (j.jobType_id = jt.id) | ||
| JOIN mtc_admin.jobStatus js on (j.jobStatus_id = js.id) | ||
| AND jt.jobTypeCode = 'ORG' | ||
| AND j.urlSlug = @slug | ||
| ` | ||
| const params = [ | ||
| { name: 'slug', value: jobSlug, type: sqlService.TYPES.UniqueIdentifier } | ||
| ] | ||
| const data = await sqlService.query(sql, params) | ||
| return R.head(data) | ||
| } | ||
| } | ||
|
|
||
| module.exports = organisationBulkUploadDataService |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| const azureBlobDataService = require('./data-access/azure-blob.data.service') | ||
| const fileValidator = require('../lib/validator/file-validator.js') | ||
| const organisationBulkUploadDataService = require('./data-access/organisation-bulk-upload.data.service') | ||
| const AdmZip = require('adm-zip') | ||
|
|
||
| // Files get uploaded to this container. dns naming conventions. | ||
| const container = 'school-import' | ||
|
|
||
| const organisationBulkUploadService = { | ||
| /** | ||
| * Validate an upload file for basic errors | ||
| * @param uploadFile | ||
| * @returns {Promise<*>} | ||
| */ | ||
| validate: function validate (uploadFile) { | ||
| return fileValidator.validate(uploadFile, 'file-upload') | ||
| }, | ||
|
|
||
| /** | ||
| * | ||
| * @param {{ file: string, filename: string }} uploadFile | ||
| * @returns {Promise<string>} job slug UUID | ||
| */ | ||
| upload: async function upload (uploadFile) { | ||
| const remoteFilename = uploadFile.filename | ||
| await azureBlobDataService.createContainerIfNotExistsAsync(container) | ||
|
|
||
| // Create the job record first as it acts as a singleton - we only want one file upload at a time. If we can't | ||
| // create the job record, then we abort. | ||
| const jobSlug = await organisationBulkUploadDataService.createJobRecord() | ||
| await azureBlobDataService.createBlockBlobFromLocalFileAsync(container, remoteFilename, uploadFile.file) | ||
| return jobSlug | ||
| }, | ||
|
|
||
| /** | ||
| * Get the status of the file upload | ||
| * @param jobSlug uuid | ||
| * @returns {Promise<{urlSlug: (string|*), code: (string|*), description: (string|*), errorOutput: (string|*), jobOutput: any}>} | ||
| */ | ||
| getUploadStatus: async function getUploadStatus (jobSlug) { | ||
| const jobData = await organisationBulkUploadDataService.sqlGetJobData(jobSlug) | ||
| if (jobData === undefined) { | ||
| throw new Error('Job ID not found') | ||
| } | ||
| return { | ||
| description: jobData.jobStatusDescription, | ||
| code: jobData.jobStatusCode, | ||
| errorOutput: jobData.errorOutput, | ||
| jobOutput: JSON.parse(jobData.jobOutput), | ||
| urlSlug: jobData.urlSlug | ||
| } | ||
| }, | ||
|
|
||
| /** | ||
| * Return a buffer containing the Zipped data (error.txt, output.txt) for the service-manager to download. | ||
| * @param jobSlug | ||
| * @returns {Promise<Buffer>} | ||
| */ | ||
| getZipResults: async function getZipResults (jobSlug) { | ||
| if (jobSlug === undefined) { | ||
| throw new Error('Missing job ID') | ||
| } | ||
| const jobData = await this.getUploadStatus(jobSlug) | ||
| const zip = new AdmZip() | ||
| // noinspection JSCheckFunctionSignatures - 3rd and 4th args are optional | ||
| zip.addFile('error.txt', jobData.jobOutput.stderr.join('\n')) | ||
| // noinspection JSCheckFunctionSignatures - 3rd and 4th args are optional | ||
| zip.addFile('output.txt', jobData.jobOutput.stdout.join('\n')) | ||
| return zip.toBuffer() | ||
| } | ||
| } | ||
|
|
||
| module.exports = organisationBulkUploadService |
Oops, something went wrong.