diff --git a/.gitignore b/.gitignore index 4558ddfc..8fab1d84 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ yarn-error.log* /frontend/.env /dist/ /local/ +/locals3root \ No newline at end of file diff --git a/README.md b/README.md index c305b9f2..d1f8c9e0 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ La création des tables et autres structures SQL se fera automatiquement lors du Pour le stockage des fichiers, l'application utilise un service S3. En local et pour les tests, il est possible d'utiliser https://github.com/adobe/S3Mock +```bash +docker compose up -d +``` ### Installation de l'application diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..1d671007 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +#s3cmd --host http://localhost:9090 ls s3://maestro --no-ssl +services: + s3mock: + image: adobe/s3mock:latest + container_name: maestro_s3 + environment: + - debug=true + - retainFilesOnExit=true + - root=containers3root + - initialBuckets=maestro + ports: + - 9090:9090 + volumes: + - ./locals3root:/containers3root \ No newline at end of file diff --git a/server/controllers/documentController.ts b/server/controllers/documentController.ts index ceb2385b..63a93afd 100644 --- a/server/controllers/documentController.ts +++ b/server/controllers/documentController.ts @@ -1,14 +1,12 @@ import { DeleteObjectCommand, GetObjectCommand, - PutObjectCommand, S3, } from '@aws-sdk/client-s3'; import { getSignedUrl as getS3SignedUrl } from '@aws-sdk/s3-request-presigner'; import { Request, Response } from 'express'; import { AuthenticatedRequest } from 'express-jwt'; import { constants } from 'http2'; -import { v4 as uuidv4 } from 'uuid'; import DocumentMissingError from '../../shared/errors/documentMissingError'; import { Document, @@ -17,6 +15,7 @@ import { import { hasPermission } from '../../shared/schema/User/User'; import documentRepository from '../repositories/documentRepository'; import config from '../utils/config'; +import { getUploadSignedUrlS3 } from '../services/s3Service'; const getDocument = async (request: Request, response: Response) => { const { documentId } = request.params; @@ -37,30 +36,19 @@ const getUploadSignedUrl = async (request: Request, response: Response) => { const user = (request as AuthenticatedRequest).user; if (kind === 'Resource' && !hasPermission(user, 'createResource')) { - return response.sendStatus(constants.HTTP_STATUS_FORBIDDEN); + return { status: constants.HTTP_STATUS_FORBIDDEN} } if ( kind === 'AnalysisReportDocument' && !hasPermission(user, 'createAnalysis') ) { - return response.sendStatus(constants.HTTP_STATUS_FORBIDDEN); + return { status: constants.HTTP_STATUS_FORBIDDEN } } - console.log('Get signed url for file', filename); - - const client = new S3(config.s3.client); - const id = uuidv4(); - const key = `${id}_${filename}`; + const result: {url: string, documentId: string} = await getUploadSignedUrlS3(filename ) - const command = new PutObjectCommand({ - Bucket: config.s3.bucket, - Key: key, - }); - - const url = await getS3SignedUrl(client, command, { expiresIn: 3600 }); - - response.status(200).json({ url, documentId: id }); -}; + return response.status(200).json(result) +} const getDownloadSignedUrl = async (request: Request, response: Response) => { const { documentId } = request.params; diff --git a/server/repositories/kysely.type.ts b/server/repositories/kysely.type.ts index 2e8160f9..7849958e 100644 --- a/server/repositories/kysely.type.ts +++ b/server/repositories/kysely.type.ts @@ -6,6 +6,7 @@ import type { ColumnType } from "kysely"; import { UserRole } from '../../shared/schema/User/UserRole'; import { Region } from '../../shared/referential/Region'; +import { DocumentKind } from '../../shared/schema/Document/DocumentKind'; export type Generated = T extends ColumnType ? ColumnType @@ -90,7 +91,7 @@ export interface Documents { createdBy: string | null; filename: string; id: Generated; - kind: string; + kind: DocumentKind; } export interface KnexMigrations { diff --git a/server/services/imapService/girpa.ts b/server/services/imapService/girpa.ts index 632f106a..719198e2 100644 --- a/server/services/imapService/girpa.ts +++ b/server/services/imapService/girpa.ts @@ -89,7 +89,7 @@ export const analyseXmlValidator = z.object({ Substance_active_anglais: residueEnglishNameValidator }); // Visible for testing -export const extractSample = (obj: unknown): ExportSample[] | null => { +export const extractSample = (obj: unknown): Omit[] | null => { const echantillonValidator = z.object({ Code_échantillon: z.string(), Commentaire: z.string(), @@ -155,12 +155,35 @@ const exportDataFromEmail: ExportDataFromEmail = (email) => { if (xmlFile !== undefined) { const parser = new XMLParser(); const obj = parser.parse(xmlFile.content); - return extractSample(obj); + + const extractAnalyse = extractSample(obj) + + if( extractAnalyse === null ){ + //FIXME error + return null + } + + const analyseWithPdf: ExportSample[] = [] + + for (const analyse of extractAnalyse) { + const pdfAttachment = email.attachments.find(({ contentType, filename }) => contentType === 'application/pdf' && filename?.startsWith(analyse.sampleReference)) + + if (pdfAttachment === undefined) { + //FIXME error + return null + } + + const pdfFile: File = new File([pdfAttachment.content], pdfAttachment.filename ?? ''); + analyseWithPdf.push({...analyse, pdfFile}) + } + + + return analyseWithPdf } else { console.log('Aucun XML', email.attachments); + return null } - return null; }; export const girpaConf: LaboratoryConf = { diff --git a/server/services/imapService/index.ts b/server/services/imapService/index.ts index cebeaf76..d558bfde 100644 --- a/server/services/imapService/index.ts +++ b/server/services/imapService/index.ts @@ -2,29 +2,39 @@ import { ImapFlow } from 'imapflow'; import { isNull } from 'lodash'; import { ParsedMail, simpleParser } from 'mailparser'; import { LaboratoryName } from '../../../shared/referential/Laboratory'; +import { Analyte } from '../../../shared/referential/Residue/Analyte'; +import { ComplexResidue } from '../../../shared/referential/Residue/ComplexResidue'; +import { SimpleResidue } from '../../../shared/referential/Residue/SimpleResidue'; +import { Sample } from '../../../shared/schema/Sample/Sample'; +import { initKysely, kysely } from '../../repositories/kysely'; import config from '../../utils/config'; +import { getUploadSignedUrlS3 } from '../s3Service'; import { girpaConf } from './girpa'; -import { Sample } from '../../../shared/schema/Sample/Sample'; -import { SimpleResidue } from '../../../shared/referential/Residue/SimpleResidue'; -import { ComplexResidue } from '../../../shared/referential/Residue/ComplexResidue'; -import { Analyte } from '../../../shared/referential/Residue/Analyte'; const laboratoriesWithConf = ['GIR 49'] as const satisfies LaboratoryName[]; type LaboratoryWithConf = (typeof laboratoriesWithConf)[number]; export type ExportResidue = -{ value: SimpleResidue, kind: 'SimpleResidue' } | -{ value:ComplexResidue, kind: 'ComplexResidue' } | -{ value: Analyte, kind: 'Analyte' } + | { value: SimpleResidue; kind: 'SimpleResidue' } + | { value: ComplexResidue; kind: 'ComplexResidue' } + | { value: Analyte; kind: 'Analyte' }; -export type ExportDataSubstance = {substance: ExportResidue} & ( {result_kind: 'NQ', result: null, lmr: null} | {result_kind: 'Q', result: number, lmr: number}) -export type IsSender = (senderAddress: string) => boolean +export type ExportDataSubstance = { substance: ExportResidue } & ( + | { result_kind: 'NQ'; result: null; lmr: null } + | { + result_kind: 'Q'; + result: number; + lmr: number; + } +); +export type IsSender = (senderAddress: string) => boolean; export type ExportSample = { - sampleReference: Sample['reference'], - notes: string, - substances: ExportDataSubstance[] + sampleReference: Sample['reference']; + notes: string; + pdfFile: File; + substances: ExportDataSubstance[]; }; -export type ExportDataFromEmail = (email: ParsedMail) => null | ExportSample[] +export type ExportDataFromEmail = (email: ParsedMail) => null | ExportSample[]; export type LaboratoryConf = { isSender: IsSender; @@ -103,7 +113,7 @@ const run = async () => { } } for (const message of messagesToRead) { - const messageUid: string = `${message.messageUid}` + const messageUid: string = `${message.messageUid}`; //undefined permet de récupérer tout l'email const downloadObject = await client.download(messageUid, undefined, { uid: true @@ -114,12 +124,47 @@ const run = async () => { //FIXME trash // await client.messageMove(messageUid, config.inbox.trashboxName, {uid: true}) - const data = laboratoriesConf[message.laboratoryName].exportDataFromEmail( - parsed - ); + const data = + laboratoriesConf[message.laboratoryName].exportDataFromEmail(parsed); - console.log(JSON.stringify(data, null, 4)) - // createWriteStream(parsed.attachments[2].filename ?? '').write(parsed.attachments[2].content) + if (data !== null) { + initKysely(config.databaseUrl) + for (const analyse of data) { + const { url, documentId } = await getUploadSignedUrlS3( + analyse.pdfFile.name + ); + + const uploadResult = await fetch(url, { + method: 'PUT', + body: analyse.pdfFile + }); + if (!uploadResult.ok) { + //FIXME + // return { + // error: { + // status: uploadResult.status, + // data: await uploadResult.json(), + // } as FetchBaseQueryError, + // }; + } + await kysely.transaction().execute(async (trx) => { + await trx + .insertInto('documents') + .values({ + id: documentId, + filename: analyse.pdfFile.name, + kind: 'AnalysisReportDocument', + createdAt: new Date(), + createdBy: null + }) + .execute(); + }); + console.log(JSON.stringify(data, null, 4)); + // createWriteStream(parsed.attachments[2].filename ?? '').write(parsed.attachments[2].content) + } + } else { + //FIXME + } } } } catch (e) { diff --git a/server/services/s3Service.ts b/server/services/s3Service.ts new file mode 100644 index 00000000..bbb8a065 --- /dev/null +++ b/server/services/s3Service.ts @@ -0,0 +1,31 @@ +import { DocumentToCreate } from '../../shared/schema/Document/Document'; +import { PutObjectCommand, S3 } from '@aws-sdk/client-s3'; +import config from '../utils/config'; +import { v4 as uuidv4 } from 'uuid'; + +import { getSignedUrl as getS3SignedUrl } from '@aws-sdk/s3-request-presigner'; +export const getUploadSignedUrlS3 = async (filename: DocumentToCreate['filename']): Promise<{url: string, documentId: string}> => { + + + console.log('Get signed url for file', filename); + + const client = new S3({ + ...config.s3.client, + //FIXME forcePathStyle for S3Mock + forcePathStyle: true + } + ); + const id = uuidv4(); + const key = `${id}_${filename}`; + + const command = new PutObjectCommand({ + Bucket: config.s3.bucket, + Key: key, + }); + + const url = await getS3SignedUrl(client, command, { expiresIn: 3600 }); + + + + return {url, documentId: id} +};