Merge pull request #2709 from gchq/dev/improve-scanner-logs

Improve Scanner logs & update config

Author: PE39806

backend/config/default.cjs

@@ -184,13 +184,13 @@ module.exports = {
 
   avScanning: {
     clamdscan: {
-      concurrency: 4,
+      concurrency: 2,
       host: '127.0.0.1',
       port: 3310,
     },
 
     modelscan: {
-      concurrency: 4,
+      concurrency: 2,
       protocol: 'http',
       host: '127.0.0.1',
       port: 3311,

backend/src/connectors/fileScanning/Base.ts

@@ -49,7 +49,7 @@ export abstract class BaseQueueFileScanningConnector extends BaseFileScanningCon
   abstract _scan(file: FileInterface): Promise<FileScanResult[]>
 
   async scan(file: FileInterface): Promise<FileScanResult[]> {
-    log.debug({ file, toolName: this.toolName, ...(this.version && { version: this.version }) }, 'Queueing scan.')
+    log.debug({ file, ...this.info(), queueSize: this.queue.size }, 'Queueing scan.')
     const scanResult = await this.queue
       .add(async () => this._scan(file))
       .catch((error) => {

backend/src/connectors/fileScanning/clamAv.ts

@@ -34,30 +34,30 @@ export class ClamAvFileScanningConnector extends BaseQueueFileScanningConnector
     this.av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan })
     const scannerVersion = await this.av.getVersion()
     this.version = safeParseVersion(scannerVersion)
-    log.debug({ version: this.version }, 'Initialised Clam AV scanner')
+    log.debug({ ...this.info() }, 'Initialised Clam AV scanner')
     return this
   }
 
   async _scan(file: FileInterfaceDoc): Promise<FileScanResult[]> {
+    const scannerInfo = this.info()
     if (!this.av) {
-      return await this.scanError(`Could not use ${this.toolName} as it is not been correctly initialised.`)
+      return await this.scanError(`Could not use ${this.toolName} as it is not been correctly initialised.`, {
+        ...scannerInfo,
+      })
     }
 
     const getObjectStreamResponse = await getObjectStream(file.path)
     const s3Stream = getObjectStreamResponse.Body as Readable | null
     if (!s3Stream) {
-      return await this.scanError(`Stream for file ${file.path} is not available`)
+      return await this.scanError(`Stream for file ${file.path} is not available`, { file, ...scannerInfo })
     }
 
     try {
       const { isInfected, viruses } = await this.av.scanStream(s3Stream)
-      log.info(
-        { modelId: file.modelId, fileId: file._id.toString(), name: file.name, result: { isInfected, viruses } },
-        'Scan complete.',
-      )
+      log.debug({ file, result: { isInfected, viruses }, ...scannerInfo }, 'Scan complete.')
       return [
         {
-          ...this.info(),
+          ...scannerInfo,
           state: ScanState.Complete,
           isInfected,
           viruses,
@@ -68,6 +68,7 @@ export class ClamAvFileScanningConnector extends BaseQueueFileScanningConnector
       return this.scanError(`This file could not be scanned due to an error caused by ${this.toolName}`, {
         error,
         file,
+        ...scannerInfo,
       })
     } finally {
       if (s3Stream) {

backend/src/connectors/fileScanning/modelScan.ts

@@ -27,13 +27,13 @@ export class ModelScanFileScanningConnector extends BaseQueueFileScanningConnect
     await this.init()
     const scannerInfo = this.info()
     if (!scannerInfo.scannerVersion) {
-      return await this.scanError('Could not use ModelScan as it is not running.')
+      return await this.scanError('Could not use ModelScan as it is not running.', { ...scannerInfo })
     }
 
     const getObjectStreamResponse = await getObjectStream(file.path)
     const s3Stream = getObjectStreamResponse.Body as Readable | null
     if (!s3Stream) {
-      return await this.scanError(`Stream for file ${file.path} is not available`)
+      return await this.scanError(`Stream for file ${file.path} is not available`, { file, ...scannerInfo })
     }
 
     try {
@@ -43,6 +43,7 @@ export class ModelScanFileScanningConnector extends BaseQueueFileScanningConnect
         return this.scanError(`This file could not be scanned due to an error caused by ${this.toolName}`, {
           errors: scanResults.errors,
           file,
+          ...scannerInfo,
         })
       }
 
@@ -51,10 +52,7 @@ export class ModelScanFileScanningConnector extends BaseQueueFileScanningConnect
       const viruses: string[] = isInfected
         ? scanResults.issues.map((issue) => `${issue.severity}: ${issue.description}. ${issue.scanner}`)
         : []
-      log.info(
-        { modelId: file.modelId, fileId: file._id.toString(), name: file.name, result: { isInfected, viruses } },
-        'Scan complete.',
-      )
+      log.debug({ file, result: { isInfected, viruses }, ...scannerInfo }, 'Scan complete.')
       return [
         {
           ...scannerInfo,

infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml

@@ -64,11 +64,13 @@ data:
 
       avScanning: {
         clamdscan: {
+          concurrency: '{{ .Values.clamav.concurrency }}'
           host: '{{ include "bailo.fullname" . }}-clamav',
           port: {{ .Values.clamav.port }},
         },
 
         modelscan: {
+          concurrency: '{{ .Values.modelscan.concurrency }}'
           host: '{{ include "bailo.fullname" . }}-modelscan',
           port: {{ .Values.modelscan.port }},
           protocol: '{{ .Values.modelscan.protocol }}',

infrastructure/helm/bailo/values.yaml

@@ -328,13 +328,15 @@ clamav:
   enabled: false
   runAsUser: 1002
   image: clamav/clamav:1.4.3_base # https://docs.clamav.net/manual/Installing/Docker.html#the-official-images-on-docker-hub
+  concurrency: 2
   port: 3310
   accessModes:
     - ReadWriteOnce
   size: 10G
 
 modelscan:
   enabled: false
+  concurrency: 2
   protocol: http
   port: 3311
   accessModes: