feat: integrated @spamscanner for phishing, viruses, executables, and arbitary filtering

Author: niftylettuce

app.js

@@ -16,8 +16,8 @@ if (!module.parent) {
 
   (async () => {
     try {
+      await app.spamscanner.load();
       await app.listen(port);
-      // await app.spamscanner.load()
       if (process.send) process.send('ready');
       logger.info(
         `ForwardEmail server listening on ${

index.js

@@ -3,17 +3,15 @@ const dns = require('dns');
 const fs = require('fs');
 const util = require('util');
 
-// const SpamScanner = require('spamscanner');
-
 const DKIM = require('nodemailer/lib/dkim');
 const Limiter = require('ratelimiter');
 const MimeNode = require('nodemailer/lib/mime-node');
 const RE2 = require('re2');
 const Redis = require('@ladjs/redis');
+const SpamScanner = require('spamscanner');
 const _ = require('lodash');
 const addressParser = require('nodemailer/lib/addressparser');
 const arrayJoinConjunction = require('array-join-conjunction');
-const zoneMTABounces = require('zone-mta/lib/bounces');
 const bytes = require('bytes');
 const dnsbl = require('dnsbl');
 const getFQDN = require('get-fqdn');
@@ -31,6 +29,7 @@ const sharedConfig = require('@ladjs/shared-config');
 const splitLines = require('split-lines');
 const superagent = require('superagent');
 const validator = require('validator');
+const zoneMTABounces = require('zone-mta/lib/bounces');
 const { Iconv } = require('iconv');
 const { SMTPServer } = require('smtp-server');
 const { SRS } = require('sender-rewriting-scheme');
@@ -257,7 +256,7 @@ class ForwardEmail {
       },
       sendingZone: 'bounces',
       userAgent: `${pkg.name}/${pkg.version}`,
-      // spamScanner: {},
+      spamScanner: {},
       ttlMs: ms('7d'),
       maxRetry: 5,
       messageIdDomain: env.MESSAGE_ID_DOMAIN,
@@ -341,7 +340,7 @@ class ForwardEmail {
     });
 
     // expose spamscanner
-    // this.scanner = new SpamScanner(this.config.spamScanner);
+    this.scanner = new SpamScanner(this.config.spamScanner);
 
     this.listen = this.listen.bind(this);
     this.close = this.close.bind(this);
@@ -958,7 +957,7 @@ class ForwardEmail {
       // 2) X ensure all email headers were parsed
       // 3) X reverse SRS bounces
       // 4) X prevent replies to [email protected] (no bottleneck)
-      // 5) O check for spam
+      // 5) X check for spam
       // 6) X validate SPF, DKIM, DMARC, and ARC
       // 7) X lookup forwarding recipients recursively
       // 8) X normalize recipients by host and without "+" symbols
@@ -1065,52 +1064,49 @@ class ForwardEmail {
         //
         // 5) check for spam
         //
-        /*
-        let results;
+        let scan;
         try {
-          results = await this.scanner.scan(originalRaw);
+          scan = await this.scanner.scan(originalRaw);
+          if (scan.is_spam)
+            this.config.logger.fatal(`spam detected: ${JSON.stringify(scan.results)}`);
         } catch (err) {
-          logger.error(err);
+          this.config.logger.fatal(err);
         }
 
-        if (results) {
-          if (results.is_spam)
-            logger.error('spam detected', {
-              originalRaw: originalRaw.toString(),
-              results
-            });
-
+        if (_.isObject(scan) && _.isObject(scan.results)) {
           //
           // NOTE: until we are confident with the accuracy
           // we are not utilizing classification right now
           // however we still want to use other detections
           //
           const messages = [];
 
-          if (
-            _.isObject(results.phishing) &&
-            _.isArray(results.phishing.messages)
-          )
-            for (const message of results.phishing.messages) {
+          if (_.isArray(scan.results.phishing))
+            for (const message of scan.results.phishing) {
+              messages.push(message);
+            }
+
+          if (_.isArray(scan.results.executables)) {
+            for (const message of scan.results.executables) {
               messages.push(message);
             }
+          }
 
-          if (_.isArray(results.executables)) {
-            for (const message of results.executables) {
+          if (_.isArray(scan.results.arbitrary)) {
+            for (const message of scan.results.arbitrary) {
               messages.push(message);
             }
           }
 
-          if (_.isArray(results.arbitrary)) {
-            for (const message of results.arbitrary) {
+          if (_.isArray(scan.results.viruses)) {
+            for (const message of scan.results.viruses) {
               messages.push(message);
             }
           }
 
           if (messages.length > 0)
             throw new CustomError(messages.join(' '), 554);
         }
-        */
 
         //
         // 6) validate SPF, DKIM, DMARC, and ARC
@@ -1820,7 +1816,7 @@ class ForwardEmail {
         err.message = err.message.split('msg:')[1];
       }
 
-      err.message += ` If you need help please forward this email to ${this.config.email} or visit ${this.config.website}`;
+      err.message += ` If you need help please forward this email to ${this.config.email} or visit ${this.config.website}.`;
       fn(err);
     });
 
@@ -1976,6 +1972,7 @@ class ForwardEmail {
     if (verifications.length > 0) {
       if (verifications.length > 1)
         throw new CustomError(
+          // TODO: we may want to replace this with "Invalid Recipients"
           `Domain ${domain} has multiple verification TXT records of "${this.config.recordPrefix}-site-verification" and should only have one`
         );
       // if there was a verification record then perform lookup
@@ -2007,6 +2004,7 @@ class ForwardEmail {
     // if the record was blank then throw an error
     if (!isSANB(record))
       throw new CustomError(
+        // TODO: we may want to replace this with "Invalid Recipients"
         `${address} domain of ${domain} has a blank "${this.config.recordPrefix}" TXT record or has zero aliases configured`,
         420
       );
@@ -2025,6 +2023,7 @@ class ForwardEmail {
 
     if (addresses.length === 0)
       throw new CustomError(
+        // TODO: we may want to replace this with "Invalid Recipients"
         `${address} domain of ${domain} has zero forwarded addresses configured in the TXT record with "${this.config.recordPrefix}"`,
         420
       );
@@ -2083,6 +2082,7 @@ class ForwardEmail {
             !validator.isURL(addr[1], this.config.isURLOptions))
         )
           throw new CustomError(
+            // TODO: we may want to replace this with "Invalid Recipients"
             `${lowerCaseAddress} domain of ${domain} has an invalid "${this.config.recordPrefix}" TXT record due to an invalid email address of "${element}"`
           );
 

package.json

@@ -41,7 +41,7 @@
     "dnsbl": "^3.2.0",
     "get-fqdn": "^0.0.4",
     "get-port": "^5.1.1",
-    "get-stream": "^5.1.0",
+    "get-stream": "^5.2.0",
     "iconv": "^3.0.0",
     "ip": "^1.1.5",
     "is-string-and-not-blank": "^0.0.2",
@@ -60,6 +60,7 @@
     "sender-rewriting-scheme": "^1.0.0",
     "signale": "^1.4.0",
     "smtp-server": "^3.7.0",
+    "spamscanner": "^0.0.8",
     "split-lines": "^2.0.0",
     "superagent": "^6.0.0",
     "uuid-by-string": "^3.0.2",

test/fixtures/eicar.com.txt

@@ -0,0 +1 @@
+X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*

test/fixtures/executable.eml

@@ -0,0 +1,22 @@
+Return-Path: <[email protected]>
+To: [email protected]
+From: [email protected]
+Subject: test Sat, 26 Jan 2019 12:04:58 +0100
+Message-Id: <[email protected]>
+Date: Sat, 26 Jan 2019 12:04:58 +0100
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="----=_MIME_BOUNDARY_000_15328"
+
+------=_MIME_BOUNDARY_000_15328
+Content-Type: text/plain
+
+This is a test mailing
+------=_MIME_BOUNDARY_000_15328
+Content-Type: application/octet-stream; name="hello_world.exe"
+Content-Description: hello_world.exe
+Content-Disposition: attachment; filename="hello_world.exe"
+Content-Transfer-Encoding: BASE64
+
+f0VMRgIBAQAAAAAAAAAAAAMAPgABAAAAUBAAAAAAAABAAAAAAAAAAGA5AAAAAAAAAAAAAEAAOAAL
+
+------=_MIME_BOUNDARY_000_15328--

test/fixtures/idn.eml

@@ -0,0 +1,4 @@
+From: [email protected]
+Content-Type: text/plain
+
+http://мойсайт.рф

test/fixtures/phishing.eml

@@ -0,0 +1,3 @@
+Content-type: text/html
+
+lol <a href="http://www.example.net">http://www.myspace.com</a>