From e76f9a1b175f08d976582fa5449cadd464ceaab7 Mon Sep 17 00:00:00 2001 From: Matt Simerson Date: Wed, 10 Apr 2024 22:15:07 -0700 Subject: [PATCH] dnsbl, dnswl, backscatterer -> dns-list --- config/dnsbl.ini | 23 --- config/plugins | 4 +- docs/{plugins => deprecated}/dnsbl.md | 0 docs/{plugins => deprecated}/dnswl.md | 0 package.json | 1 + plugins.js | 4 + plugins/backscatterer.js | 25 --- plugins/data.uribl.js | 4 - plugins/dnsbl.js | 146 --------------- plugins/dnswl.js | 58 ------ tests/plugins/dns_list_base.js | 259 -------------------------- tests/plugins/dnsbl.js | 101 ---------- 12 files changed, 7 insertions(+), 618 deletions(-) delete mode 100644 config/dnsbl.ini rename docs/{plugins => deprecated}/dnsbl.md (100%) rename docs/{plugins => deprecated}/dnswl.md (100%) delete mode 100644 plugins/backscatterer.js delete mode 100644 plugins/data.uribl.js delete mode 100644 plugins/dnsbl.js delete mode 100644 plugins/dnswl.js delete mode 100644 tests/plugins/dns_list_base.js delete mode 100644 tests/plugins/dnsbl.js diff --git a/config/dnsbl.ini b/config/dnsbl.ini deleted file mode 100644 index 717f5adef..000000000 --- a/config/dnsbl.ini +++ /dev/null @@ -1,23 +0,0 @@ - -; reject: (default: true) -; denies connections from IPs on any active DNSBL -reject=true - -; periodically check each DNSBL, disabling ones that fail checks -periodic_checks = 30 - -; search: Default (first) -; first: consider first DNSBL response conclusive. End processing. -; all: process all DNSBL results -search=first - -; enable_stats (Default: false) -; stores stats in a Redis DB (see plugins/dns_list_base) -;enable_stats=true - -; stats_redis_host (Default: localhost) - - -; zones: a comma separated list of DNSBL zones -; or list DNSBL zones in config/dnsbl.zones -zones=zen.spamhaus.org diff --git a/config/plugins b/config/plugins index 08b13b3c0..a362bb3de 100644 --- a/config/plugins +++ b/config/plugins @@ -23,8 +23,8 @@ # geoip # asn # fcrdns -# block mails from known bad hosts (see config/dnsbl.zones for the DNS zones queried) -dnsbl +# block & allow lists (see config/dns-list.ini) +dns-list # HELO #early_talker diff --git a/docs/plugins/dnsbl.md b/docs/deprecated/dnsbl.md similarity index 100% rename from docs/plugins/dnsbl.md rename to docs/deprecated/dnsbl.md diff --git a/docs/plugins/dnswl.md b/docs/deprecated/dnswl.md similarity index 100% rename from docs/plugins/dnswl.md rename to docs/deprecated/dnswl.md diff --git a/package.json b/package.json index e3406a7d3..3974c8dcc 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "haraka-plugin-asn": "^2.0.2", "haraka-plugin-auth-ldap": "^1.1.0", "haraka-plugin-dcc": "^1.0.2", + "haraka-plugin-dns-list": "^1.1.0", "haraka-plugin-elasticsearch": "^8.0.2", "haraka-plugin-fcrdns": "^1.1.0", "haraka-plugin-graph": "^1.0.5", diff --git a/plugins.js b/plugins.js index 261fe926a..76dea95b2 100644 --- a/plugins.js +++ b/plugins.js @@ -343,6 +343,7 @@ plugins.load_plugins = override => { } plugins.deprecated = { + 'backscatterer' : 'dns-list', 'connect.asn' : 'asn', 'connect.fcrdns' : 'fcrdns', 'connect.geoip' : 'geoip', @@ -352,6 +353,9 @@ plugins.deprecated = { 'data.noreceived' : 'headers', 'data.rfc5322_header_checks': 'headers', 'data.headers' : 'headers', + 'data.uribl' : 'uribl', + 'dnsbl' : 'dns-list', + 'dnswl' : 'dns-list', 'log.syslog' : 'syslog', 'mail_from.access' : 'access', 'mail_from.blocklist' : 'access', diff --git a/plugins/backscatterer.js b/plugins/backscatterer.js deleted file mode 100644 index 87f9dd6ab..000000000 --- a/plugins/backscatterer.js +++ /dev/null @@ -1,25 +0,0 @@ -// backscatterer plugin - -exports.register = function () { - this.inherits('dns_list_base'); -} - -exports.hook_mail = function (next, connection, params) { - const user = ((params[0]?.user) ? - params[0].user.toLowerCase() : null); - if (!(!user || user === 'postmaster')) return next(); - // Check remote IP on ips.backscatterer.org - const plugin = this; - - function resultCb (err, zone, a) { - if (err) { - connection.logerror(plugin, err); - return next(); - } - if (!a) return next(); - const msg = `Host ${connection.remote.host} [${connection.remote.ip}] is blacklisted by ${zone}`; - return next(DENY, msg); - } - - this.first(connection.remote.ip, [ 'ips.backscatterer.org' ], resultCb); -} diff --git a/plugins/data.uribl.js b/plugins/data.uribl.js deleted file mode 100644 index 487224d22..000000000 --- a/plugins/data.uribl.js +++ /dev/null @@ -1,4 +0,0 @@ - -exports.register = function () { - this.logerror(this, "data.uribl was moved to haraka-plugin-uribl. Update the plugin name in config/plugins to just 'uribl'"); -} diff --git a/plugins/dnsbl.js b/plugins/dnsbl.js deleted file mode 100644 index f73fc394c..000000000 --- a/plugins/dnsbl.js +++ /dev/null @@ -1,146 +0,0 @@ -// dnsbl plugin - -exports.register = function () { - this.inherits('dns_list_base'); - - this.load_config(); - - if (this.cfg.main.periodic_checks) { - this.check_zones(this.cfg.main.periodic_checks); - } - - if (this.cfg.main.search === 'all') { - this.register_hook('connect', 'connect_multi'); - } - else { - this.register_hook('connect', 'connect_first'); - } -} - -exports.load_config = function () { - - this.cfg = this.config.get('dnsbl.ini', { - booleans: ['+main.reject', '-main.enable_stats'], - }, () => { - this.load_config(); - }); - - if (this.cfg.main.enable_stats && !this.enable_stats) { - this.loginfo('stats reporting enabled'); - this.enable_stats = true; - } - if (!this.cfg.main.enable_stats && this.enable_stats) { - this.loginfo('stats reporting disabled'); - this.enable_stats = false; - } - - if (this.cfg.main.stats_redis_host && - this.cfg.main.stats_redis_host !== this.redis_host) { - this.redis_host = this.cfg.main.stats_redis_host; - this.loginfo(`set stats redis host to: ${this.redis_host}`); - } - - this.get_uniq_zones(); -} - -exports.get_uniq_zones = function () { - this.zones = []; - - const unique_zones = {}; - - // Compatibility with old plugin - const legacy_zones = this.config.get('dnsbl.zones', 'list'); - for (const legacyZone of legacy_zones) { - unique_zones[legacyZone] = true; - } - - if (this.cfg.main.zones) { - const new_zones = this.cfg.main.zones.split(/[\s,;]+/); - for (const newZone of new_zones) { - unique_zones[newZone] = true; - } - } - - for (const key in unique_zones) { this.zones.push(key); } - return this.zones; -} - -exports.should_skip = function (connection) { - - if (!connection) { return true; } - - if (connection.remote.is_private) { - connection.logdebug(this, `skip private: ${connection.remote.ip}`); - return true; - } - - if (!this.zones || !this.zones.length) { - connection.logerror(this, "no zones"); - return true; - } - - return false; -} - -exports.connect_first = function (next, connection) { - const plugin = this; - const remote_ip = connection.remote.ip; - - if (plugin.should_skip(connection)) { return next(); } - - plugin.first(remote_ip, plugin.zones, (err, zone, a) => { - if (err) { - connection.results.add(plugin, {err: err.message}); - return next(); - } - if (!a) return next(); - - const msg = `host [${remote_ip}] is blacklisted by ${zone}`; - if (plugin.cfg.main.reject) return next(DENY, msg); - - connection.loginfo(plugin, msg); - return next(); - }, function each_result (err, zone, a) { - if (err) return; - const result = a ? {fail: zone} : {pass: zone}; - connection.results.add(plugin, result); - }); -} - -exports.connect_multi = function (next, connection) { - const remote_ip = connection.remote.ip; - - if (this.should_skip(connection)) { return next(); } - - const hits = []; - function get_deny_msg () { - return `host [${remote_ip}] is blacklisted by ${hits.join(', ')}`; - } - - this.multi(remote_ip, this.zones, (err, zone, a, pending) => { - if (err) { - connection.results.add(this, {err: err.message}); - if (pending) return; - if (this.cfg.main.reject && hits.length) { - return next(DENY, get_deny_msg()); - } - return next(); - } - - if (a) { - hits.push(zone); - connection.results.add(this, {fail: zone}); - } - else { - if (zone) connection.results.add(this, {pass: zone}); - } - - if (pending) return; - connection.results.add(this, {emit: true}); - - if (this.cfg.main.reject && hits.length) { - return next(DENY, get_deny_msg()); - } - return next(); - }); -} diff --git a/plugins/dnswl.js b/plugins/dnswl.js deleted file mode 100644 index e5c79c21e..000000000 --- a/plugins/dnswl.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; -// dnswl plugin - -exports.register = function () { - this.inherits('dns_list_base'); - - this.load_dnswl_ini(); - - // IMPORTANT: don't run this on hook_rcpt otherwise we're an open relay... - ['ehlo','helo','mail'].forEach(hook => { - this.register_hook(hook, 'check_dnswl'); - }); -} - -exports.load_dnswl_ini = function () { - this.cfg = this.config.get('dnswl.ini', () => { - this.load_dnswl_ini(); - }); - - if (this.cfg.main.enable_stats) { - this.logdebug('stats reporting enabled'); - this.enable_stats = true; - } - - if (this.cfg.main.stats_redis_host) { - this.redis_host = this.cfg.main.stats_redis_host; - this.logdebug(`set stats redis host to: ${this.redis_host}`); - } - - this.zones = []; - // Compatibility with old-plugin - this.zones = this.zones.concat( - this.config.get('dnswl.zones', 'list') - ); - if (this.cfg.main.zones) { - this.zones = this.zones.concat( - this.cfg.main.zones.replace(/\s+/g,'').split(/[;,]/)); - } - - if (this.cfg.main.periodic_checks) { - this.check_zones(this.cfg.main.periodic_checks); - } -} - -exports.check_dnswl = (next, connection) => connection.notes.dnswl ? next(OK) : next() - -exports.hook_connect = function (next, connection) { - if (!this.zones || !this.zones.length) { - connection.logerror(this, 'no zones'); - return next(); - } - this.first(connection.remote.ip, this.zones, (err, zone, a) => { - if (!a) return next(); - connection.loginfo(this, `${connection.remote.ip} is whitelisted by ${zone}: ${a}`); - connection.notes.dnswl = true; - return next(OK); - }); -} diff --git a/tests/plugins/dns_list_base.js b/tests/plugins/dns_list_base.js deleted file mode 100644 index c5cf5ae41..000000000 --- a/tests/plugins/dns_list_base.js +++ /dev/null @@ -1,259 +0,0 @@ -'use strict'; - -const fixtures = require('haraka-test-fixtures'); - -const _set_up = function (done) { - - this.plugin = new fixtures.plugin('dns_list_base'); - - done(); -} - -exports.disable_zone = { - setUp : _set_up, - 'empty request' (test) { - test.expect(1); - const res = this.plugin.disable_zone(); - test.equal(false, res); - test.done(); - }, - 'testbl1, no zones' (test) { - test.expect(1); - const res = this.plugin.disable_zone('testbl1', 'test result'); - test.equal(false, res); - test.done(); - }, - 'testbl1, zones miss' (test) { - test.expect(2); - this.plugin.disable_allowed=true; - this.plugin.zones = [ 'testbl2' ]; - const res = this.plugin.disable_zone('testbl1', 'test result'); - test.equal(false, res); - test.equal(1, this.plugin.zones.length); - test.done(); - }, - 'testbl1, zones hit' (test) { - test.expect(2); - this.plugin.disable_allowed=true; - this.plugin.zones = [ 'testbl1' ]; - const res = this.plugin.disable_zone('testbl1', 'test result'); - test.equal(true, res); - test.equal(0, this.plugin.zones.length); - test.done(); - }, -} - -exports.lookup = { - setUp : _set_up, - 'Spamcop, test IP' (test) { - test.expect(2); - function cb (err, a) { - test.equal(null, err); - test.ok(a); - test.done(); - } - this.plugin.lookup('127.0.0.2', 'bl.spamcop.net', cb); - }, - 'Spamcop, unlisted IP' (test) { - test.expect(2); - const cb = function (err, a) { - test.equal(null, err); - test.equal(null, a); - test.done(); - }.bind(this); - this.plugin.lookup('127.0.0.1', 'bl.spamcop.net', cb); - }, -} - -exports.multi = { - setUp : _set_up, - 'Spamcop' (test) { - test.expect(4); - this.plugin.multi('127.0.0.2', 'bl.spamcop.net', (err, zone, a, pending) => { - test.equal(null, err); - if (pending) { - test.ok((Array.isArray(a) && a.length > 0)); - test.equal(true, pending); - } - else { - test.done(); - } - }) - }, - 'CBL' (test) { - test.expect(4); - this.plugin.multi('127.0.0.2', 'xbl.spamhaus.org', (err, zone, a, pending) => { - test.equal(null, err); - if (pending) { - test.ok((Array.isArray(a) && a.length > 0)); - test.equal(true, pending); - } - else { - test.done(); - } - }) - }, - 'Spamcop + CBL' (test) { - test.expect(12); - const dnsbls = ['bl.spamcop.net','xbl.spamhaus.org']; - this.plugin.multi('127.0.0.2', dnsbls, (err, zone, a, pending) => { - test.equal(null, err); - if (pending) { - test.ok(zone); - test.ok((Array.isArray(a) && a.length > 0)); - test.equal(true, pending); - } - else { - test.equal(null, zone); - test.equal(null, a); - test.equal(false, pending); - test.done(); - } - }) - }, - 'Spamcop + CBL + negative result' (test) { - test.expect(12); - const dnsbls = [ 'bl.spamcop.net','xbl.spamhaus.org' ]; - this.plugin.multi('127.0.0.1', dnsbls, (err, zone, a, pending) => { - test.equal(null, err); - if (a && a[0] && a[0] === '127.255.255.254') { - test.deepEqual(['127.255.255.254'], a) - console.warn(`ERROR: DNSBLs don't work with PUBLIC DNS!`) - } - else { - test.equal(null, a) - } - if (pending) { - test.equal(true, pending); - test.ok(zone); - } - else { - test.equal(false, pending); - test.equal(null, zone); - test.done(); - } - }) - }, - 'IPv6 addresses supported' (test) { - test.expect(12); - const dnsbls = ['bl.spamcop.net','xbl.spamhaus.org']; - this.plugin.multi('::1', dnsbls, (err, zone, a, pending) => { - if (a && a[0] && a[0] === '127.255.255.254') { - test.deepEqual(['127.255.255.254'], a) - console.warn(`ERROR: DNSBLs don't work with PUBLIC DNS!`) - } - else { - test.equal(null, a); - } - if (pending) { - test.deepEqual(null, err); - test.equal(true, pending); - test.ok(zone); - } - else { - test.equal(null, err); - test.equal(false, pending); - test.equal(null, zone); - test.done(); - } - }) - } -} - -exports.first = { - setUp : _set_up, - 'positive result' (test) { - test.expect(3); - const dnsbls = [ 'xbl.spamhaus.org', 'bl.spamcop.net' ]; - this.plugin.first('127.0.0.2', dnsbls, (err, zone, a) => { - test.equal(null, err); - test.ok(zone); - test.ok((Array.isArray(a) && a.length > 0)); - test.done(); - }) - }, - 'negative result' (test) { - test.expect(2); - const dnsbls = [ 'xbl.spamhaus.org', 'bl.spamcop.net' ]; - this.plugin.first('127.0.0.1', dnsbls, (err, zone, a) => { - test.equal(null, err); - if (a && a[0] && a[0] === '127.255.255.254') { - test.deepEqual(['127.255.255.254'], a) - console.warn(`ERROR: DNSBLs don't work with PUBLIC DNS!`) - } - else { - test.equal(null, a); - } - test.done(); - }) - }, - 'each_cb' (test) { - test.expect(7); - const dnsbls = [ 'xbl.spamhaus.org', 'bl.spamcop.net' ]; - let pending = dnsbls.length; - function cb () { - test.ok(pending); - } - function cb_each (err, zone, a) { - pending--; - test.equal(null, err); - test.ok(zone); - test.ok((Array.isArray(a) && a.length > 0)); - if (pending === 0) test.done(); - } - this.plugin.first('127.0.0.2', dnsbls, cb, cb_each); - } -} - -function zone_disable_test_func (zones, test, cb) { - this.plugin.disabled_zones = zones; - this.plugin.zones = [] - - this.plugin.check_zones(9000); - - const fin_check = () => { - this.plugin.shutdown(); - cb(); - test.done(); - }; - - let i = 0; - const again = () => { - i++; - setTimeout(() => { - if (this.plugin.zones.length === zones.length) return fin_check(); - if (i > 4) return fin_check(); - again(); - }, 1000); - }; - again(); -} - -exports.lookback_is_rejected = { - setUp: _set_up, - 'zones with quirks pass through when lookback_is_rejected=true' (test) { - const zones = [ 'hostkarma.junkemailfilter.com', 'bl.spamcop.net' ]; - this.plugin.lookback_is_rejected = true; - - zone_disable_test_func.call(this, zones, test, () => { - if (this.plugin.zones.length === 0) { - // just AppVeyor being annoying - if (!['win32','win64'].includes(process.platform)) { - console.error("Didn't enable all zones back"); - } - test.deepEqual(this.plugin.zones.length, 0); - } - else { - test.deepEqual(this.plugin.zones.sort(), zones.sort(), "Didn't enable all zones back"); - } - }); - }, - 'zones with quirks are disabled when lookback_is_rejected=false' (test) { - const zones = [ 'hostkarma.junkemailfilter.com', 'bl.spamcop.net' ]; - // this.plugin.lookback_is_rejected = true; - - zone_disable_test_func.call(this, zones, test, () => { - test.deepEqual(this.plugin.zones.sort(), ['bl.spamcop.net'], "Enabled all zones back? This should not have happened"); - }); - } -} diff --git a/tests/plugins/dnsbl.js b/tests/plugins/dnsbl.js deleted file mode 100644 index 62ae831d5..000000000 --- a/tests/plugins/dnsbl.js +++ /dev/null @@ -1,101 +0,0 @@ -'use strict'; - -const path = require('path'); -const fixtures = require('haraka-test-fixtures'); - -const Connection = fixtures.connection; - -function _set_up (done) { - - this.plugin = new fixtures.plugin('dnsbl'); - this.plugin.config.root_path = path.resolve(__dirname, '../../config'); - - this.connection = Connection.createConnection(); - - done(); -} - -exports.load_config = { - setUp : _set_up, - 'none' (test) { - test.expect(1); - test.equal(undefined, this.plugin.cfg); - test.done(); - }, - 'defaults' (test) { - test.expect(3); - this.plugin.load_config(); - test.equal(true, this.plugin.cfg.main.reject); - test.equal(30, this.plugin.cfg.main.periodic_checks); - test.equal('first', this.plugin.cfg.main.search); - test.done(); - }, -} - -exports.get_uniq_zones = { - setUp : _set_up, - 'none' (test) { - test.expect(1); - test.equal(undefined, this.plugin.zones); - test.done(); - }, - 'dnsbl.zones' (test) { - test.expect(2); - this.plugin.load_config(); - this.plugin.cfg.main.zones = 'dnsbl.test, dnsbl2.test'; - this.plugin.get_uniq_zones(); - test.notEqual(-1, this.plugin.zones.indexOf('dnsbl.test')); - test.notEqual(-1, this.plugin.zones.indexOf('dnsbl2.test')); - - test.done(); - }, -} - -exports.should_skip = { - setUp : _set_up, - 'no connection' (test) { - test.expect(1); - test.equal(true, this.plugin.should_skip()); - test.done(); - }, - 'no remote_ip' (test) { - test.expect(1); - - test.equal(true, this.plugin.should_skip(this.connection)); - test.done(); - }, - 'private remote_ip, no zones' (test) { - test.expect(1); - this.connection.remote.ip = '192.168.1.1'; - test.equal(true, this.plugin.should_skip(this.connection)); - test.done(); - }, - 'private remote_ip' (test) { - test.expect(1); - this.connection.remote.is_private = true; - - this.plugin.load_config(); - this.plugin.cfg.main.zones = 'dnsbl.test, dnsbl2.test'; - this.plugin.get_uniq_zones(); - - test.equal(true, this.plugin.should_skip(this.connection)); - test.done(); - }, - 'public remote_ip' (test) { - test.expect(1); - this.connection.remote.ip = '208.1.1.1'; - - this.plugin.load_config(); - this.plugin.cfg.main.zones = 'dnsbl.test, dnsbl2.test'; - this.plugin.get_uniq_zones(); - - test.equal(false, this.plugin.should_skip(this.connection)); - test.done(); - }, - 'public remote_ip, no zones' (test) { - test.expect(1); - this.connection.remote.ip = '208.1.1.1'; - test.equal(true, this.plugin.should_skip(this.connection)); - test.done(); - }, -}