next/1454/80x/20260818/v1

doc/userguide/rules/datasets.rst

@@ -81,7 +81,7 @@ Syntax::
     dataset:<set|unset|isset|isnotset>,<name> \
         [, type <string|md5|sha256|ipv4|ip>, save <file name>, load <file name>, state <file name>, memcap <size>, hashsize <size>
          , format <csv|json|ndjson>, context_key <output_key>, value_key <json_key>, array_key <json_path>,
-         remove_key];
+         remove_key, match subdomain];
 
 type <type>
   the data type: string, md5, sha256, ipv4, ip
@@ -112,7 +112,11 @@ array_key <key>
 remove_key
   if set, the JSON object pointed by value key will be removed
   from the alert event
-
+match subdomain
+  if set to ``subdomain``, enables hierarchical domain matching.
+  On lookup, the dataset walks up the domain label hierarchy until
+  a match is found. Only valid with ``isset``/``isnotset`` commands
+  and ``type string``. Best used with the ``dotprefix`` transform.
 
 .. note:: 'type' is mandatory and needs to be set.
 
@@ -137,6 +141,28 @@ on domain names to find TLDs in the dataset ``dns-tld-seen``:
 
 .. image:: dataset-examples/detect-unique-tlds.png
 
+3. Block domains and all their subdomains using a blocklist dataset:
+
+.. container:: example-rule
+
+    reject dns any any -> any any (msg:"Blocked domain"; dns.query; dotprefix; dataset:isset,blocked-domains, type string, match subdomain, load blocked-domains.lst; sid:8000003; rev:1;)
+
+The ``match subdomain`` option walks up the domain hierarchy on each
+lookup. Combined with ``dotprefix``, a query for ``mail.evil.com``
+becomes ``.mail.evil.com`` and is checked against the dataset as:
+``.mail.evil.com``, ``.evil.com``, ``.com``. If ``.evil.com`` is in the
+dataset, the rule matches.
+
+The dataset file should contain entries with a leading dot::
+
+    LmV2aWwuY29tCg==
+
+which is the base64 encoding of ``.evil.com``.
+
+When using ``ndjson`` format, use the raw dotted value in the JSON::
+
+    {"domain": ".evil.com"}
+
 Notice how it is not possible to do certain operations alone with datasets
 (example 2 above), but, it is possible to use a combination of other rule
 keywords. Keep in mind the cost of additional keywords though e.g. in the
@@ -184,7 +210,7 @@ Syntax::
 
     dataset:<isset|isnotset>,<name> \
         [, type <string|md5|sha256|ipv4|ip>, load <file name>, format <json|ndjson>, memcap <size>, hashsize <size>, context_key <json_key> \
-         , value_key <json_key>, array_key <json_path>];
+         , value_key <json_key>, array_key <json_path>, match subdomain];
 
 Example rules could look like::
 

src/detect-dataset.c

@@ -49,6 +49,8 @@
 #define DETECT_DATASET_CMD_ISNOTSET 2
 #define DETECT_DATASET_CMD_ISSET    3
 
+#define DATASET_SUBDOMAIN_MAX_LOOKUPS 126
+
 static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
 void DetectDatasetFree (DetectEngineCtx *, void *);
 
@@ -61,6 +63,26 @@ void DetectDatasetRegister (void)
     sigmatch_table[DETECT_DATASET].Free  = DetectDatasetFree;
 }
 
+/** \brief walk up the domain hierarchy looking for a match in a JSON dataset */
+static DataJsonResultType DatajsonLookupSubdomain(
+        Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    DataJsonResultType r = {
+        .found = false, .json = { .value = NULL, .len = 0 }, .hashdata = NULL
+    };
+    int lookups = 0;
+    for (uint32_t i = 1; i < data_len; i++) {
+        if (data[i] == '.' && data[i - 1] != '.') {
+            if (++lookups > DATASET_SUBDOMAIN_MAX_LOOKUPS)
+                break;
+            r = DatajsonLookup(set, data + i, data_len - i);
+            if (r.found)
+                return r;
+        }
+    }
+    return r;
+}
+
 /*
     1 match
     0 no match
@@ -76,6 +98,9 @@ static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const Detec
             // PrintRawDataFp(stdout, data, data_len);
             DataJsonResultType r = DatajsonLookup(sd->set, data, data_len);
             SCLogDebug("r found: %d, len: %u", r.found, r.json.len);
+            if (!r.found && sd->match_subdomain) {
+                r = DatajsonLookupSubdomain(sd->set, data, data_len);
+            }
             if (!r.found)
                 return 0;
             if (r.json.len > 0) {
@@ -101,6 +126,9 @@ static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const Detec
             // PrintRawDataFp(stdout, data, data_len);
             DataJsonResultType r = DatajsonLookup(sd->set, data, data_len);
             SCLogDebug("r found: %d, len: %u", r.found, r.json.len);
+            if (!r.found && sd->match_subdomain) {
+                r = DatajsonLookupSubdomain(sd->set, data, data_len);
+            }
             if (r.found) {
                 DatajsonUnlockElt(&r);
                 return 0;
@@ -113,6 +141,22 @@ static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const Detec
     return 0;
 }
 
+/** \brief walk up the domain hierarchy looking for a match in a dataset */
+static int DatasetLookupSubdomain(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    int lookups = 0;
+    for (uint32_t i = 1; i < data_len; i++) {
+        if (data[i] == '.' && data[i - 1] != '.') {
+            if (++lookups > DATASET_SUBDOMAIN_MAX_LOOKUPS)
+                break;
+            int r = DatasetLookup(set, data + i, data_len - i);
+            if (r == 1)
+                return 1;
+        }
+    }
+    return 0;
+}
+
 /*
     1 match
     0 no match
@@ -132,6 +176,9 @@ int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
         case DETECT_DATASET_CMD_ISSET: {
             //PrintRawDataFp(stdout, data, data_len);
             int r = DatasetLookup(sd->set, data, data_len);
+            if (r != 1 && sd->match_subdomain) {
+                r = DatasetLookupSubdomain(sd->set, data, data_len);
+            }
             SCLogDebug("r %d", r);
             if (r == 1)
                 return 1;
@@ -140,6 +187,9 @@ int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
         case DETECT_DATASET_CMD_ISNOTSET: {
             //PrintRawDataFp(stdout, data, data_len);
             int r = DatasetLookup(sd->set, data, data_len);
+            if (r != 1 && sd->match_subdomain) {
+                r = DatasetLookupSubdomain(sd->set, data, data_len);
+            }
             SCLogDebug("r %d", r);
             if (r < 1)
                 return 1;
@@ -168,7 +218,7 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
         enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
         uint64_t *memcap, uint32_t *hashsize, DatasetFormats *format, char *value_key,
         size_t value_key_size, char *array_key, size_t array_key_size, char *enrichment_key,
-        size_t enrichment_key_size, bool *remove_key)
+        size_t enrichment_key_size, bool *remove_key, bool *match_subdomain)
 {
     bool cmd_set = false;
     bool name_set = false;
@@ -220,6 +270,13 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
                     *remove_key = true;
                 } else
                     return -1;
+            } else if (strcmp(key, "match") == 0) {
+                if (strcmp(val, "subdomain") == 0) {
+                    *match_subdomain = true;
+                } else {
+                    SCLogError("unknown match mode '%s'", val);
+                    return -1;
+                }
             } else if (strcmp(key, "type") == 0) {
                 SCLogDebug("type %s", val);
 
@@ -474,6 +531,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     char array_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     char enrichment_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     bool remove_key = false;
+    bool match_subdomain = false;
 
     if (DetectBufferGetActiveList(de_ctx, s) == -1) {
         SCLogError("datasets are only supported for sticky buffers");
@@ -489,7 +547,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
                 sizeof(load), save, sizeof(save), &memcap, &hashsize, &format, value_key,
                 sizeof(value_key), array_key, sizeof(array_key), enrichment_key,
-                sizeof(enrichment_key), &remove_key)) {
+                sizeof(enrichment_key), &remove_key, &match_subdomain)) {
         return -1;
     }
 
@@ -514,6 +572,17 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
         return -1;
     }
 
+    if (match_subdomain) {
+        if (cmd != DETECT_DATASET_CMD_ISSET && cmd != DETECT_DATASET_CMD_ISNOTSET) {
+            SCLogError("'match subdomain' only supports isset/isnotset commands");
+            return -1;
+        }
+        if (type != DATASET_TYPE_STRING) {
+            SCLogError("'match subdomain' only supports type string");
+            return -1;
+        }
+    }
+
     if ((format == DATASET_FORMAT_JSON) || (format == DATASET_FORMAT_NDJSON)) {
         if (strlen(save) != 0) {
             SCLogError("json format is not supported with 'save' or 'state' option");
@@ -579,6 +648,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     cd->set = set;
     cd->cmd = cmd;
     cd->format = format;
+    cd->match_subdomain = match_subdomain;
     if ((format == DATASET_FORMAT_JSON) || (format == DATASET_FORMAT_NDJSON)) {
         strlcpy(cd->json_key, enrichment_key, sizeof(cd->json_key));
     }

src/detect-dataset.h

@@ -30,6 +30,7 @@
 typedef struct DetectDatasetData_ {
     Dataset *set;
     uint8_t cmd;
+    bool match_subdomain;
     DatasetFormats format;
     DataJsonType json;
     char json_key[SIG_JSON_CONTENT_KEY_LEN];

src/source-erf-dag.c

@@ -403,6 +403,12 @@ ProcessErfDagRecords(ErfDagThreadVars *ewtn, uint8_t *top, uint32_t *pkts_read)
         rlen = SCNtohs(dr->rlen);
         hdr_type = dr->type;
 
+        if (rlen < dag_record_size) {
+            SCLogError("Bad ERF record length %d (< dag_record_size) on stream: %d, DAG: %s", rlen,
+                    ewtn->dagstream, ewtn->dagname);
+            SCReturnInt(TM_ECODE_FAILED);
+        }
+
         /* If we don't have enough data to finish processing this ERF
          * record return and maybe next time we will.
          */