tests: test for filestore fd exhaustion

tests/filestore-fd-exhaustion/README.md

@@ -0,0 +1,38 @@
+# filestore-fd-exhaustion
+
+Regression test for the file-store open-file-descriptor accounting at the
+default `max-open-files: 0`.
+
+The default is documented as "files get closed after each write to the file",
+so only a couple of descriptors are ever open regardless of how many files are
+being extracted concurrently. A regression in `OutputFilestoreLogger` instead
+keeps a descriptor open per in-progress stored file when `max-open-files` is 0,
+so the count is bounded only by the number of simultaneous transfers — i.e. by
+network traffic — and can exhaust the process file-descriptor table.
+
+## How it works
+
+`input.pcap` is 48 concurrent HTTP downloads whose bodies are delivered
+round-robin, so every file is mid-transfer at the same time. Each file is
+larger than the file-store incremental-write threshold (~100 KiB), so
+file-store writes it while it is still open — that is what causes a descriptor
+to be held on the affected build. The test runs Suricata with `force-filestore`
+and a 32-descriptor `ulimit`:
+
+- Correct build: closes after each write, ~1 fd open at a time, no error.
+- Regressed build: holds ~48 fds at once, exceeds the limit, and logs
+  `Filestore (v2) failed to create ... Too many open files` (EMFILE).
+
+Note the extracted **file count is the same** either way (failed opens are
+retried as other files close and everything flushes at EOF), so the check is on
+the EMFILE log message, not on how many files were stored.
+
+The pcap is ~6 MB because each of the 48 files must exceed the ~100 KiB
+incremental-write threshold; that is the floor for this mechanism, not padding.
+
+## Regenerate the pcap
+
+    ./generate-pcap.py -o input.pcap
+
+Requires scapy. The number of flows must stay above the `ulimit` set in
+`test.yaml`.

tests/filestore-fd-exhaustion/generate-pcap.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Generate input.pcap for the filestore-fd-exhaustion test.
+
+The pcap contains N concurrent HTTP downloads whose bodies are delivered
+round-robin (interleaved) across all flows, so every file is mid-transfer at
+the same time. Each file is larger than the file-store incremental-write
+threshold (~100 KiB), so file-store writes it out while it is still open and,
+on builds that keep the descriptor open, holds one fd per concurrent file.
+With force-filestore and a low fd ulimit this exhausts the process fd table.
+
+Each body carries a per-flow marker so the files have distinct SHA-256s and are
+not collapsed by file-store de-duplication.
+
+Regenerate:  ./generate-pcap.py -o input.pcap
+"""
+
+import argparse
+
+from scapy.all import Ether, IP, TCP, Raw, PcapWriter
+
+N = 48              # concurrent flows (must exceed the ulimit used in test.yaml)
+FILE_SIZE = 131072  # > ~100 KiB so each file is written incrementally
+CHUNK = 8192        # body bytes per flow per interleave round
+CMAC, SMAC = "02:00:00:00:00:01", "02:00:00:00:00:02"
+
+
+def body(flow_id, off, length):
+    data = bytearray(b"A" * length)
+    marker = ("flow=%020d\n" % flow_id).encode()
+    if off < len(marker):
+        n = min(length, len(marker) - off)
+        data[:n] = marker[off:off + n]
+    return bytes(data)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("-n", type=int, default=N)
+    ap.add_argument("-o", "--output", default="input.pcap")
+    a = ap.parse_args()
+
+    w = PcapWriter(a.output, sync=True)
+    t = [1.0]
+
+    def wr(p):
+        p.time = t[0]
+        t[0] += 1e-6
+        w.write(p)
+
+    flows = []
+    for i in range(a.n):
+        f = dict(id=i + 1, cip="10.0.%d.%d" % ((i >> 8) & 0xff, i & 0xff),
+                 sip="10.9.0.1", cp=1024 + (i % 60000), sp=80,
+                 cs=(0x1000 + i * 8191) & 0xffffffff,
+                 ss=(0x9000 + i * 104729) & 0xffffffff, sent=0)
+        flows.append(f)
+
+    def pkt(f, d, flags, seq, ack=0, pl=b""):
+        if d == "c":
+            e = Ether(src=CMAC, dst=SMAC) / IP(src=f["cip"], dst=f["sip"])
+            tcp = TCP(sport=f["cp"], dport=f["sp"], flags=flags, seq=seq, ack=ack, window=65535)
+        else:
+            e = Ether(src=SMAC, dst=CMAC) / IP(src=f["sip"], dst=f["cip"])
+            tcp = TCP(sport=f["sp"], dport=f["cp"], flags=flags, seq=seq, ack=ack, window=65535)
+        p = e / tcp
+        return p / Raw(pl) if pl else p
+
+    # Handshake + request + response headers for every flow first.
+    for f in flows:
+        wr(pkt(f, "c", "S", f["cs"])); f["cs"] = (f["cs"] + 1) & 0xffffffff
+        wr(pkt(f, "s", "SA", f["ss"], f["cs"])); f["ss"] = (f["ss"] + 1) & 0xffffffff
+        wr(pkt(f, "c", "A", f["cs"], f["ss"]))
+        req = ("GET /f%d.bin HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n" % f["id"]).encode()
+        wr(pkt(f, "c", "PA", f["cs"], f["ss"], req)); f["cs"] = (f["cs"] + len(req)) & 0xffffffff
+        hdr = ("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n"
+               "Content-Length: %d\r\nConnection: close\r\n\r\n" % FILE_SIZE).encode()
+        wr(pkt(f, "s", "PA", f["ss"], f["cs"], hdr)); f["ss"] = (f["ss"] + len(hdr)) & 0xffffffff
+        wr(pkt(f, "c", "A", f["cs"], f["ss"]))
+
+    # Interleave body chunks round-robin so all files stay open at once.
+    active = list(flows)
+    while active:
+        still = []
+        for f in active:
+            length = min(CHUNK, FILE_SIZE - f["sent"])
+            wr(pkt(f, "s", "PA", f["ss"], f["cs"], body(f["id"], f["sent"], length)))
+            f["ss"] = (f["ss"] + length) & 0xffffffff
+            f["sent"] += length
+            wr(pkt(f, "c", "A", f["cs"], f["ss"]))
+            if f["sent"] < FILE_SIZE:
+                still.append(f)
+            else:
+                wr(pkt(f, "s", "FA", f["ss"], f["cs"])); f["ss"] = (f["ss"] + 1) & 0xffffffff
+                wr(pkt(f, "c", "A", f["cs"], f["ss"]))
+                wr(pkt(f, "c", "FA", f["cs"], f["ss"])); f["cs"] = (f["cs"] + 1) & 0xffffffff
+                wr(pkt(f, "s", "A", f["ss"], f["cs"]))
+        active = still
+
+    w.close()
+    print("wrote %s: %d flows, %d bytes/file" % (a.output, a.n, FILE_SIZE))
+
+
+if __name__ == "__main__":
+    main()

tests/filestore-fd-exhaustion/suricata.yaml

@@ -0,0 +1,23 @@
+%YAML 1.1
+---
+
+logging:
+  default-log-level: notice
+  outputs:
+    - console:
+        enabled: yes
+    - file:
+        enabled: yes
+        level: info
+        filename: suricata.log
+
+outputs:
+  - eve-log:
+      enabled: yes
+      types:
+        - files
+  - file-store:
+      version: 2
+      enabled: yes
+      force-filestore: yes
+      max-open-files: 0

tests/filestore-fd-exhaustion/test.yaml

@@ -0,0 +1,33 @@
+requires:
+  min-version: 8
+
+  # I'm not sure how portable ulimit is.
+  os: linux
+
+# Run with a low file-descriptor ulimit. The pcap drives 48 concurrent HTTP
+# downloads that file-store extracts at the same time. With the documented
+# default (max-open-files: 0) Suricata should close each file after every
+# write, so only a couple of descriptors are ever open. The regression keeps
+# one descriptor open per in-progress file, so all ~48 are held at once, the
+# 32-fd limit is exceeded, and file-store logs EMFILE ("Too many open files").
+command: |
+  ulimit -Sn 32 && \
+  ${SRCDIR}/src/suricata \
+      --set classification-file="${SRCDIR}/etc/classification.config" \
+      --set reference-config-file="${SRCDIR}/etc/reference.config" \
+      -c "${TEST_DIR}/suricata.yaml" \
+      -l "${OUTPUT_DIR}" \
+      --runmode single -k none \
+      -r "${TEST_DIR}/input.pcap"
+
+checks:
+  # Sanity: a correct build extracts all 48 forced files.
+  - filter:
+      count: 48
+      match:
+        event_type: fileinfo
+        fileinfo.stored: true
+  # Regression: a correct build never exhausts the fd table here.
+  # (test -s guards against a silent pass if suricata.log is ever missing.)
+  - shell:
+      args: "test -s suricata.log && ! grep -q 'Too many open files' suricata.log"