diff --git a/tests/filestore-fd-exhaustion/README.md b/tests/filestore-fd-exhaustion/README.md new file mode 100644 index 0000000000..4616faa808 --- /dev/null +++ b/tests/filestore-fd-exhaustion/README.md @@ -0,0 +1,38 @@ +# filestore-fd-exhaustion + +Regression test for the file-store open-file-descriptor accounting at the +default `max-open-files: 0`. + +The default is documented as "files get closed after each write to the file", +so only a couple of descriptors are ever open regardless of how many files are +being extracted concurrently. A regression in `OutputFilestoreLogger` instead +keeps a descriptor open per in-progress stored file when `max-open-files` is 0, +so the count is bounded only by the number of simultaneous transfers — i.e. by +network traffic — and can exhaust the process file-descriptor table. + +## How it works + +`input.pcap` is 48 concurrent HTTP downloads whose bodies are delivered +round-robin, so every file is mid-transfer at the same time. Each file is +larger than the file-store incremental-write threshold (~100 KiB), so +file-store writes it while it is still open — that is what causes a descriptor +to be held on the affected build. The test runs Suricata with `force-filestore` +and a 32-descriptor `ulimit`: + +- Correct build: closes after each write, ~1 fd open at a time, no error. +- Regressed build: holds ~48 fds at once, exceeds the limit, and logs + `Filestore (v2) failed to create ... Too many open files` (EMFILE). + +Note the extracted **file count is the same** either way (failed opens are +retried as other files close and everything flushes at EOF), so the check is on +the EMFILE log message, not on how many files were stored. + +The pcap is ~6 MB because each of the 48 files must exceed the ~100 KiB +incremental-write threshold; that is the floor for this mechanism, not padding. + +## Regenerate the pcap + + ./generate-pcap.py -o input.pcap + +Requires scapy. The number of flows must stay above the `ulimit` set in +`test.yaml`. diff --git a/tests/filestore-fd-exhaustion/generate-pcap.py b/tests/filestore-fd-exhaustion/generate-pcap.py new file mode 100755 index 0000000000..dd11a2cb73 --- /dev/null +++ b/tests/filestore-fd-exhaustion/generate-pcap.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Generate input.pcap for the filestore-fd-exhaustion test. + +The pcap contains N concurrent HTTP downloads whose bodies are delivered +round-robin (interleaved) across all flows, so every file is mid-transfer at +the same time. Each file is larger than the file-store incremental-write +threshold (~100 KiB), so file-store writes it out while it is still open and, +on builds that keep the descriptor open, holds one fd per concurrent file. +With force-filestore and a low fd ulimit this exhausts the process fd table. + +Each body carries a per-flow marker so the files have distinct SHA-256s and are +not collapsed by file-store de-duplication. + +Regenerate: ./generate-pcap.py -o input.pcap +""" + +import argparse + +from scapy.all import Ether, IP, TCP, Raw, PcapWriter + +N = 48 # concurrent flows (must exceed the ulimit used in test.yaml) +FILE_SIZE = 131072 # > ~100 KiB so each file is written incrementally +CHUNK = 8192 # body bytes per flow per interleave round +CMAC, SMAC = "02:00:00:00:00:01", "02:00:00:00:00:02" + + +def body(flow_id, off, length): + data = bytearray(b"A" * length) + marker = ("flow=%020d\n" % flow_id).encode() + if off < len(marker): + n = min(length, len(marker) - off) + data[:n] = marker[off:off + n] + return bytes(data) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("-n", type=int, default=N) + ap.add_argument("-o", "--output", default="input.pcap") + a = ap.parse_args() + + w = PcapWriter(a.output, sync=True) + t = [1.0] + + def wr(p): + p.time = t[0] + t[0] += 1e-6 + w.write(p) + + flows = [] + for i in range(a.n): + f = dict(id=i + 1, cip="10.0.%d.%d" % ((i >> 8) & 0xff, i & 0xff), + sip="10.9.0.1", cp=1024 + (i % 60000), sp=80, + cs=(0x1000 + i * 8191) & 0xffffffff, + ss=(0x9000 + i * 104729) & 0xffffffff, sent=0) + flows.append(f) + + def pkt(f, d, flags, seq, ack=0, pl=b""): + if d == "c": + e = Ether(src=CMAC, dst=SMAC) / IP(src=f["cip"], dst=f["sip"]) + tcp = TCP(sport=f["cp"], dport=f["sp"], flags=flags, seq=seq, ack=ack, window=65535) + else: + e = Ether(src=SMAC, dst=CMAC) / IP(src=f["sip"], dst=f["cip"]) + tcp = TCP(sport=f["sp"], dport=f["cp"], flags=flags, seq=seq, ack=ack, window=65535) + p = e / tcp + return p / Raw(pl) if pl else p + + # Handshake + request + response headers for every flow first. + for f in flows: + wr(pkt(f, "c", "S", f["cs"])); f["cs"] = (f["cs"] + 1) & 0xffffffff + wr(pkt(f, "s", "SA", f["ss"], f["cs"])); f["ss"] = (f["ss"] + 1) & 0xffffffff + wr(pkt(f, "c", "A", f["cs"], f["ss"])) + req = ("GET /f%d.bin HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n" % f["id"]).encode() + wr(pkt(f, "c", "PA", f["cs"], f["ss"], req)); f["cs"] = (f["cs"] + len(req)) & 0xffffffff + hdr = ("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n" + "Content-Length: %d\r\nConnection: close\r\n\r\n" % FILE_SIZE).encode() + wr(pkt(f, "s", "PA", f["ss"], f["cs"], hdr)); f["ss"] = (f["ss"] + len(hdr)) & 0xffffffff + wr(pkt(f, "c", "A", f["cs"], f["ss"])) + + # Interleave body chunks round-robin so all files stay open at once. + active = list(flows) + while active: + still = [] + for f in active: + length = min(CHUNK, FILE_SIZE - f["sent"]) + wr(pkt(f, "s", "PA", f["ss"], f["cs"], body(f["id"], f["sent"], length))) + f["ss"] = (f["ss"] + length) & 0xffffffff + f["sent"] += length + wr(pkt(f, "c", "A", f["cs"], f["ss"])) + if f["sent"] < FILE_SIZE: + still.append(f) + else: + wr(pkt(f, "s", "FA", f["ss"], f["cs"])); f["ss"] = (f["ss"] + 1) & 0xffffffff + wr(pkt(f, "c", "A", f["cs"], f["ss"])) + wr(pkt(f, "c", "FA", f["cs"], f["ss"])); f["cs"] = (f["cs"] + 1) & 0xffffffff + wr(pkt(f, "s", "A", f["ss"], f["cs"])) + active = still + + w.close() + print("wrote %s: %d flows, %d bytes/file" % (a.output, a.n, FILE_SIZE)) + + +if __name__ == "__main__": + main() diff --git a/tests/filestore-fd-exhaustion/input.pcap b/tests/filestore-fd-exhaustion/input.pcap new file mode 100644 index 0000000000..cef289b351 Binary files /dev/null and b/tests/filestore-fd-exhaustion/input.pcap differ diff --git a/tests/filestore-fd-exhaustion/suricata.yaml b/tests/filestore-fd-exhaustion/suricata.yaml new file mode 100644 index 0000000000..e2f81dfd0f --- /dev/null +++ b/tests/filestore-fd-exhaustion/suricata.yaml @@ -0,0 +1,23 @@ +%YAML 1.1 +--- + +logging: + default-log-level: notice + outputs: + - console: + enabled: yes + - file: + enabled: yes + level: info + filename: suricata.log + +outputs: + - eve-log: + enabled: yes + types: + - files + - file-store: + version: 2 + enabled: yes + force-filestore: yes + max-open-files: 0 diff --git a/tests/filestore-fd-exhaustion/test.yaml b/tests/filestore-fd-exhaustion/test.yaml new file mode 100644 index 0000000000..2846d70353 --- /dev/null +++ b/tests/filestore-fd-exhaustion/test.yaml @@ -0,0 +1,33 @@ +requires: + min-version: 8 + + # I'm not sure how portable ulimit is. + os: linux + +# Run with a low file-descriptor ulimit. The pcap drives 48 concurrent HTTP +# downloads that file-store extracts at the same time. With the documented +# default (max-open-files: 0) Suricata should close each file after every +# write, so only a couple of descriptors are ever open. The regression keeps +# one descriptor open per in-progress file, so all ~48 are held at once, the +# 32-fd limit is exceeded, and file-store logs EMFILE ("Too many open files"). +command: | + ulimit -Sn 32 && \ + ${SRCDIR}/src/suricata \ + --set classification-file="${SRCDIR}/etc/classification.config" \ + --set reference-config-file="${SRCDIR}/etc/reference.config" \ + -c "${TEST_DIR}/suricata.yaml" \ + -l "${OUTPUT_DIR}" \ + --runmode single -k none \ + -r "${TEST_DIR}/input.pcap" + +checks: + # Sanity: a correct build extracts all 48 forced files. + - filter: + count: 48 + match: + event_type: fileinfo + fileinfo.stored: true + # Regression: a correct build never exhausts the fd table here. + # (test -s guards against a silent pass if suricata.log is ever missing.) + - shell: + args: "test -s suricata.log && ! grep -q 'Too many open files' suricata.log"