diff --git a/Directory.Packages.props b/Directory.Packages.props index e8982479d7..6ab5c8f72a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -55,6 +55,7 @@ + @@ -86,7 +87,6 @@ - diff --git a/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj b/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj index 0f96c00f34..501904008c 100644 --- a/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj +++ b/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/Elastic.Documentation.Configuration/FileSystemFactory.cs b/src/Elastic.Documentation.Configuration/FileSystemFactory.cs index 6bcbd362bf..4af7c9814b 100644 --- a/src/Elastic.Documentation.Configuration/FileSystemFactory.cs +++ b/src/Elastic.Documentation.Configuration/FileSystemFactory.cs @@ -67,6 +67,21 @@ public static class FileSystemFactory /// public static ScopedFileSystem InMemory() => new(new MockFileSystem(), WorkingDirectoryReadOptions); + /// + /// Creates a new wrapping a fresh , + /// scoped to the git root of so that paths such as + /// {sourceRoot}/.artifacts/docs/html are within the allowed write scope. + /// Falls back to when is . + /// + public static ScopedFileSystem InMemoryForSourceRoot(string? sourcePath) + { + if (sourcePath is null) + return InMemory(); + var root = Paths.FindGitRoot(sourcePath); + var inner = new MockFileSystem(); + return new ScopedFileSystem(inner, BuildWriteOptions(inner, root, Paths.ApplicationData.FullName)); + } + /// /// Scopes to and /// for reading. Use when the inner FS contains files diff --git a/src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs b/src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs index 6751278a2d..fd92a37e83 100644 --- a/src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs +++ b/src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs @@ -10,12 +10,22 @@ public record DetectionRuleOverviewRef : FileRef { public IReadOnlyCollection DetectionRuleFolders { get; } + /// Optional path to a markdown file whose content prefixes the deprecated rules listing page. + public string? DeprecatedFile { get; init; } + + /// + /// The resolved deprecated-rules overview FileRef that should appear as a sibling to this ref in the nav. + /// Set by ResolveRuleOverviewReference when a _deprecated subfolder is detected. + /// + public FileRef? DeprecatedSiblingRef { get; init; } + public DetectionRuleOverviewRef( string pathRelativeToDocumentationSet, string pathRelativeToContainer, IReadOnlyCollection detectionRulesFolders, IReadOnlyCollection children, - string context + string context, + string? deprecatedFile = null ) : base(pathRelativeToDocumentationSet, pathRelativeToContainer, false, children, context) { PathRelativeToDocumentationSet = pathRelativeToDocumentationSet; @@ -23,6 +33,7 @@ string context DetectionRuleFolders = detectionRulesFolders; Children = children; Context = context; + DeprecatedFile = deprecatedFile; } public static IReadOnlyCollection CreateTableOfContentItems(IReadOnlyCollection sourceFolders, string context, IDirectoryInfo baseDirectory) @@ -38,6 +49,18 @@ public static IReadOnlyCollection CreateTableOfContentItem .ToArray(); } + public static IReadOnlyCollection CreateDeprecatedTableOfContentItems(IReadOnlyCollection sourceFolders, string context, IDirectoryInfo baseDirectory) + { + var tocItems = new List(); + foreach (var detectionRuleFolder in sourceFolders) + { + var children = ReadDeprecatedDetectionRuleFolder(detectionRuleFolder, context, baseDirectory); + tocItems.AddRange(children); + } + + return tocItems.ToArray(); + } + private static IReadOnlyCollection ReadDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory) { IReadOnlyCollection children = directory @@ -62,4 +85,25 @@ private static IReadOnlyCollection ReadDetectionRuleFolder return children; } + + private static IReadOnlyCollection ReadDeprecatedDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory) + { + IReadOnlyCollection children = directory + .EnumerateFiles("*.*", SearchOption.AllDirectories) + .Where(f => !f.Attributes.HasFlag(FileAttributes.Hidden) && !f.Attributes.HasFlag(FileAttributes.System)) + .Where(f => !f.Directory!.Attributes.HasFlag(FileAttributes.Hidden) && !f.Directory!.Attributes.HasFlag(FileAttributes.System)) + // skip symlinks + .Where(f => f.LinkTarget == null) + .Where(f => f.Extension == ".toml") + // only include files inside _deprecated subdirectories + .Where(f => f.FullName.Contains($"{Path.DirectorySeparatorChar}_deprecated{Path.DirectorySeparatorChar}")) + .Select(f => + { + var relativePath = Path.GetRelativePath(baseDirectory.Parent!.FullName, f.FullName); + return (ITableOfContentsItem)new DetectionRuleRef(f, relativePath, context); + }) + .ToArray(); + + return children; + } } diff --git a/src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs b/src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs index 98c092de44..52320b9cd8 100644 --- a/src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs +++ b/src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs @@ -164,7 +164,12 @@ private static TableOfContents ResolveTableOfContents( }; if (resolvedItem != null) + { resolved.Add(resolvedItem); + // Emit the deprecated rules overview as a sibling immediately after the active rules ref + if (resolvedItem is DetectionRuleOverviewRef { DeprecatedSiblingRef: { } deprecatedSibling }) + resolved.Add(deprecatedSibling); + } } return resolved; @@ -443,9 +448,33 @@ private static ITableOfContentsItem ResolveRuleOverviewReference(IDiagnosticsCol .ToList(); var tomlChildren = DetectionRuleOverviewRef.CreateTableOfContentItems(tocSourceFolders, context, baseDirectory); - var children = resolvedChildren.Concat(tomlChildren).ToList(); + var children = resolvedChildren.ToList(); + children.AddRange(tomlChildren); - return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context); + // Auto-detect _deprecated subdirectories. When found, build the deprecated overview FileRef + // and attach it as DeprecatedSiblingRef so ResolveTableOfContents can emit it as a sibling, + // not as a child nested under the active rules. + FileRef? deprecatedSiblingRef = null; + var hasDeprecatedRules = tocSourceFolders.Any(d => + d.Exists && d.EnumerateDirectories("_deprecated", SearchOption.TopDirectoryOnly).Any()); + if (hasDeprecatedRules) + { + var deprecatedFileName = detectionRuleRef.DeprecatedFile ?? "deprecated-detection-rules.md"; + var overviewDir = fileSystem.Path.GetDirectoryName(fullPath); + var deprecatedFullPath = string.IsNullOrEmpty(overviewDir) + ? deprecatedFileName + : $"{overviewDir}/{deprecatedFileName}"; + var deprecatedPathRelativeToContainer = string.IsNullOrEmpty(containerPath) + ? deprecatedFullPath + : deprecatedFullPath.Substring(containerPath.Length + 1); + var deprecatedTomlChildren = DetectionRuleOverviewRef.CreateDeprecatedTableOfContentItems(tocSourceFolders, context, baseDirectory); + deprecatedSiblingRef = new FileRef(deprecatedFullPath, deprecatedPathRelativeToContainer, false, deprecatedTomlChildren, context); + } + + return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context, detectionRuleRef.DeprecatedFile) + { + DeprecatedSiblingRef = deprecatedSiblingRef + }; } diff --git a/src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs b/src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs index 428f6003a2..3cc11c7516 100644 --- a/src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs +++ b/src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs @@ -72,7 +72,7 @@ public class TocItemYamlConverter : IYamlTypeConverter } value = childrenList; } - else if (key.Value is "detection_rules" or "exclude") + else if (key.Value is "detection_rules" or "exclude" or "deprecated_detection_rules") { // Parse the children list manually var childrenList = new List(); @@ -135,11 +135,8 @@ public class TocItemYamlConverter : IYamlTypeConverter if (dictionary.TryGetValue("detection_rules", out var detectionRulesObj) && detectionRulesObj is string[] detectionRulesFolders && dictionary.TryGetValue("file", out var detectionRulesFilePath) && detectionRulesFilePath is string detectionRulesFile) { - // Create the index file reference (FolderIndexFileRef to mark it as the folder's index) - // Store ONLY the file name - the folder path will be prepended during resolution - // This allows validation to check if the file itself has deep paths - // PathRelativeToContainer will be set during resolution - return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext); + var deprecatedFile = dictionary.TryGetValue("deprecated_file", out var deprecatedFileObj) && deprecatedFileObj is string df ? df : null; + return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext, deprecatedFile); } // Check for file reference (file: or hidden:) diff --git a/src/Elastic.Documentation.Site/Assets/pages-nav.ts b/src/Elastic.Documentation.Site/Assets/pages-nav.ts index 5f9419abbe..a47260b956 100644 --- a/src/Elastic.Documentation.Site/Assets/pages-nav.ts +++ b/src/Elastic.Documentation.Site/Assets/pages-nav.ts @@ -134,8 +134,16 @@ export function initNav() { // Normalize pathname by removing trailing slash to handle both URL variants const pathname = window.location.pathname.replace(/\/$/, '') + + // When the page is a hidden nav item (e.g. an individual detection rule), the server + // emits docs:nav-active pointing to the nearest visible ancestor so we can highlight it. + const navActiveMeta = document.querySelector( + 'meta[name="docs:nav-active"]' + ) + const activePathname = navActiveMeta?.content ?? pathname + const navItems = $$( - 'a[href="' + pathname + '"], a[href="' + pathname + '/"]', + 'a[href="' + activePathname + '"], a[href="' + activePathname + '/"]', pagesNav ) navItems.forEach((el) => { diff --git a/src/Elastic.Documentation.Site/Layout/_Head.cshtml b/src/Elastic.Documentation.Site/Layout/_Head.cshtml index bf65fa113c..a961e3b23a 100644 --- a/src/Elastic.Documentation.Site/Layout/_Head.cshtml +++ b/src/Elastic.Documentation.Site/Layout/_Head.cshtml @@ -44,6 +44,10 @@ @await RenderPartialAsync(_Favicon.Create()) + @if (!string.IsNullOrEmpty(Model.NavigationActiveUrl)) + { + + } diff --git a/src/Elastic.Documentation.Site/_ViewModels.cs b/src/Elastic.Documentation.Site/_ViewModels.cs index d171e961e8..eed50e4462 100644 --- a/src/Elastic.Documentation.Site/_ViewModels.cs +++ b/src/Elastic.Documentation.Site/_ViewModels.cs @@ -44,6 +44,13 @@ public record GlobalLayoutViewModel /// Breadcrumb trail for codex sub-header (Home / Group / Docset). public IReadOnlyList? CodexBreadcrumbs { get; init; } + /// + /// When the current page is a hidden nav item (e.g. an individual detection rule page), + /// the URL of its nearest visible ancestor. The client uses this to highlight the correct + /// nav entry when the page has no rendered nav link of its own. + /// + public string? NavigationActiveUrl { get; init; } + // Header properties for isolated mode public string? HeaderTitle { get; init; } diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj index ed5f7a3a37..70425216c2 100644 --- a/src/Elastic.Markdown/Elastic.Markdown.csproj +++ b/src/Elastic.Markdown/Elastic.Markdown.csproj @@ -46,6 +46,10 @@ + + + + diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs index 7a66e4147a..b3af5994b0 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs @@ -9,8 +9,9 @@ using System.Text.Json; using System.Text.Json.Serialization; using Cysharp.IO; -using Tomlet; -using Tomlet.Models; +using Tomlyn; +using Tomlyn.Model; +using Tomlyn.Serialization; namespace Elastic.Markdown.Extensions.DetectionRules; @@ -35,6 +36,9 @@ public record VersionLockEntry [JsonSerializable(typeof(Dictionary))] internal sealed partial class VersionLockJsonContext : JsonSerializerContext; +[TomlSerializable(typeof(TomlTable))] +internal sealed partial class DetectionRuleTomlContext : TomlSerializerContext; + public record DetectionRuleThreat { public required string Framework { get; init; } @@ -63,9 +67,6 @@ public record DetectionRuleTechnique : DetectionRuleSubTechnique public record DetectionRule { - // Reuse a single TomlParser instance for better performance - private static readonly TomlParser Parser = new(); - // Cached version lock data, loaded once per build private static FrozenDictionary? VersionLock; @@ -103,6 +104,9 @@ public record DetectionRule public required DetectionRuleThreat[] Threats { get; init; } = []; + public required string? DeprecationDate { get; init; } + public required string? Maturity { get; init; } + /// /// Initializes the version lock cache from the version.lock.json file. /// This should be called once before processing detection rules. @@ -139,27 +143,46 @@ public static void InitializeVersionLock(IFileSystem fileSystem, IDirectoryInfo? [SuppressMessage("Reliability", "CA2012:Use ValueTasks correctly")] public static DetectionRule From(IFileInfo source) { - TomlDocument model; + TomlTable model; try { - // Use optimized Utf8StreamReader for better I/O performance using var reader = new Utf8StreamReader(source.FullName, fileOpenMode: FileOpenMode.Throughput); var sourceText = Encoding.UTF8.GetString(reader.ReadToEndAsync().GetAwaiter().GetResult()); - model = Parser.Parse(sourceText); + model = TomlSerializer.Deserialize(sourceText, DetectionRuleTomlContext.Default.TomlTable)!; } catch (Exception e) { throw new Exception($"Could not parse toml in: {source.FullName}", e); } - if (!model.TryGetValue("metadata", out var node) || node is not TomlTable) + if (!model.TryGetValue("metadata", out var metadataObj) || metadataObj is not TomlTable metadata) throw new Exception($"Could not find metadata section in {source.FullName}"); - if (!model.TryGetValue("rule", out node) || node is not TomlTable rule) + if (!model.TryGetValue("rule", out var ruleObj) || ruleObj is not TomlTable rule) throw new Exception($"Could not find rule section in {source.FullName}"); + try + { + return BuildRule(metadata, rule); + } + catch (Exception e) + { + throw new Exception($"Could not read fields from: {source.FullName}", e); + } + } + + internal static DetectionRule FromToml(string toml) + { + var model = TomlSerializer.Deserialize(toml, DetectionRuleTomlContext.Default.TomlTable)!; + var metadata = (TomlTable)model["metadata"]!; + var rule = (TomlTable)model["rule"]!; + return BuildRule(metadata, rule); + } + + private static DetectionRule BuildRule(TomlTable metadata, TomlTable rule) + { var threats = GetThreats(rule); - var ruleId = rule.GetString("rule_id"); + var ruleId = GetString(rule, "rule_id"); // Get max_signals from TOML, default to 100 if not specified var maxSignals = TryGetInt(rule, "max_signals") ?? 100; @@ -172,13 +195,13 @@ public static DetectionRule From(IFileInfo source) return new DetectionRule { Authors = TryGetStringArray(rule, "author"), - Description = rule.GetString("description"), - Type = rule.GetString("type"), + Description = GetString(rule, "description"), + Type = GetString(rule, "type"), Language = TryGetString(rule, "language"), - License = rule.GetString("license"), + License = GetString(rule, "license"), RiskScore = TryGetInt(rule, "risk_score") ?? 0, RuleId = ruleId, - Severity = rule.GetString("severity"), + Severity = GetString(rule, "severity"), Tags = TryGetStringArray(rule, "tags"), Indices = TryGetStringArray(rule, "index"), References = TryGetStringArray(rule, "references"), @@ -186,107 +209,97 @@ public static DetectionRule From(IFileInfo source) Setup = TryGetString(rule, "setup"), Query = TryGetString(rule, "query"), Note = TryGetString(rule, "note"), - Name = rule.GetString("name"), + Name = GetString(rule, "name"), RunsEvery = TryGetString(rule, "interval"), MaximumAlertsPerExecution = maxSignals, Version = version, - Threats = threats + Threats = threats, + DeprecationDate = TryGetString(metadata, "deprecation_date"), + Maturity = TryGetString(metadata, "maturity") }; } private static DetectionRuleThreat[] GetThreats(TomlTable model) { - if (!model.TryGetValue("threat", out var node) || node is not TomlArray threats) + if (!model.TryGetValue("threat", out var node) || node is not TomlTableArray threats) return []; - var threatsList = new List(threats.ArrayValues.Count); - foreach (var value in threats) + var threatsList = new List(threats.Count); + foreach (var threatTable in threats.OfType()) { - if (value is not TomlTable threatTable) - continue; - - var framework = threatTable.GetString("framework"); + var framework = GetString(threatTable, "framework"); var techniques = ReadTechniques(threatTable); - var tactic = ReadTactic(threatTable); - var threat = new DetectionRuleThreat + threatsList.Add(new DetectionRuleThreat { Framework = framework, - Techniques = techniques.ToArray(), + Techniques = techniques, Tactic = tactic - }; - threatsList.Add(threat); + }); } - return threatsList.ToArray(); + return [.. threatsList]; } - private static IReadOnlyCollection ReadTechniques(TomlTable threatTable) + private static DetectionRuleTechnique[] ReadTechniques(TomlTable threatTable) { - var techniquesArray = threatTable.TryGetValue("technique", out var node) && node is TomlArray ta ? ta : null; - if (techniquesArray is null) + if (!threatTable.TryGetValue("technique", out var node) || node is not TomlTableArray techniquesArray) return []; + var techniques = new List(techniquesArray.Count); - foreach (var t in techniquesArray) + foreach (var techniqueTable in techniquesArray.OfType()) { - if (t is not TomlTable techniqueTable) - continue; - var id = techniqueTable.GetString("id"); - var name = techniqueTable.GetString("name"); - var reference = techniqueTable.GetString("reference"); techniques.Add(new DetectionRuleTechnique { - Id = id, - Name = name, - Reference = reference, - SubTechniques = ReadSubTechniques(techniqueTable).ToArray() + Id = GetString(techniqueTable, "id"), + Name = GetString(techniqueTable, "name"), + Reference = GetString(techniqueTable, "reference"), + SubTechniques = ReadSubTechniques(techniqueTable) }); } - return techniques; + return [.. techniques]; } - private static IReadOnlyCollection ReadSubTechniques(TomlTable techniqueTable) + + private static DetectionRuleSubTechnique[] ReadSubTechniques(TomlTable techniqueTable) { - var subArray = techniqueTable.TryGetValue("subtechnique", out var node) && node is TomlArray ta ? ta : null; - if (subArray is null) + if (!techniqueTable.TryGetValue("subtechnique", out var node) || node is not TomlTableArray subArray) return []; + var subTechniques = new List(subArray.Count); - foreach (var t in subArray) + foreach (var subTable in subArray.OfType()) { - if (t is not TomlTable subTechniqueTable) - continue; - var id = subTechniqueTable.GetString("id"); - var name = subTechniqueTable.GetString("name"); - var reference = subTechniqueTable.GetString("reference"); subTechniques.Add(new DetectionRuleSubTechnique { - Id = id, - Name = name, - Reference = reference + Id = GetString(subTable, "id"), + Name = GetString(subTable, "name"), + Reference = GetString(subTable, "reference") }); } - return subTechniques; + return [.. subTechniques]; } private static DetectionRuleTactic ReadTactic(TomlTable threatTable) { - var tacticTable = threatTable.GetSubTable("tactic"); - var id = tacticTable.GetString("id"); - var name = tacticTable.GetString("name"); - var reference = tacticTable.GetString("reference"); + if (!threatTable.TryGetValue("tactic", out var tacticObj) || tacticObj is not TomlTable tacticTable) + throw new InvalidOperationException("Threat entry is missing required 'tactic' section"); + return new DetectionRuleTactic { - Id = id, - Name = name, - Reference = reference + Id = GetString(tacticTable, "id"), + Name = GetString(tacticTable, "name"), + Reference = GetString(tacticTable, "reference") }; } + private static string GetString(TomlTable table, string key) => + (string)table[key]; + private static string[]? TryGetStringArray(TomlTable table, string key) => - table.TryGetValue(key, out var node) && node is TomlArray t ? t.ArrayValues.Select(value => value.StringValue).ToArray() : null; + table.TryGetValue(key, out var node) && node is TomlArray t ? t.OfType().ToArray() : null; private static string? TryGetString(TomlTable table, string key) => - table.TryGetValue(key, out var node) && node is TomlString t ? t.Value : null; + table.TryGetValue(key, out var node) && node is string s ? s : null; private static int? TryGetInt(TomlTable table, string key) => - table.TryGetValue(key, out var node) && node is TomlLong t ? (int)t.Value : null; + table.TryGetValue(key, out var node) && node is long l ? (int)l : null; } diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs index 059fe728e7..eae417fcea 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs @@ -11,6 +11,68 @@ namespace Elastic.Markdown.Extensions.DetectionRules; +public record DeprecatedDetectionRuleOverviewFile : MarkdownFile +{ + public DeprecatedDetectionRuleOverviewFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext build) + : base(sourceFile, rootPath, parser, build) + { + } + + internal ILeafNavigationItem[] RuleNavigations { get; set; } = []; + + protected override Task GetMinimalParseDocumentAsync(Cancel ctx) + { + var markdown = GetMarkdown(); + var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null); + return Task.FromResult(document); + } + + protected override Task GetParseDocumentAsync(Cancel ctx) + { + var markdown = GetMarkdown(); + var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null); + return Task.FromResult(document); + } + + private string GetMarkdown() + { + var rules = RuleNavigations.Select(nav => (Navigation: nav, Model: (DetectionRuleFile)nav.Model)).ToList(); + + string intro; + if (SourceFile.Exists) + { + var content = SourceFile.FileSystem.File.ReadAllText(SourceFile.FullName); + // Extract H1 title from the file if present, use remainder as intro body + var lines = content.Split('\n'); + var titleLine = lines.FirstOrDefault(l => l.TrimStart().StartsWith("# ", StringComparison.Ordinal)); + if (titleLine != null) + Title = titleLine.TrimStart().Substring(2).Trim(); + intro = content; + } + else + { + Title = "Deprecated prebuilt detection rules"; + intro = "# Deprecated prebuilt detection rules\n\n"; + } + + var markdown = intro + "\n\n"; + + var groupedRules = rules + .GroupBy(r => r.Model.Rule.Domain ?? "Unspecified") + .OrderBy(g => g.Key) + .ToArray(); + + foreach (var group in groupedRules) + { + markdown += $"\n## {group.Key}\n\n"; + foreach (var (navigation, model) in group.OrderBy(r => r.Model.Rule.Name)) + markdown += $"[{model.Rule.Name}](!{navigation.Url})
\n"; + } + + return markdown; + } +} + public record DetectionRuleOverviewFile : MarkdownFile { public DetectionRuleOverviewFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext build) @@ -102,7 +164,8 @@ BuildContext build private static IFileInfo SourcePath(IFileInfo rulePath, BuildContext build) { - var relative = Path.GetRelativePath(build.DocumentationCheckoutDirectory!.FullName, rulePath.FullName); + var checkoutDir = build.DocumentationCheckoutDirectory ?? build.DocumentationSourceDirectory.Parent!; + var relative = Path.GetRelativePath(checkoutDir.FullName, rulePath.FullName); var newPath = Path.Join(build.DocumentationSourceDirectory.FullName, relative); var md = Path.ChangeExtension(newPath, ".md"); return rulePath.FileSystem.FileInfo.New(md); @@ -136,12 +199,21 @@ protected override Task GetParseDocumentAsync(Cancel ctx) private string GetMarkdown() { + var deprecationNotice = Rule.Maturity == "deprecated" + ? $""" + :::{"{warning}"} + This rule has been deprecated{(Rule.DeprecationDate != null ? $" as of {Rule.DeprecationDate}" : "")}. + ::: + + """ + : string.Empty; + // language=markdown var markdown = $""" # {Rule.Name} -{Rule.Description} +{deprecationNotice}{Rule.Description} **Rule type**: {Rule.Type}
**Rule indices**: {RenderArray(Rule.Indices)} diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs index 82924f929b..0c7424540b 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs @@ -16,13 +16,16 @@ namespace Elastic.Markdown.Extensions.DetectionRules; public class DetectionRulesDocsBuilderExtension(BuildContext build) : IDocsBuilderExtension { - private BuildContext Build { get; } = build; + private BuildContext Build => build; private bool _versionLockInitialized; + private IReadOnlySet DeprecatedOverviewFileNames { get; } = + GetAllDetectionRuleOverviewRefs(build.ConfigurationYaml.TableOfContents) + .Select(r => r.DeprecatedFile ?? "deprecated-detection-rules.md") + .ToHashSet(); + public IEnumerable ExternalScopeRoots => - Build.ConfigurationYaml.TableOfContents - .OfType() - .SelectMany(f => f.Children.OfType()) + GetAllDetectionRuleOverviewRefs(Build.ConfigurationYaml.TableOfContents) .SelectMany(r => r.DetectionRuleFolders) .Select(f => Path.GetFullPath(f, Build.DocumentationSourceDirectory.FullName)) .Distinct(); @@ -31,6 +34,10 @@ public class DetectionRulesDocsBuilderExtension(BuildContext build) : IDocsBuild public DocumentationFile? CreateDocumentationFile(IFileInfo file, MarkdownParser markdownParser) { + // Handle the synthetic deprecated overview .md file when no physical file exists on disk + if (file.Extension == ".md" && DeprecatedOverviewFileNames.Contains(file.Name)) + return new DeprecatedDetectionRuleOverviewFile(file, Build.DocumentationSourceDirectory, markdownParser, Build); + if (file.Extension != ".toml") return null; @@ -44,22 +51,36 @@ public class DetectionRulesDocsBuilderExtension(BuildContext build) : IDocsBuild return new DetectionRuleFile(file, Build.DocumentationSourceDirectory, markdownParser, Build); } - public MarkdownFile? CreateMarkdownFile(IFileInfo file, IDirectoryInfo sourceDirectory, MarkdownParser markdownParser) => - file.Name != "index.md" ? null : new DetectionRuleOverviewFile(file, sourceDirectory, markdownParser, Build); + public MarkdownFile? CreateMarkdownFile(IFileInfo file, IDirectoryInfo sourceDirectory, MarkdownParser markdownParser) + { + if (file.Name == "index.md") + return new DetectionRuleOverviewFile(file, sourceDirectory, markdownParser, Build); + // Physical deprecated_file on disk takes priority over the synthetic path + if (DeprecatedOverviewFileNames.Contains(file.Name)) + return new DeprecatedDetectionRuleOverviewFile(file, sourceDirectory, markdownParser, Build); + return null; + } /// public void VisitNavigation(INavigationItem navigation, IDocumentationFile model) { - if (model is not DetectionRuleOverviewFile overview) - return; if (navigation is not VirtualFileNavigation node) return; - var detectionRuleNavigations = node.NavigationItems + + var ruleNavigations = node.NavigationItems .OfType>() .Where(n => n.Model is DetectionRuleFile) .ToArray(); - overview.RuleNavigations = detectionRuleNavigations; + switch (model) + { + case DetectionRuleOverviewFile overview: + overview.RuleNavigations = ruleNavigations; + break; + case DeprecatedDetectionRuleOverviewFile deprecatedOverview: + deprecatedOverview.RuleNavigations = ruleNavigations; + break; + } } public bool TryGetDocumentationFileBySlug(DocumentationSet documentationSet, string slug, out DocumentationFile? documentationFile) @@ -71,11 +92,65 @@ public bool TryGetDocumentationFileBySlug(DocumentationSet documentationSet, str public IReadOnlyCollection<(IFileInfo, DocumentationFile)> ScanDocumentationFiles(Func defaultFileHandling) { - var rules = Build.ConfigurationYaml.TableOfContents.OfType().First().Children.OfType().ToArray(); - if (rules.Length == 0) + var overviewRefs = GetAllDetectionRuleOverviewRefs(Build.ConfigurationYaml.TableOfContents).ToArray(); + + // Pass each overviewRef as a single-element sequence so the switch case that + // checks DeprecatedSiblingRef is triggered, picking up both active and deprecated rules. + var rules = overviewRefs + .SelectMany(r => GetAllDetectionRuleRefs([r])) + .ToArray(); + + var result = rules + .Select(r => (r.FileInfo, defaultFileHandling(r.FileInfo, r.FileInfo.Directory!))) + .ToList(); + + // Pre-register synthetic deprecated overview files for overviews without a physical file. + // When the physical file exists it is already picked up by the normal source directory scan. + foreach (var overviewRef in overviewRefs) + { + var deprecatedFileName = overviewRef.DeprecatedFile ?? "deprecated-detection-rules.md"; + var syntheticPath = Build.ReadFileSystem.Path.Join(Build.DocumentationSourceDirectory.FullName, deprecatedFileName); + var syntheticFileInfo = Build.ReadFileSystem.FileInfo.New(syntheticPath); + + if (syntheticFileInfo.Exists) + continue; // physical file handled by normal scan + + // Only register if this overview actually has deprecated rule children (now in DeprecatedSiblingRef) + var hasDeprecatedChildren = overviewRef.DeprecatedSiblingRef?.Children.OfType().Any() == true; + + if (!hasDeprecatedChildren) + continue; + + var deprecatedFile = defaultFileHandling(syntheticFileInfo, Build.DocumentationSourceDirectory); + if (deprecatedFile is not ExcludedFile) + result.Add((syntheticFileInfo, deprecatedFile)); + } + + if (result.Count == 0) return []; - return rules.Select(r => (r.FileInfo, defaultFileHandling(r.FileInfo, r.FileInfo.Directory!))).ToArray(); + return result.ToArray(); } + // Finds all DetectionRuleOverviewRef instances at any depth in the TOC tree + private static IEnumerable GetAllDetectionRuleOverviewRefs(IEnumerable items) => + items.SelectMany(item => item switch + { + DetectionRuleOverviewRef r => [r], + FileRef fr => GetAllDetectionRuleOverviewRefs(fr.Children), + _ => [] + }); + + // Finds all DetectionRuleRef instances at any depth within a set of TOC items. + // Also scans DeprecatedSiblingRef on DetectionRuleOverviewRef since deprecated rules + // are no longer nested in Children — they live in the sibling's Children instead. + private static IEnumerable GetAllDetectionRuleRefs(IEnumerable items) => + items.SelectMany(item => item switch + { + DetectionRuleRef dr => [dr], + DetectionRuleOverviewRef r when r.DeprecatedSiblingRef is { } dep => + GetAllDetectionRuleRefs(r.Children).Concat(GetAllDetectionRuleRefs(dep.Children)), + FileRef fr => GetAllDetectionRuleRefs(fr.Children), + _ => [] + }); } diff --git a/src/Elastic.Markdown/HtmlWriter.cs b/src/Elastic.Markdown/HtmlWriter.cs index b2bcfa7e05..5c5da224ec 100644 --- a/src/Elastic.Markdown/HtmlWriter.cs +++ b/src/Elastic.Markdown/HtmlWriter.cs @@ -82,6 +82,10 @@ private async Task RenderLayout(MarkdownFile markdown, MarkdownDoc var next = NavigationTraversable.GetNext(markdown); var parents = NavigationTraversable.GetParentsOfMarkdownFile(markdown); + // For hidden nav items (e.g. individual detection rule pages) there is no rendered nav link, + // so JS can't mark anything as current. Point it at the nearest visible ancestor instead. + var navActiveUrl = current.Hidden ? parents.FirstOrDefault(p => !p.Hidden)?.Url : null; + var remote = DocumentationSet.Context.Git.RepositoryName; var branch = DocumentationSet.Context.Git.Branch; string? editUrl = null; @@ -157,6 +161,7 @@ private async Task RenderLayout(MarkdownFile markdown, MarkdownDoc PreviousDocument = previous, NextDocument = next, Breadcrumbs = breadcrumbs, + NavigationActiveUrl = navActiveUrl, NavigationHtml = navigationHtmlRenderResult.Html, UrlPathPrefix = markdown.UrlPathPrefix, SiteRootPath = DocumentationSet.Context.SiteRootPath, diff --git a/src/Elastic.Markdown/Page/Index.cshtml b/src/Elastic.Markdown/Page/Index.cshtml index c73cdb6f55..43054f6dd4 100644 --- a/src/Elastic.Markdown/Page/Index.cshtml +++ b/src/Elastic.Markdown/Page/Index.cshtml @@ -48,6 +48,7 @@ StaticFileContentHashProvider = Model.StaticFileContentHashProvider, ReportIssueUrl = Model.ReportIssueUrl, Breadcrumbs = Model.Breadcrumbs, + NavigationActiveUrl = Model.NavigationActiveUrl, Htmx = new DefaultHtmxAttributeProvider(GetRootPath(Model.SiteRootPath, Model.UrlPathPrefix)), CurrentVersion = Model.CurrentDocument.YamlFrontMatter?.Layout == MarkdownPageLayout.LandingPage ? Model.VersionsConfig.VersioningSystems[0].Current : Model.CurrentVersion, AllVersionsUrl = Model.AllVersionsUrl, diff --git a/src/Elastic.Markdown/Page/IndexViewModel.cs b/src/Elastic.Markdown/Page/IndexViewModel.cs index f64968f5e1..bf22d03a80 100644 --- a/src/Elastic.Markdown/Page/IndexViewModel.cs +++ b/src/Elastic.Markdown/Page/IndexViewModel.cs @@ -34,6 +34,12 @@ public class IndexViewModel public required INavigationItem? NextDocument { get; init; } public required INavigationItem[] Breadcrumbs { get; init; } + /// + /// When the current page is a hidden nav item, the URL of its nearest visible ancestor. + /// Emitted as a meta tag so JavaScript can highlight the correct nav entry. + /// + public string? NavigationActiveUrl { get; init; } + public required string NavigationHtml { get; init; } public required string? CurrentVersion { get; init; } diff --git a/src/tooling/docs-builder/Http/InMemoryBuildState.cs b/src/tooling/docs-builder/Http/InMemoryBuildState.cs index 43154396c9..df67729f09 100644 --- a/src/tooling/docs-builder/Http/InMemoryBuildState.cs +++ b/src/tooling/docs-builder/Http/InMemoryBuildState.cs @@ -53,8 +53,11 @@ public class InMemoryBuildState(ILoggerFactory loggerFactory, IConfigurationCont private readonly Lock _diagnosticsLock = new(); private readonly List _diagnostics = []; - // Reuse MockFileSystem across builds to benefit from caching - private readonly ScopedFileSystem _writeFs = FileSystemFactory.InMemory(); + // Reuse MockFileSystem across builds to benefit from caching. + // Scoped lazily to the git root of the first sourcePath seen, so paths like + // {externalRepo}/.artifacts/docs/html are within the allowed write scope. + private ScopedFileSystem? _writeFs; + private readonly Lock _writeFsLock = new(); // Broadcast: maintain list of connected client channels private readonly Lock _clientsLock = new(); @@ -170,6 +173,7 @@ private async Task ExecuteBuildAsync(string sourcePath, Cancel ct) var streamingCollector = new StreamingDiagnosticsCollector(_loggerFactory, this); var readFs = FileSystemFactory.RealGitRootForPath(sourcePath); + var writeFs = GetOrCreateWriteFs(sourcePath); var service = new IsolatedBuildService(_loggerFactory, _configurationContext, new NullCoreService(), SystemEnvironmentVariables.Instance); _logger.LogInformation("Starting in-memory validation build for {Path}", sourcePath); @@ -186,7 +190,7 @@ private async Task ExecuteBuildAsync(string sourcePath, Cancel ct) false, // metadataOnly ExportOptions.Default, null, // canonicalBaseUrl - _writeFs, // reuse MockFileSystem across builds for caching + writeFs, // reuse MockFileSystem across builds for caching true, // skipOpenApi - skip for faster validation builds false, // skipCrossLinks - enable cross-links (cached in MockFileSystem) ct @@ -308,6 +312,19 @@ public DiagnosticDto[] GetStoredDiagnostics() Diagnostics: GetStoredDiagnostics() ); + private ScopedFileSystem GetOrCreateWriteFs(string sourcePath) + { + if (_writeFs is not null) + return _writeFs; + lock (_writeFsLock) + { + if (_writeFs is not null) + return _writeFs; + _writeFs = FileSystemFactory.InMemoryForSourceRoot(sourcePath); + return _writeFs; + } + } + public void Dispose() { _currentBuildCts?.Cancel(); diff --git a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs index 1cf66d4d90..c20762a943 100644 --- a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs +++ b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs @@ -42,7 +42,8 @@ public async Task StartAsync(Cancel cancellationToken) _serviceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); // Run live reload and in-memory validation build in parallel - var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; + var sourcePath = ReloadableGenerator.Generator.Context.DocumentationCheckoutDirectory?.FullName + ?? ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; await Task.WhenAll( ReloadableGenerator.ReloadAsync(cancellationToken), InMemoryBuildState.StartBuildAsync(sourcePath, cancellationToken) @@ -88,7 +89,8 @@ private void Reload(bool reloadConfiguration = false) var token = _serviceCts?.Token ?? Cancel.None; _ = _debouncer.ExecuteAsync(async ctx => { - var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; + var sourcePath = ReloadableGenerator.Generator.Context.DocumentationCheckoutDirectory?.FullName + ?? ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; // Start in-memory validation build (runs in parallel) var validationTask = InMemoryBuildState.StartBuildAsync(sourcePath, ctx); diff --git a/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs b/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs new file mode 100644 index 0000000000..e9fc329e03 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs @@ -0,0 +1,274 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using AwesomeAssertions; +using Elastic.Markdown.Extensions.DetectionRules; + +namespace Elastic.Markdown.Tests.DetectionRules; + +public class DetectionRuleParsingTests +{ + private const string MinimalRule = """ + [metadata] + creation_date = "2024/08/01" + maturity = "production" + + [rule] + author = ["Elastic"] + description = "Test rule" + name = "Test Rule" + rule_id = "abc-123" + risk_score = 47 + severity = "medium" + type = "query" + license = "Elastic License v2" + language = "kuery" + query = "process.name : evil.exe" + """; + + [Fact] + public void FromToml_MinimalRule_ParsesCorrectly() + { + var rule = DetectionRule.FromToml(MinimalRule); + + rule.Name.Should().Be("Test Rule"); + rule.RuleId.Should().Be("abc-123"); + rule.Type.Should().Be("query"); + rule.RiskScore.Should().Be(47); + rule.Severity.Should().Be("medium"); + rule.Authors.Should().ContainSingle().Which.Should().Be("Elastic"); + } + + [Fact] + public void FromToml_ImplicitIntermediateTable_ParsesTransformInvestigate() + { + var toml = MinimalRule + """ + + [[transform.investigate]] + label = "Alerts associated with the user" + description = "" + providers = [] + + [[transform.investigate]] + label = "Alerts associated with the host" + description = "" + providers = [] + + [[rule.threat]] + framework = "MITRE ATT&CK" + [rule.threat.tactic] + id = "TA0011" + name = "Command and Control" + reference = "https://attack.mitre.org/tactics/TA0011/" + [[rule.threat.technique]] + id = "T1071" + name = "Application Layer Protocol" + reference = "https://attack.mitre.org/techniques/T1071/" + """; + + var rule = DetectionRule.FromToml(toml); + + rule.Threats.Should().HaveCount(1); + rule.Threats[0].Tactic.Id.Should().Be("TA0011"); + rule.Threats[0].Techniques.Should().HaveCount(1); + rule.Threats[0].Techniques[0].Id.Should().Be("T1071"); + } + + [Fact] + public void FromToml_MultiLineStringWithMarkdownLinks_ParsesCorrectly() + { + // TOML uses """ for multi-line strings; use 4-quote C# raw literals to embed them + var toml = """" + [metadata] + creation_date = "2024/08/01" + maturity = "production" + + [rule] + author = ["Elastic"] + description = "Test rule" + name = "Test Rule With Setup" + rule_id = "abc-456" + risk_score = 73 + severity = "high" + type = "esql" + license = "Elastic License v2" + setup = """ + ## Setup + + Follow the [helper guide](https://www.elastic.co/docs/some/path#anchor) to configure. + + Also see [another link](https://example.com). + """ + """"; + + var rule = DetectionRule.FromToml(toml); + + rule.Name.Should().Be("Test Rule With Setup"); + rule.Setup.Should().Contain("[helper guide]"); + rule.Setup.Should().Contain("elastic.co/docs"); + } + + [Fact] + public void FromToml_MixedMultiLineDelimiters_ParsesCorrectly() + { + // Triple-quoted """ appears inside a '''-delimited multi-line string + var toml = """" + [metadata] + creation_date = "2024/08/01" + maturity = "production" + + [rule] + author = ["Elastic"] + description = "Test rule" + name = "Mixed Delimiters" + rule_id = "abc-789" + risk_score = 50 + severity = "medium" + type = "esql" + license = "Elastic License v2" + query = ''' + from logs-endpoint.events.process-* + | grok process.command_line """e=Access&y=Guest&h=(?[^&]+)&p""" + | where Esql.server is not null + ''' + note = """## Triage + Check the process tree.""" + """"; + + var rule = DetectionRule.FromToml(toml); + + rule.Query.Should().Contain("grok process.command_line"); + rule.Query.Should().Contain("e=Access"); + rule.Note.Should().Contain("Triage"); + } + + [Fact] + public void FromToml_DeprecatedRule_ParsesDeprecationDate() + { + var toml = """ + [metadata] + creation_date = "2024/08/01" + deprecation_date = "2025/03/15" + maturity = "deprecated" + + [rule] + author = ["Elastic"] + description = "Deprecated rule" + name = "Old Rule" + rule_id = "dep-001" + risk_score = 21 + severity = "low" + type = "query" + license = "Elastic License v2" + """; + + var rule = DetectionRule.FromToml(toml); + + rule.DeprecationDate.Should().Be("2025/03/15"); + rule.Maturity.Should().Be("deprecated"); + } + + [Fact] + public void FromToml_ThreatWithSubTechniques_ParsesFullHierarchy() + { + var toml = MinimalRule + """ + + [[rule.threat]] + framework = "MITRE ATT&CK" + [rule.threat.tactic] + id = "TA0001" + name = "Initial Access" + reference = "https://attack.mitre.org/tactics/TA0001/" + [[rule.threat.technique]] + id = "T1566" + name = "Phishing" + reference = "https://attack.mitre.org/techniques/T1566/" + [[rule.threat.technique.subtechnique]] + id = "T1566.001" + name = "Spearphishing Attachment" + reference = "https://attack.mitre.org/techniques/T1566/001/" + [[rule.threat.technique.subtechnique]] + id = "T1566.002" + name = "Spearphishing Link" + reference = "https://attack.mitre.org/techniques/T1566/002/" + """; + + var rule = DetectionRule.FromToml(toml); + + rule.Threats.Should().HaveCount(1); + var technique = rule.Threats[0].Techniques[0]; + technique.Id.Should().Be("T1566"); + technique.SubTechniques.Should().HaveCount(2); + technique.SubTechniques[0].Id.Should().Be("T1566.001"); + technique.SubTechniques[1].Id.Should().Be("T1566.002"); + } + + [Fact] + public void FromToml_MultipleThreats_ParsesAll() + { + var toml = MinimalRule + """ + + [[rule.threat]] + framework = "MITRE ATT&CK" + [rule.threat.tactic] + id = "TA0011" + name = "Command and Control" + reference = "https://attack.mitre.org/tactics/TA0011/" + + [[rule.threat]] + framework = "MITRE ATT&CK" + [rule.threat.tactic] + id = "TA0005" + name = "Defense Evasion" + reference = "https://attack.mitre.org/tactics/TA0005/" + """; + + var rule = DetectionRule.FromToml(toml); + + rule.Threats.Should().HaveCount(2); + rule.Threats[0].Tactic.Id.Should().Be("TA0011"); + rule.Threats[1].Tactic.Id.Should().Be("TA0005"); + } + + [Fact] + public void FromToml_OptionalFieldsMissing_DefaultsCorrectly() + { + var rule = DetectionRule.FromToml(MinimalRule); + + rule.Setup.Should().BeNull(); + rule.Note.Should().BeNull(); + rule.References.Should().BeNull(); + rule.Indices.Should().BeNull(); + rule.RunsEvery.Should().BeNull(); + rule.IndicesFromDateMath.Should().BeNull(); + rule.DeprecationDate.Should().BeNull(); + rule.MaximumAlertsPerExecution.Should().Be(100); + rule.Threats.Should().BeEmpty(); + } + + [Fact] + public void FromToml_DomainTag_ExtractedCorrectly() + { + var toml = """ + [metadata] + creation_date = "2024/08/01" + maturity = "production" + + [rule] + author = ["Elastic"] + description = "Test" + name = "Domain Test" + rule_id = "dom-001" + risk_score = 50 + severity = "medium" + type = "query" + license = "Elastic License v2" + tags = ["Domain: Endpoint", "Use Case: Threat Detection"] + """; + + var rule = DetectionRule.FromToml(toml); + + rule.Domain.Should().Be("Endpoint"); + } +}