Surface deprecated prebuilt detection rules as a sibling nav page

Directory.Packages.props

@@ -43,7 +43,7 @@
     <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="10.0.0" />
     <PackageVersion Include="Nullean.ScopedFileSystem" Version="0.4.0" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.13.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.15.2" />
     <PackageVersion Include="Generator.Equals" Version="4.0.0" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
     <PackageVersion Include="KubernetesClient" Version="18.0.5" />
     <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
@@ -102,11 +102,11 @@
   <ItemGroup>
     <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.0.0" />
-    <PackageVersion Include="OpenTelemetry" Version="1.13.1" />
-    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.13.1" />
-    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.13.1" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.13.0" />
-    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.13.0" />
+    <PackageVersion Include="OpenTelemetry" Version="1.15.3" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.3" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.15.3" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.15.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.15.1" />
   </ItemGroup>
   <!-- Test packages -->
   <ItemGroup>
@@ -120,7 +120,7 @@
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.3" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
-    <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.13.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.15.3" />
     <PackageVersion Include="System.IO.Abstractions.TestingHelpers" Version="22.1.1" />
     <PackageVersion Include="Unquote" Version="7.0.1" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.0">

src/Elastic.Documentation.Configuration/FileSystemFactory.cs

@@ -67,6 +67,21 @@ public static class FileSystemFactory
 	/// </summary>
 	public static ScopedFileSystem InMemory() => new(new MockFileSystem(), WorkingDirectoryReadOptions);
 
+	/// <summary>
+	/// Creates a new <see cref="ScopedFileSystem"/> wrapping a fresh <see cref="MockFileSystem"/>,
+	/// scoped to the git root of <paramref name="sourcePath"/> so that paths such as
+	/// <c>{sourceRoot}/.artifacts/docs/html</c> are within the allowed write scope.
+	/// Falls back to <see cref="InMemory()"/> when <paramref name="sourcePath"/> is <see langword="null"/>.
+	/// </summary>
+	public static ScopedFileSystem InMemoryForSourceRoot(string? sourcePath)
+	{
+		if (sourcePath is null)
+			return InMemory();
+		var root = Paths.FindGitRoot(sourcePath);
+		var inner = new MockFileSystem();
+		return new ScopedFileSystem(inner, BuildWriteOptions(inner, root, Paths.ApplicationData.FullName));
+	}
+
 	/// <summary>
 	/// Scopes <paramref name="inner"/> to <see cref="Paths.WorkingDirectoryRoot"/> and
 	/// <see cref="Paths.ApplicationData"/> for reading. Use when the inner FS contains files

src/Elastic.Documentation.Configuration/Toc/DetectionRules/DetectionRuleOverviewRef.cs

@@ -10,19 +10,30 @@ public record DetectionRuleOverviewRef : FileRef
 {
 	public IReadOnlyCollection<string> DetectionRuleFolders { get; }
 
+	/// <summary>Optional path to a markdown file whose content prefixes the deprecated rules listing page.</summary>
+	public string? DeprecatedFile { get; init; }
+
+	/// <summary>
+	/// The resolved deprecated-rules overview FileRef that should appear as a sibling to this ref in the nav.
+	/// Set by <c>ResolveRuleOverviewReference</c> when a <c>_deprecated</c> subfolder is detected.
+	/// </summary>
+	public FileRef? DeprecatedSiblingRef { get; init; }
+
 	public DetectionRuleOverviewRef(
 		string pathRelativeToDocumentationSet,
 		string pathRelativeToContainer,
 		IReadOnlyCollection<string> detectionRulesFolders,
 		IReadOnlyCollection<ITableOfContentsItem> children,
-		string context
+		string context,
+		string? deprecatedFile = null
 	) : base(pathRelativeToDocumentationSet, pathRelativeToContainer, false, children, context)
 	{
 		PathRelativeToDocumentationSet = pathRelativeToDocumentationSet;
 		PathRelativeToContainer = pathRelativeToContainer;
 		DetectionRuleFolders = detectionRulesFolders;
 		Children = children;
 		Context = context;
+		DeprecatedFile = deprecatedFile;
 	}
 
 	public static IReadOnlyCollection<ITableOfContentsItem> CreateTableOfContentItems(IReadOnlyCollection<IDirectoryInfo> sourceFolders, string context, IDirectoryInfo baseDirectory)
@@ -38,6 +49,18 @@ public static IReadOnlyCollection<ITableOfContentsItem> CreateTableOfContentItem
 			.ToArray();
 	}
 
+	public static IReadOnlyCollection<ITableOfContentsItem> CreateDeprecatedTableOfContentItems(IReadOnlyCollection<IDirectoryInfo> sourceFolders, string context, IDirectoryInfo baseDirectory)
+	{
+		var tocItems = new List<ITableOfContentsItem>();
+		foreach (var detectionRuleFolder in sourceFolders)
+		{
+			var children = ReadDeprecatedDetectionRuleFolder(detectionRuleFolder, context, baseDirectory);
+			tocItems.AddRange(children);
+		}
+
+		return tocItems.ToArray();
+	}
+
 	private static IReadOnlyCollection<ITableOfContentsItem> ReadDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory)
 	{
 		IReadOnlyCollection<ITableOfContentsItem> children = directory
@@ -62,4 +85,25 @@ private static IReadOnlyCollection<ITableOfContentsItem> ReadDetectionRuleFolder
 
 		return children;
 	}
+
+	private static IReadOnlyCollection<ITableOfContentsItem> ReadDeprecatedDetectionRuleFolder(IDirectoryInfo directory, string context, IDirectoryInfo baseDirectory)
+	{
+		IReadOnlyCollection<ITableOfContentsItem> children = directory
+			.EnumerateFiles("*.*", SearchOption.AllDirectories)
+			.Where(f => !f.Attributes.HasFlag(FileAttributes.Hidden) && !f.Attributes.HasFlag(FileAttributes.System))
+			.Where(f => !f.Directory!.Attributes.HasFlag(FileAttributes.Hidden) && !f.Directory!.Attributes.HasFlag(FileAttributes.System))
+			// skip symlinks
+			.Where(f => f.LinkTarget == null)
+			.Where(f => f.Extension == ".toml")
+			// only include files inside _deprecated subdirectories
+			.Where(f => f.FullName.Contains($"{Path.DirectorySeparatorChar}_deprecated{Path.DirectorySeparatorChar}"))
+			.Select(f =>
+			{
+				var relativePath = Path.GetRelativePath(baseDirectory.Parent!.FullName, f.FullName);
+				return (ITableOfContentsItem)new DetectionRuleRef(f, relativePath, context);
+			})
+			.ToArray();
+
+		return children;
+	}
 }

src/Elastic.Documentation.Configuration/Toc/DocumentationSetFile.cs

@@ -164,7 +164,12 @@ private static TableOfContents ResolveTableOfContents(
 			};
 
 			if (resolvedItem != null)
+			{
 				resolved.Add(resolvedItem);
+				// Emit the deprecated rules overview as a sibling immediately after the active rules ref
+				if (resolvedItem is DetectionRuleOverviewRef { DeprecatedSiblingRef: { } deprecatedSibling })
+					resolved.Add(deprecatedSibling);
+			}
 		}
 
 		return resolved;
@@ -443,9 +448,33 @@ private static ITableOfContentsItem ResolveRuleOverviewReference(IDiagnosticsCol
 			.ToList();
 		var tomlChildren = DetectionRuleOverviewRef.CreateTableOfContentItems(tocSourceFolders, context, baseDirectory);
 
-		var children = resolvedChildren.Concat(tomlChildren).ToList();
+		var children = resolvedChildren.ToList();
+		children.AddRange(tomlChildren);
 
-		return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context);
+		// Auto-detect _deprecated subdirectories. When found, build the deprecated overview FileRef
+		// and attach it as DeprecatedSiblingRef so ResolveTableOfContents can emit it as a sibling,
+		// not as a child nested under the active rules.
+		FileRef? deprecatedSiblingRef = null;
+		var hasDeprecatedRules = tocSourceFolders.Any(d =>
+			d.Exists && d.EnumerateDirectories("_deprecated", SearchOption.TopDirectoryOnly).Any());
+		if (hasDeprecatedRules)
+		{
+			var deprecatedFileName = detectionRuleRef.DeprecatedFile ?? "deprecated-detection-rules.md";
+			var overviewDir = fileSystem.Path.GetDirectoryName(fullPath);
+			var deprecatedFullPath = string.IsNullOrEmpty(overviewDir)
+				? deprecatedFileName
+				: $"{overviewDir}/{deprecatedFileName}";
+			var deprecatedPathRelativeToContainer = string.IsNullOrEmpty(containerPath)
+				? deprecatedFullPath
+				: deprecatedFullPath.Substring(containerPath.Length + 1);
+			var deprecatedTomlChildren = DetectionRuleOverviewRef.CreateDeprecatedTableOfContentItems(tocSourceFolders, context, baseDirectory);
+			deprecatedSiblingRef = new FileRef(deprecatedFullPath, deprecatedPathRelativeToContainer, false, deprecatedTomlChildren, context);
+		}
+
+		return new DetectionRuleOverviewRef(fullPath, pathRelativeToContainer, detectionRuleRef.DetectionRuleFolders, children, context, detectionRuleRef.DeprecatedFile)
+		{
+			DeprecatedSiblingRef = deprecatedSiblingRef
+		};
 	}
 
 

src/Elastic.Documentation.Configuration/Toc/TableOfContentsYamlConverters.cs

@@ -72,7 +72,7 @@ public class TocItemYamlConverter : IYamlTypeConverter
 					}
 					value = childrenList;
 				}
-				else if (key.Value is "detection_rules" or "exclude")
+				else if (key.Value is "detection_rules" or "exclude" or "deprecated_detection_rules")
 				{
 					// Parse the children list manually
 					var childrenList = new List<string>();
@@ -135,11 +135,8 @@ public class TocItemYamlConverter : IYamlTypeConverter
 		if (dictionary.TryGetValue("detection_rules", out var detectionRulesObj) && detectionRulesObj is string[] detectionRulesFolders &&
 			dictionary.TryGetValue("file", out var detectionRulesFilePath) && detectionRulesFilePath is string detectionRulesFile)
 		{
-			// Create the index file reference (FolderIndexFileRef to mark it as the folder's index)
-			// Store ONLY the file name - the folder path will be prepended during resolution
-			// This allows validation to check if the file itself has deep paths
-			// PathRelativeToContainer will be set during resolution
-			return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext);
+			var deprecatedFile = dictionary.TryGetValue("deprecated_file", out var deprecatedFileObj) && deprecatedFileObj is string df ? df : null;
+			return new DetectionRuleOverviewRef(detectionRulesFile, detectionRulesFile, detectionRulesFolders, children, placeholderContext, deprecatedFile);
 		}
 
 		// Check for file reference (file: or hidden:)

src/Elastic.Documentation.Site/Assets/pages-nav.ts

@@ -134,8 +134,16 @@ export function initNav() {
 
     // Normalize pathname by removing trailing slash to handle both URL variants
     const pathname = window.location.pathname.replace(/\/$/, '')
+
+    // When the page is a hidden nav item (e.g. an individual detection rule), the server
+    // emits docs:nav-active pointing to the nearest visible ancestor so we can highlight it.
+    const navActiveMeta = document.querySelector<HTMLMetaElement>(
+        'meta[name="docs:nav-active"]'
+    )
+    const activePathname = navActiveMeta?.content ?? pathname
+
     const navItems = $$(
-        'a[href="' + pathname + '"], a[href="' + pathname + '/"]',
+        'a[href="' + activePathname + '"], a[href="' + activePathname + '/"]',
         pagesNav
     )
     navItems.forEach((el) => {

src/Elastic.Documentation.Site/Layout/_Head.cshtml

@@ -44,6 +44,10 @@
 	@await RenderPartialAsync(_Favicon.Create())
 	<meta name="robots" content="@(Model.AllowIndexing ? "index, follow" : "noindex, nofollow")">
 	<meta name="htmx-config" content='{"selfRequestsOnly": true}'>
+	@if (!string.IsNullOrEmpty(Model.NavigationActiveUrl))
+	{
+		<meta name="docs:nav-active" content="@Model.NavigationActiveUrl">
+	}
 	<meta property="og:type" content="website"/>
 	<meta property="og:title" content="@Model.Title"/>
 	<meta property="og:description" content="@Model.Description"/>

src/Elastic.Documentation.Site/_ViewModels.cs

@@ -44,6 +44,13 @@ public record GlobalLayoutViewModel
 	/// <summary>Breadcrumb trail for codex sub-header (Home / Group / Docset).</summary>
 	public IReadOnlyList<CodexBreadcrumb>? CodexBreadcrumbs { get; init; }
 
+	/// <summary>
+	/// When the current page is a hidden nav item (e.g. an individual detection rule page),
+	/// the URL of its nearest visible ancestor. The client uses this to highlight the correct
+	/// nav entry when the page has no rendered nav link of its own.
+	/// </summary>
+	public string? NavigationActiveUrl { get; init; }
+
 
 	// Header properties for isolated mode
 	public string? HeaderTitle { get; init; }

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs

@@ -63,8 +63,7 @@ public record DetectionRuleTechnique : DetectionRuleSubTechnique
 
 public record DetectionRule
 {
-	// Reuse a single TomlParser instance for better performance
-	private static readonly TomlParser Parser = new();
+	// TomlParser is stateful across calls; create a new instance per parse to avoid accumulation bugs in serve/reload mode
 
 	// Cached version lock data, loaded once per build
 	private static FrozenDictionary<string, VersionLockEntry>? VersionLock;
@@ -103,6 +102,9 @@ public record DetectionRule
 
 	public required DetectionRuleThreat[] Threats { get; init; } = [];
 
+	public required string? DeprecationDate { get; init; }
+	public required string? Maturity { get; init; }
+
 	/// <summary>
 	/// Initializes the version lock cache from the version.lock.json file.
 	/// This should be called once before processing detection rules.
@@ -145,14 +147,14 @@ public static DetectionRule From(IFileInfo source)
 			// Use optimized Utf8StreamReader for better I/O performance
 			using var reader = new Utf8StreamReader(source.FullName, fileOpenMode: FileOpenMode.Throughput);
 			var sourceText = Encoding.UTF8.GetString(reader.ReadToEndAsync().GetAwaiter().GetResult());
-			model = Parser.Parse(sourceText);
+			model = new TomlParser().Parse(sourceText);
 		}
 		catch (Exception e)
 		{
 			throw new Exception($"Could not parse toml in: {source.FullName}", e);
 		}
 
-		if (!model.TryGetValue("metadata", out var node) || node is not TomlTable)
+		if (!model.TryGetValue("metadata", out var node) || node is not TomlTable metadata)
 			throw new Exception($"Could not find metadata section in {source.FullName}");
 
 		if (!model.TryGetValue("rule", out node) || node is not TomlTable rule)
@@ -190,7 +192,9 @@ public static DetectionRule From(IFileInfo source)
 			RunsEvery = TryGetString(rule, "interval"),
 			MaximumAlertsPerExecution = maxSignals,
 			Version = version,
-			Threats = threats
+			Threats = threats,
+			DeprecationDate = TryGetString(metadata, "deprecation_date"),
+			Maturity = TryGetString(metadata, "maturity")
 		};
 	}
 

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs

@@ -11,6 +11,68 @@
 
 namespace Elastic.Markdown.Extensions.DetectionRules;
 
+public record DeprecatedDetectionRuleOverviewFile : MarkdownFile
+{
+	public DeprecatedDetectionRuleOverviewFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext build)
+		: base(sourceFile, rootPath, parser, build)
+	{
+	}
+
+	internal ILeafNavigationItem<IDocumentationFile>[] RuleNavigations { get; set; } = [];
+
+	protected override Task<MarkdownDocument> GetMinimalParseDocumentAsync(Cancel ctx)
+	{
+		var markdown = GetMarkdown();
+		var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null);
+		return Task.FromResult(document);
+	}
+
+	protected override Task<MarkdownDocument> GetParseDocumentAsync(Cancel ctx)
+	{
+		var markdown = GetMarkdown();
+		var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null);
+		return Task.FromResult(document);
+	}
+
+	private string GetMarkdown()
+	{
+		var rules = RuleNavigations.Select(nav => (Navigation: nav, Model: (DetectionRuleFile)nav.Model)).ToList();
+
+		string intro;
+		if (SourceFile.Exists)
+		{
+			var content = SourceFile.FileSystem.File.ReadAllText(SourceFile.FullName);
+			// Extract H1 title from the file if present, use remainder as intro body
+			var lines = content.Split('\n');
+			var titleLine = lines.FirstOrDefault(l => l.TrimStart().StartsWith("# ", StringComparison.Ordinal));
+			if (titleLine != null)
+				Title = titleLine.TrimStart().Substring(2).Trim();
+			intro = content;
+		}
+		else
+		{
+			Title = "Deprecated prebuilt detection rules";
+			intro = "# Deprecated prebuilt detection rules\n\n";
+		}
+
+		var markdown = intro + "\n\n";
+
+		var groupedRules = rules
+			.GroupBy(r => r.Model.Rule.Domain ?? "Unspecified")
+			.OrderBy(g => g.Key)
+			.ToArray();
+
+		foreach (var group in groupedRules)
+		{
+			markdown += $"\n## {group.Key}\n\n";
+			foreach (var (navigation, model) in group.OrderBy(r => r.Model.Rule.Name))
+				markdown += $"[{model.Rule.Name}](!{navigation.Url}) <br>\n";
+		}
+
+		return markdown;
+	}
+}
+
 public record DetectionRuleOverviewFile : MarkdownFile
 {
 	public DetectionRuleOverviewFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext build)
@@ -102,7 +164,8 @@ BuildContext build
 
 	private static IFileInfo SourcePath(IFileInfo rulePath, BuildContext build)
 	{
-		var relative = Path.GetRelativePath(build.DocumentationCheckoutDirectory!.FullName, rulePath.FullName);
+		var checkoutDir = build.DocumentationCheckoutDirectory ?? build.DocumentationSourceDirectory.Parent!;
+		var relative = Path.GetRelativePath(checkoutDir.FullName, rulePath.FullName);
 		var newPath = Path.Join(build.DocumentationSourceDirectory.FullName, relative);
 		var md = Path.ChangeExtension(newPath, ".md");
 		return rulePath.FileSystem.FileInfo.New(md);
@@ -136,12 +199,21 @@ protected override Task<MarkdownDocument> GetParseDocumentAsync(Cancel ctx)
 
 	private string GetMarkdown()
 	{
+		var deprecationNotice = Rule.Maturity == "deprecated"
+			? $"""
+			:::{"{warning}"}
+			This rule has been deprecated{(Rule.DeprecationDate != null ? $" as of {Rule.DeprecationDate}" : "")}.
+			:::
+
+			"""
+			: string.Empty;
+
 		// language=markdown
 		var markdown =
 $"""
 # {Rule.Name}
 
-{Rule.Description}
+{deprecationNotice}{Rule.Description}
 
 **Rule type**: {Rule.Type}<br>
 **Rule indices**: {RenderArray(Rule.Indices)}