Skip to content

Commit e2fa494

Browse files
daniel-nolandclaude
andcommitted
feat(tracectl): print what a panic report needs to be symbolized offline
A panic report from the field had a backtrace with function names but no file or line on any frame, and no way to recover them. The image's binaries are stripped of DWARF; it lives in the build's debug output, indexed by build ID. color-eyre resolves the backtrace in the process, where only the symbol table is left, and it prints no addresses, so the report could not be resolved afterwards either. Each panic report now ends with a SYMBOLIZE section: every frame as an offset into the module it is in (load bias removed, return addresses moved back into their call), and per module its build ID with an llvm-symbolizer command line to run against the debug output. The image does not change. The section is printed whether or not RUST_BACKTRACE is set. A panic in production is rare, and these addresses are the only record of where it happened. backtrace and findshlibs are already in the lockfile (color-eyre and pyroscope pull them in). findshlibs builds C only for Android. Tested end to end with a release build with a build ID, its DWARF split into a .build-id tree and the binary stripped, as default.nix does. The printed command resolved the panicking frame to its source line. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Daniel Noland <daniel@githedgehog.com>
1 parent b401f5d commit e2fa494

6 files changed

Lines changed: 188 additions & 1 deletion

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ async-trait = { version = "0.1.92", default-features = false, features = [] }
146146
atomic-instant-full = { version = "0.1.1", default-features = false, features = [] }
147147
axum = { version = "0.8.9", default-features = false, features = [] }
148148
axum-server = { version = "0.8.0", default-features = false, features = [] }
149+
backtrace = { version = "0.3.76", default-features = false, features = [] }
149150
bindgen = { version = "0.73.2", default-features = false, features = [] }
150151
bitflags = { version = "2.13.2", default-features = false, features = [] }
151152
bnum = { version = "0.14.4", default-features = false, features = [] }
@@ -176,6 +177,7 @@ dotenvy = { version = "0.15.7", default-features = false, features = [] }
176177
downcast-rs = { version = "2.0.2", default-features = false, features = [] }
177178
doxygen-bindgen = { version = "0.1.4", default-features = false, features = [] }
178179
etherparse = { version = "0.21.0", default-features = false, features = [] }
180+
findshlibs = { version = "0.10.2", default-features = false, features = [] }
179181
fixin = { git = "https://github.com/githedgehog/fixin", branch = "main", features = [] }
180182
futures = { version = "0.3.34", default-features = false, features = [] }
181183
futures-util = { version = "0.3.34", default-features = false, features = [] }

‎tracectl/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@ loom = ["concurrency/loom"]
1212
shuttle = ["concurrency/shuttle"]
1313

1414
[dependencies]
15+
backtrace = { workspace = true, features = ["std"] }
1516
clock = { workspace = true }
1617
color-eyre = { workspace = true , features = [ "capture-spantrace", "color-spantrace", "tracing-error", "track-caller" ] }
1718
common = { workspace = true }
1819
concurrency = { workspace = true }
20+
findshlibs = { workspace = true }
1921
linkme = { workspace = true }
2022
ordermap = { workspace = true, features = ["std"] }
2123
thiserror = { workspace = true }

‎tracectl/src/control.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ impl TracingControl {
549549

550550
Self::init_subscriber(filter, throttle);
551551

552-
if let Err(e) = color_eyre::install() {
552+
if let Err(e) = crate::symbolize::install() {
553553
eprintln!("Failed to initialize color_eyre:\n{e}");
554554
}
555555
Self {

‎tracectl/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ pub mod evidence;
1212
pub mod control;
1313
pub mod display;
1414
mod stamp;
15+
mod symbolize;
1516
pub mod targets;
1617
mod throttle;
1718

‎tracectl/src/symbolize.rs‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright Open Network Fabric Authors
3+
4+
//! Panic reports that can be symbolized after the fact.
5+
//!
6+
//! Shipped binaries are stripped of DWARF; it lives in the build's `debug` output, indexed by
7+
//! build ID. The backtrace color-eyre prints is resolved in-process, so it gets function names
8+
//! from the symbol table but no file or line, and it prints no addresses to resolve later.
9+
//!
10+
//! This adds a section to each panic report with what is needed to resolve the trace offline:
11+
//! every frame as an offset into the module it is in, and each module's build ID.
12+
13+
use findshlibs::{Avma, Segment, SharedLibrary, TargetSharedLibrary};
14+
use std::collections::BTreeMap;
15+
use std::fmt::Write;
16+
17+
/// Install color-eyre's hooks, with [`report`] appended to the panic report.
18+
pub(crate) fn install() -> Result<(), color_eyre::Report> {
19+
let (panic_hook, eyre_hook) = color_eyre::config::HookBuilder::default().try_into_hooks()?;
20+
eyre_hook.install()?;
21+
std::panic::set_hook(Box::new(move |info| {
22+
eprintln!("{}", panic_hook.panic_report(info));
23+
eprintln!("{}", report(&frames()));
24+
}));
25+
Ok(())
26+
}
27+
28+
/// The return addresses on the current stack, innermost first.
29+
fn frames() -> Vec<usize> {
30+
let mut ips = Vec::new();
31+
backtrace::trace(|frame| {
32+
ips.push(frame.ip() as usize);
33+
true
34+
});
35+
ips
36+
}
37+
38+
/// A loaded object, as `llvm-symbolizer` needs to know it.
39+
struct Module {
40+
name: String,
41+
build_id: Option<String>,
42+
}
43+
44+
/// Where an address falls: which module, and at which address in that module's own (unrelocated)
45+
/// address space.
46+
struct Located {
47+
module: usize,
48+
svma: usize,
49+
}
50+
51+
fn module_name(lib: &impl SharedLibrary) -> String {
52+
let path = lib.name().to_string_lossy();
53+
if path.is_empty() || path == "/proc/self/exe" {
54+
// glibc's loader reports the main program without a name; musl's static
55+
// `dl_iterate_phdr` reports it as `/proc/self/exe`.
56+
return std::env::current_exe()
57+
.ok()
58+
.and_then(|exe| exe.file_name().map(|n| n.to_string_lossy().into_owned()))
59+
.unwrap_or_else(|| "<main>".to_string());
60+
}
61+
path.rsplit('/').next().unwrap_or(&path).to_string()
62+
}
63+
64+
/// Map each address to the module whose code contains it.
65+
fn locate(ips: &[usize]) -> (Vec<Module>, Vec<Option<Located>>) {
66+
let mut modules = Vec::new();
67+
let mut located: Vec<Option<Located>> = ips.iter().map(|_| None).collect();
68+
TargetSharedLibrary::each(|lib| {
69+
let mut index = None;
70+
for (ip, slot) in ips.iter().zip(located.iter_mut()) {
71+
if slot.is_some() {
72+
continue;
73+
}
74+
// A return address points past its call; step back into the call so the line is
75+
// the call's, not whatever follows it (which may even be another function).
76+
let avma = Avma(ip.saturating_sub(1));
77+
if !lib
78+
.segments()
79+
.any(|seg| seg.is_code() && seg.contains_avma(lib, avma))
80+
{
81+
continue;
82+
}
83+
let module = *index.get_or_insert_with(|| {
84+
modules.push(Module {
85+
name: module_name(lib),
86+
build_id: lib.id().map(|id| id.to_string()),
87+
});
88+
modules.len() - 1
89+
});
90+
*slot = Some(Located {
91+
module,
92+
svma: lib.avma_to_svma(avma).0,
93+
});
94+
}
95+
});
96+
(modules, located)
97+
}
98+
99+
/// The symbolization section of a panic report for the stack `ips`.
100+
fn report(ips: &[usize]) -> String {
101+
let (modules, located) = locate(ips);
102+
let mut out = String::new();
103+
let _ = writeln!(
104+
out,
105+
" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ SYMBOLIZE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
106+
);
107+
for (n, (ip, at)) in ips.iter().zip(&located).enumerate() {
108+
let _ = match at {
109+
Some(at) => writeln!(out, " {n:>3}: {}+{:#x}", modules[at.module].name, at.svma),
110+
None => writeln!(out, " {n:>3}: {ip:#x} (in no loaded module)"),
111+
};
112+
}
113+
let mut by_module: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
114+
for at in located.iter().flatten() {
115+
by_module.entry(at.module).or_default().push(at.svma);
116+
}
117+
let _ = writeln!(
118+
out,
119+
"\n Resolve with the build's debug output (`<debug>` below):"
120+
);
121+
for (module, svmas) in by_module {
122+
let Module { name, build_id } = &modules[module];
123+
let addrs: Vec<String> = svmas.iter().map(|a| format!("{a:#x}")).collect();
124+
let _ = match build_id {
125+
Some(id) => writeln!(
126+
out,
127+
" # {name}\n llvm-symbolizer --debug-file-directory=<debug>/lib/debug --build-id={id} {}",
128+
addrs.join(" ")
129+
),
130+
None => writeln!(
131+
out,
132+
" # {name} (no build ID)\n llvm-symbolizer --obj=<path to {name} with debug info> {}",
133+
addrs.join(" ")
134+
),
135+
};
136+
}
137+
out
138+
}
139+
140+
#[cfg(test)]
141+
mod tests {
142+
use super::{frames, locate, report};
143+
144+
#[test]
145+
fn the_callers_frame_is_found_in_the_test_binary() {
146+
let ips = frames();
147+
let (modules, located) = locate(&ips);
148+
let exe = std::env::current_exe().unwrap_or_else(|e| unreachable!("{e}"));
149+
let exe = exe
150+
.file_name()
151+
.unwrap_or_else(|| unreachable!())
152+
.to_string_lossy();
153+
// Frame 0 is inside `backtrace`; this test's own frame follows it, in the test binary.
154+
let mine = located
155+
.iter()
156+
.flatten()
157+
.find(|at| modules[at.module].name == exe)
158+
.unwrap_or_else(|| unreachable!("no frame in {exe}: {}", report(&ips)));
159+
assert_ne!(mine.svma, 0);
160+
}
161+
162+
#[test]
163+
fn the_report_lists_every_frame_and_a_command_per_module() {
164+
let ips = frames();
165+
let text = report(&ips);
166+
let (modules, _) = locate(&ips);
167+
assert!(text.contains(&format!("{:>3}: ", ips.len() - 1)), "{text}");
168+
assert_eq!(
169+
text.matches("llvm-symbolizer ").count(),
170+
modules.len(),
171+
"{text}"
172+
);
173+
}
174+
175+
#[test]
176+
fn an_address_outside_every_module_is_reported_raw() {
177+
let text = report(&[0x10]);
178+
assert!(text.contains("0x10 (in no loaded module)"), "{text}");
179+
}
180+
}

0 commit comments

Comments
 (0)