Basic attempt at unwinding, with bogus file names

Author: GregMefford

interpreter/beam/beam.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"regexp"
 	"strconv"
+	"unsafe"
 
 	log "github.com/sirupsen/logrus"
 
@@ -27,11 +28,16 @@ import (
 	"go.opentelemetry.io/ebpf-profiler/support"
 )
 
+// #include "../../support/ebpf/types.h"
+// #include "../../support/ebpf/v8_tracer.h"
+import "C"
+
 var (
 	// regex for matching the process name
-	beamRegex                      = regexp.MustCompile(`beam.smp`)
-	_         interpreter.Data     = &beamData{}
-	_         interpreter.Instance = &beamInstance{}
+	beamRegex                        = regexp.MustCompile(`beam.smp`)
+	_           interpreter.Data     = &beamData{}
+	_           interpreter.Instance = &beamInstance{}
+	BogusFileID                      = libpf.NewFileID(0xf00d, 0x1001)
 )
 
 type beamData struct {
@@ -124,6 +130,14 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
 
 func (d *beamData) Attach(ebpf interpreter.EbpfHandler, pid libpf.PID, bias libpf.Address, rm remotememory.RemoteMemory) (interpreter.Instance, error) {
 	log.Infof("BEAM interpreter attaching")
+
+	data := C.BEAMProcInfo{
+		version: C.uint(d.version),
+	}
+	if err := ebpf.UpdateProcData(libpf.BEAM, pid, unsafe.Pointer(&data)); err != nil {
+		return nil, err
+	}
+
 	return &beamInstance{
 		data:     d,
 		rm:       rm,
@@ -255,6 +269,12 @@ func (i *beamInstance) Symbolize(symbolReporter reporter.SymbolReporter, frame *
 		log.Warnf("BEAM failed to symbolize")
 		return interpreter.ErrMismatchInterpreterType
 	}
-	log.Infof("BEAM symbolizing")
+	log.Infof("BEAM symbolizing %v", frame)
+	frameID := libpf.NewFrameID(BogusFileID, frame.Lineno)
+	symbolReporter.FrameMetadata(&reporter.FrameMetadataArgs{
+		FrameID:      frameID,
+		FunctionName: "Some Bogus Name",
+	})
+	trace.AppendFrameID(libpf.BEAMFrame, frameID)
 	return nil
 }

support/ebpf/beam_tracer.ebpf.c

@@ -2,36 +2,95 @@
 #include "tracemgmt.h"
 #include "types.h"
 
+// The number of frames to unwind per frame-unwinding eBPF program.
+#define FRAMES_PER_PROGRAM   8
+
 bpf_map_def SEC("maps") beam_procs = {
   .type = BPF_MAP_TYPE_HASH,
   .key_size = sizeof(pid_t),
   .value_size = sizeof(BEAMProcInfo),
   .max_entries = 1024,
 };
 
+static inline __attribute__((__always_inline__))
+ErrorCode unwind_one_frame(PerCPURecord *record, BEAMProcInfo *info, bool top) {
+  UnwindState *state = &record->state;
+  Trace *trace = &record->trace;
+  unsigned long regs[2], sp = state->sp, fp = state->fp, pc = state->pc;
+
+  DEBUG_PRINT("beam: pc: %lx, sp: %lx, fp: %lx", pc, sp, fp);
+
+  if (fp) {
+    unwinder_mark_nonleaf_frame(state);
+  }
+  _push_with_return_address(trace, 0xf00d, pc, FRAME_MARKER_BEAM, state->return_address);
+
+  // Data that will be sent to HA is in these variables.
+  //uintptr_t pointer_and_type = 0, delta_or_marker = 0;
+
+frame_done:
+  // Unwind with frame pointer
+  if (bpf_probe_read_user(regs, sizeof(regs), (void*)fp)) {
+    DEBUG_PRINT("beam:  --> bad frame pointer");
+    return ERR_UNREACHABLE;
+  }
+
+  state->sp = fp + sizeof(regs);
+  state->fp = regs[0];
+  state->pc = regs[1];
+  if (state->fp) {
+    unwinder_mark_nonleaf_frame(state);
+  }
+
+  DEBUG_PRINT("beam: pc: %lx, sp: %lx, fp: %lx",
+              (unsigned long) state->pc, (unsigned long) state->sp,
+              (unsigned long) state->fp);
+
+  return ERR_OK;
+}
+
 SEC("perf_event/unwind_beam")
 int unwind_beam(struct pt_regs *ctx) {
-  static const char fmt[] = "Unwinding BEAM stack";
-  bpf_trace_printk(fmt, sizeof(fmt));
-  DEBUG_PRINT("Unwinding BEAM stack");
+  DEBUG_PRINT(">>>>>>>>>>>>>>>>>Unwinding BEAM stack<<<<<<<<<<<<<<<<<");
 
   PerCPURecord *record = get_per_cpu_record();
   if (!record) {
     return -1;
   }
 
-  int unwinder = get_next_unwinder_after_interpreter(record);
-  u32 pid = record->trace.pid;
+  Trace *trace = &record->trace;
+  u32 pid = trace->pid;
+  DEBUG_PRINT("==== unwind_beam %d ====", trace->stack_len);
 
-  BEAMProcInfo *beaminfo = bpf_map_lookup_elem(&beam_procs, &pid);
-  if (!beaminfo) {
-    DEBUG_PRINT("No BEAM introspection data");
+  int unwinder = PROG_UNWIND_STOP;
+  ErrorCode error = ERR_OK;
+  BEAMProcInfo *info = bpf_map_lookup_elem(&beam_procs, &pid);
+  if (!info) {
+    DEBUG_PRINT("beam: no BEAMProcInfo for this pid");
     goto exit;
   }
 
-  DEBUG_PRINT("==== unwind_beam stack_len: %d, pid: %d ====", record->trace.stack_len, record->trace.pid);
+#pragma unroll
+  for (int i = 0; i < FRAMES_PER_PROGRAM; i++) {
+    error = unwind_one_frame(record, info, i == 0);
+    if (error) {
+      break;
+    }
+
+    if (record->state.fp == 0) {
+      unwinder = PROG_UNWIND_STOP;
+      break;
+    }
+
+    error = get_next_unwinder_after_native_frame(record, &unwinder);
+    if (error || unwinder != PROG_UNWIND_BEAM) {
+      break;
+    }
+  }
 
 exit:
+  record->state.unwind_error = error;
   tail_call(ctx, unwinder);
+  DEBUG_PRINT("beam: tail call for next frame unwinder (%d) failed", unwinder);
   return -1;
 }

tracer/tracer.go

@@ -483,6 +483,11 @@ func initializeMapsAndPrograms(kernelSymbols *libpf.SymbolMap, cfg *Config) (
 			name:   "unwind_dotnet",
 			enable: cfg.IncludeTracers.Has(types.DotnetTracer),
 		},
+		{
+			progID: uint32(support.ProgUnwindBeam),
+			name:   "unwind_beam",
+			enable: cfg.IncludeTracers.Has(types.BEAMTracer),
+		},
 	}
 
 	if err = loadPerfUnwinders(coll, ebpfProgs, ebpfMaps["perf_progs"], tailCallProgs,