Skip to content

Commit

Permalink
kvm_watcher:优化代码,更新目录结构 (linuxkerneltravel#790)
Browse files Browse the repository at this point in the history
* 修改makefile

* 更新目录结构

* 添加用户态帮助函数

* 更新目录结构,添加用户态帮助函数

* update

* 更新目录结构,添加用户态帮助函数

* update
  • Loading branch information
nanshuaibo authored May 10, 2024
1 parent 8ba3acf commit 439159c
Show file tree
Hide file tree
Showing 16 changed files with 1,693 additions and 70 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,4 @@ eBPF_Supermarket/CPU_Subsystem/eBPF_proc_image/controller
# Stack_Analyser
eBPF_Supermarket/Stack_Analyser/stack_analyzer
eBPF_Supermarket/Stack_Analyser/exporter/exporter
eBPF_Supermarket/Stack_Analyser/bpf_skel
eBPF_Supermarket/Stack_Analyser/bpf_skel
71 changes: 44 additions & 27 deletions eBPF_Supermarket/kvm_watcher/Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
.SILENT:
ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
| sed 's/arm.*/arm/' \
| sed 's/aarch64/arm64/' \
Expand All @@ -18,19 +17,12 @@ BPF_CFLAGS=-g -O2 -target bpf
LIBS=-lbpf -lelf -lz -lzstd

# 头文件目录
INCLUDE_DIRS=-I/usr/include/x86_64-linux-gnu -I.
INCLUDE_DIRS=-I/usr/include/x86_64-linux-gnu -I. -I./include -I./include/bpf -I./include/helpers

# qemu 命令行参数变量化
QEMU_CMD=sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -smp 4 -drive file=cirros-0.5.2-x86_64-disk.img,format=qcow2 -boot c -nographic

CIRROS_IMG_URL=https://gitee.com/nan-shuaibo/cirros/releases/download/0.5.2/cirros-0.5.2-x86_64-disk.img
CIRROS_IMG_FILE=cirros-0.5.2-x86_64-disk.img

# 定义检查虚拟化支持的命令
CHECK_VIRT_SUPPORT = [ $$(grep -Eoc '(vmx|svm)' /proc/cpuinfo) -eq 0 ]

# 定义检查 qemu-system-x86_64 进程是否存在的命令
CHECK_QEMU_RUNNING = [ "$$(pgrep -f qemu-system-x86_64)" ]
# 帮助函数
HELPERS_OBJ_DIR := src/helpers
HELPERS_FILES := $(wildcard $(HELPERS_OBJ_DIR)/*.c)
HELPERS_OBJ_FILES := $(patsubst $(HELPERS_OBJ_DIR)/%.c,$(HELPERS_OBJ_DIR)/%.o,$(HELPERS_FILES))

# 默认目标
.PHONY: default
Expand All @@ -39,28 +31,53 @@ default: bpf
# 安装必要的依赖
.PHONY: deps
deps:
sudo apt-get update
sudo apt-get update && \
sudo apt-get install -y clang libelf1 libelf-dev zlib1g-dev libbpf-dev \
linux-tools-$$(uname -r) linux-cloud-tools-$$(uname -r) \
libpcap-dev gcc-multilib build-essential
sudo apt-get install -y lolcat qemu-kvm wget
libpcap-dev gcc-multilib build-essential lolcat qemu-kvm wget

# 生成 vmlinux.h
.PHONY: vmlinux
vmlinux:
bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h

# 编译helpers目录下的所有.c文件
$(HELPERS_OBJ_DIR)/%.o: $(HELPERS_OBJ_DIR)/%.c
clang $(CFLAGS) $(INCLUDE_DIRS) -c $< -o $@

# 编译BPF程序
$(APP).bpf.o: $(APP).bpf.c vmlinux
clang $(BPF_CFLAGS) -D__TARGET_ARCH_$(ARCH) $(INCLUDE_DIRS) -c $< -o $@

# 生成BPF骨架文件
$(APP).skel.h: $(APP).bpf.o
bpftool gen skeleton $< > $@

# 编译用户空间应用程序
${APP}.o: ${APP}.c
clang $(CFLAGS) $(INCLUDE_DIRS) -c $< -o $@

# 链接用户空间应用程序与库
$(notdir $(APP)): ${APP}.o $(HELPERS_OBJ_FILES)
clang -Wall $(CFLAGS) ${APP}.o $(HELPERS_OBJ_FILES) $(LIBS) -o $@
@echo "BPF program compiled successfully."

# bpf 目标
.PHONY: bpf
bpf: vmlinux
# 编译BPF程序
clang $(BPF_CFLAGS) -D__TARGET_ARCH_$(ARCH) $(INCLUDE_DIRS) -c $(APP).bpf.c -o $(APP).bpf.o
# 生成BPF骨架文件
bpftool gen skeleton ${APP}.bpf.o > $(APP).skel.h
# 编译用户空间应用程序
clang $(CFLAGS) $(INCLUDE_DIRS) -c $(APP).c -o ${APP}.o
# 将用户空间应用程序与库链接
clang -Wall $(CFLAGS) ${APP}.o $(LIBS) -o $(notdir $(APP))
echo "BPF program compiled successfully."
bpf: $(APP).skel.h $(APP).bpf.o ${APP}.o $(HELPERS_OBJ_FILES) $(notdir $(APP))


# qemu 命令行参数变量化
QEMU_CMD=sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -smp 4 -drive file=cirros-0.5.2-x86_64-disk.img,format=qcow2 -boot c -nographic

CIRROS_IMG_URL=https://gitee.com/nan-shuaibo/cirros/releases/download/0.5.2/cirros-0.5.2-x86_64-disk.img
CIRROS_IMG_FILE=cirros-0.5.2-x86_64-disk.img

# 定义检查虚拟化支持的命令
CHECK_VIRT_SUPPORT = [ $$(grep -Eoc '(vmx|svm)' /proc/cpuinfo) -eq 0 ]

# 定义检查 qemu-system-x86_64 进程是否存在的命令
CHECK_QEMU_RUNNING = [ "$$(pgrep -f qemu-system-x86_64)" ]

.PHONY: test
test: bpf
Expand Down Expand Up @@ -96,7 +113,7 @@ test: bpf


clean:
cd src && rm -f *.o *.skel.h *.bpf.o
rm -f src/*.o src/*.skel.h src/helpers/*.o
sudo rm -rf $(notdir $(APP)) include/vmlinux.h temp


33 changes: 20 additions & 13 deletions eBPF_Supermarket/kvm_watcher/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,26 +118,33 @@ Report bugs to <[email protected]>.
## 四、代码结构

```
├── docs //功能模块说明文档
├── docs //功能模块说明文档
│ ├── kvm_exit.md
│ ├── kvm_hypercall.md
│ ├── kvm_irq.md
│ ├── kvm_mmu.md
│ └── kvm_vcpu.md
├── include //内核态bpf程序
│ ├── kvm_exits.h
│ ├── kvm_hypercall.h
│ ├── kvm_ioctl.h
│ ├── kvm_irq.h
│ ├── kvm_mmu.h
│ ├── kvm_vcpu.h
│ └── kvm_watcher.h //公共头文件
├── include
│ ├── bpf //内核态bpf程序
│ │ ├── kvm_exits.h
│ │ ├── kvm_hypercall.h
│ │ ├── kvm_ioctl.h
│ │ ├── kvm_irq.h
│ │ ├── kvm_mmu.h
│ │ └── kvm_vcpu.h
│ ├── common.h //内核态和用户态公共头文件
│ └── helpers //用户态帮助函数
│ ├── trace_helpers.h
│ └── uprobe_helpers.h
├── Makefile //编译脚本
├── README.md
├── src
│ ├── kvm_watcher.bpf.c //内核态bpf程序入口
│ └── kvm_watcher.c //用户态bpf程序
└── temp //临时文件目录
├── src
│ ├── helpers //用户态帮助函数
│ │ ├── trace_helpers.c
│ │ └── uprobe_helpers.c
│ ├── kvm_watcher.bpf.c //内核态bpf程序入口
│ └── kvm_watcher.c //用户态bpf程序
└── temp //临时文件目录
```

## 五、测试
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef __KVM_EXITS_H
#define __KVM_EXITS_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef __KVM_HYPERCALL_H
#define __KVM_HYPERCALL_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef __KVM_IOCTL_H
#define __KVM_IOCTL_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#ifndef __KVM_IRQ_H
#define __KVM_IRQ_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef __KVM_MMU_H
#define __KVM_MMU_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down Expand Up @@ -60,16 +60,18 @@ static int trace_tdp_page_fault(struct kvm_vcpu *vcpu,
struct common_event *e) {
u64 addr;
bpf_probe_read_kernel(&addr, sizeof(u64), &fault->addr);
u64 *ts;
ts = bpf_map_lookup_elem(&pf_delay, &addr);
if (!ts) {
return 0;
}
u32 *count;
u32 new_count = 1;
u32 error_code;
u64 hva, pfn;
bpf_probe_read_kernel(&error_code, sizeof(u32), &fault->error_code);
u64 *ts;
ts = bpf_map_lookup_elem(&pf_delay, &addr);
if (!ts) {
int a = *ts;
bpf_printk("trace_tdp_page_fault:ts = %d", a);
return 0;
}
bpf_probe_read_kernel(&hva, sizeof(u64), &fault->hva);
bpf_probe_read_kernel(&pfn, sizeof(u64), &fault->pfn);
short memslot_id = BPF_CORE_READ(fault, slot, id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef __KVM_VCPU_H
#define __KVM_VCPU_H

#include "kvm_watcher.h"
#include "common.h"
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ static const char binary_path[] = "/bin/qemu-system-x86_64";
// IOCTL
#include <asm-generic/ioctl.h>
#define KVMIO 0xAE
#define KVM_CREATE_VM _IO(KVMIO, 0x01)
#define KVM_CREATE_VM _IO(KVMIO, 0x01)
#define KVM_CREATE_VCPU _IO(KVMIO, 0x41)
#define KVM_GET_VCPU_EVENTS _IOR(KVMIO, 0x9f, struct kvm_vcpu_events)
#define KVM_SET_VCPU_EVENTS _IOW(KVMIO, 0xa0, struct kvm_vcpu_events)
Expand Down
105 changes: 105 additions & 0 deletions eBPF_Supermarket/kvm_watcher/include/helpers/trace_helpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
#ifndef __TRACE_HELPERS_H
#define __TRACE_HELPERS_H

#include <stdbool.h>

#define NSEC_PER_SEC 1000000000ULL

struct ksym {
const char *name;
unsigned long addr;
};

struct ksyms;

struct ksyms *ksyms__load(void);
void ksyms__free(struct ksyms *ksyms);
const struct ksym *ksyms__map_addr(const struct ksyms *ksyms,
unsigned long addr);
const struct ksym *ksyms__get_symbol(const struct ksyms *ksyms,
const char *name);

struct sym {
const char *name;
unsigned long start;
unsigned long size;
unsigned long offset;
};

struct syms;

struct syms *syms__load_pid(int tgid);
struct syms *syms__load_file(const char *fname);
void syms__free(struct syms *syms);
const struct sym *syms__map_addr(const struct syms *syms, unsigned long addr);
const struct sym *syms__map_addr_dso(const struct syms *syms,
unsigned long addr, char **dso_name,
unsigned long *dso_offset);

struct syms_cache;

struct syms_cache *syms_cache__new(int nr);
struct syms *syms_cache__get_syms(struct syms_cache *syms_cache, int tgid);
void syms_cache__free(struct syms_cache *syms_cache);

struct partition {
char *name;
unsigned int dev;
};

struct partitions;

struct partitions *partitions__load(void);
void partitions__free(struct partitions *partitions);
const struct partition *partitions__get_by_dev(
const struct partitions *partitions, unsigned int dev);
const struct partition *partitions__get_by_name(
const struct partitions *partitions, const char *name);

void print_log2_hist(unsigned int *vals, int vals_size, const char *val_type);
void print_linear_hist(unsigned int *vals, int vals_size, unsigned int base,
unsigned int step, const char *val_type);

unsigned long long get_ktime_ns(void);

bool is_kernel_module(const char *name);

/*
* When attempting to use kprobe/kretprobe, please check out new fentry/fexit
* probes, as they provide better performance and usability. But in some
* situations we have to fallback to kprobe/kretprobe probes. This helper
* is used to detect fentry/fexit support for the specified kernel function.
*
* 1. A gap between kernel versions, kernel BTF is exposed
* starting from 5.4 kernel. but fentry/fexit is actually
* supported starting from 5.5.
* 2. Whether kernel supports module BTF or not
*
* *name* is the name of a kernel function to be attached to, which can be
* from vmlinux or a kernel module.
* *mod* is a hint that indicates the *name* may reside in module BTF,
* if NULL, it means *name* belongs to vmlinux.
*/
bool fentry_can_attach(const char *name, const char *mod);

/*
* The name of a kernel function to be attached to may be changed between
* kernel releases. This helper is used to confirm whether the target kernel
* uses a certain function name before attaching.
*
* It is achieved by scaning
* /sys/kernel/debug/tracing/available_filter_functions
* If this file does not exist, it fallbacks to parse /proc/kallsyms,
* which is slower.
*/
bool kprobe_exists(const char *name);
bool tracepoint_exists(const char *category, const char *event);

bool vmlinux_btf_exists(void);
bool module_btf_exists(const char *mod);

bool probe_tp_btf(const char *name);
bool probe_ringbuf();

#endif /* __TRACE_HELPERS_H */
19 changes: 19 additions & 0 deletions eBPF_Supermarket/kvm_watcher/include/helpers/uprobe_helpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
/* Copyright (c) 2021 Google LLC. */
#ifndef __UPROBE_HELPERS_H
#define __UPROBE_HELPERS_H

#include <sys/types.h>
#include <unistd.h>
#include <gelf.h>

int get_pid_binary_path(pid_t pid, char *path, size_t path_sz);
int get_pid_lib_path(pid_t pid, const char *lib, char *path, size_t path_sz);
int resolve_binary_path(const char *binary, pid_t pid, char *path,
size_t path_sz);
off_t get_elf_func_offset(const char *path, const char *func);
Elf *open_elf(const char *path, int *fd_close);
Elf *open_elf_by_fd(int fd);
void close_elf(Elf *e, int fd_close);

#endif /* __UPROBE_HELPERS_H */
Loading

0 comments on commit 439159c

Please sign in to comment.