Skip to content

Commit

Permalink
Merge pull request #907 from albertxu216/develop
Browse files Browse the repository at this point in the history 
proc_image：增加新功能来监控futex
  • Loading branch information
@chenamy2017
chenamy2017 authored Sep 23, 2024
2 parents ae48beb + 3dd669b commit 8607cc5
Show file tree
Hide file tree
Showing 6 changed files with 395 additions and 53 deletions.
182 changes: 168 additions & 14 deletions eBPF_Supermarket/CPU_Subsystem/eBPF_proc_image/bpf/mfutex.bpf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,18 @@ struct {
__uint(max_entries,528 * 10240);
} mfutex_rb SEC(".maps");

// struct {
// __uint(type, BPF_MAP_TYPE_HASH);
// __uint(max_entries, 10240);
// __type(key, struct record_lock_key);
// __type(value, struct per_request);
// } futex_wait_queue SEC(".maps");//记录futex陷入内核

struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10240);
__type(key, struct lock_record_key);
__type(value, struct per_request);
} futex_wait_queue SEC(".maps");//记录futex陷入内核
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 10240);
__type(key, struct lock_record_key);
__type(value, struct per_request);
} futex_wake_queue SEC(".maps");//记录futex陷入内核

#define MUTEX_FLAG 1
#define RWLOCK_FLAG 2
Expand Down Expand Up @@ -280,15 +285,164 @@ int BPF_KPROBE(pthread_mutex_unlock_enter, void *__mutex)
// return 0;
// }

// SEC("tracepoint/syscall/sys_enter_futex")
// int trace_sys_enter_futex(struct sys_futex_args ctx){

// /*获取到要加入等待队列的线程，拿到时间*/
// SEC("kprobe/futex_wait")
// int BPF_KPROBE(trace_futex_wait,
// u32 __user *uaddr, unsigned int flags,
// u32 val, ktime_t *abs_time, u32 bitset)
// {
// }

/*1.将线程加入等待队列，并记录*/
SEC("kprobe/futex_wait")
int BPF_KPROBE(trace_futex_wait,
u32 *uaddr, unsigned int flags,
u32 val, ktime_t *abs_time, u32 bitset)
{
struct mfutex_ctrl *mfutex_ctrl = get_mfutex_ctrl();
if(!mfutex_ctrl)
return 0;
pid_t pid = bpf_get_current_pid_tgid();
int tgid = bpf_get_current_pid_tgid() >> 32;

if(mfutex_ctrl->target_pid == -1 && mfutex_ctrl->target_tgid == -1)//未指定目标进程或线程
return 0;
if((mfutex_ctrl->target_pid != -1 && pid!=mfutex_ctrl->target_pid)||
(mfutex_ctrl->target_tgid != -1 && tgid != mfutex_ctrl->target_tgid))//当前进程或线程非目标进程或线程
return 0;

u64 lock_ptr = (u64)uaddr;
int cpu = bpf_get_smp_processor_id();//获取当前cpu

// }
/*1.将 单个线程请求信息块 放入 请求等待队列futex_wait_queue*/
struct per_request per_request = {};
per_request.pid = pid;
per_request.start_request_time = bpf_ktime_get_ns();
per_request.cpu_id = cpu;
struct lock_record_key key = {};
key.lock_ptr = lock_ptr;
key.pid = pid;
bpf_map_update_elem(&futex_wait_queue, &key, &per_request, BPF_ANY);
// bpf_printk("Push_info:pid:%d ,lock_ptr:%lu, cnt:%d\n",per_request.pid,key.lock_ptr,key.cnt);
return 0;
}

/*2.将线程加入唤醒队列，从等待队列中删除
*2.1 将执行futex_wake的线程pid与锁地址进行匹配，便于在后面futex_wake_mark找到锁地址
*/
SEC("kprobe/futex_wake")
int BPF_KPROBE(trace_futex_wake_enter,
u32 *uaddr, unsigned int flags,
int nr_wake, u32 bitset)
{
struct mfutex_ctrl *mfutex_ctrl = get_mfutex_ctrl();
if(!mfutex_ctrl)
return 0;
pid_t pid = bpf_get_current_pid_tgid();
int tgid = bpf_get_current_pid_tgid() >> 32;

if(mfutex_ctrl->target_pid == -1 && mfutex_ctrl->target_tgid == -1)//未指定目标进程或线程
return 0;
if((mfutex_ctrl->target_pid != -1 && pid!=mfutex_ctrl->target_pid)||
(mfutex_ctrl->target_tgid != -1 && tgid != mfutex_ctrl->target_tgid))//当前进程或线程非目标进程或线程
return 0;

u64 lock_ptr = (u64)uaddr;
struct proc_flag key={};
key.pid = pid;
key.flag = FUTEX_FLAG;
bpf_map_update_elem(&proc_unlock, &key, &lock_ptr, BPF_ANY);//将锁地址存在proc_unlock map中
return 0;
}
/*2.将线程加入唤醒队列，从等待队列中删除
*2.2 将要被唤醒的线程加入唤醒队列，并从等待队列中删除掉；
*/
SEC("kprobe/futex_wake_mark")
int BPF_KPROBE(trace_futex_wake_mark, struct wake_q_head *wake_q, struct futex_q *q)
{
struct mfutex_ctrl *mfutex_ctrl = get_mfutex_ctrl();
if(!mfutex_ctrl)
return 0;
pid_t pid = bpf_get_current_pid_tgid();
int tgid = bpf_get_current_pid_tgid() >> 32;

if(mfutex_ctrl->target_pid == -1 && mfutex_ctrl->target_tgid == -1)//未指定目标进程或线程
return 0;
if((mfutex_ctrl->target_pid != -1 && pid!=mfutex_ctrl->target_pid)||
(mfutex_ctrl->target_tgid != -1 && tgid != mfutex_ctrl->target_tgid))//当前进程或线程非目标进程或线程
return 0;

u64 *lock_ptr;
u64 temp_lock_ptr;
u64 ts = bpf_ktime_get_ns();

/*1.找到锁的地址*/
struct proc_flag proc_flag = {};
proc_flag.pid = pid;
proc_flag.flag = FUTEX_FLAG;
lock_ptr = bpf_map_lookup_elem(&proc_unlock, &proc_flag);
if(!lock_ptr) return 0;
temp_lock_ptr = *lock_ptr;

/*2.make key*/
struct lock_record_key key = {};
key.lock_ptr = temp_lock_ptr;
key.pid = BPF_CORE_READ(q,task,pid);

/*3.将线程从等待队列中删除*/
struct per_request *per_request;
per_request = bpf_map_lookup_elem(&futex_wait_queue, &key);
if(per_request) {//如果等待队列中找到该task 则尝试删除
bpf_map_delete_elem(&futex_wait_queue, &key);
per_request->start_hold_time = ts;
per_request->wait_delay = ts - per_request->start_hold_time;
}else{//如果没找到，说明该任务陷入阻塞时未记录，则创建per_request
struct per_request new_per_request = {};
new_per_request.pid = key.pid;
new_per_request.start_hold_time = ts;
per_request = &new_per_request;
}
/*4.将任务放到唤醒队列中*/
bpf_map_update_elem(&futex_wake_queue, &key, per_request, BPF_ANY);
return 0;
}

/*2.将线程加入唤醒队列，从等待队列中删除
*2.1 将执行futex_wake的线程pid与锁地址进行匹配，便于在后面futex_wake_mark找到锁地址
*/
SEC("kretprobe/futex_wake")
int BPF_KRETPROBE(trace_futex_wake_exit)
{
struct mfutex_ctrl *mfutex_ctrl = get_mfutex_ctrl();
if(!mfutex_ctrl)
return 0;
pid_t pid = bpf_get_current_pid_tgid();
int tgid = bpf_get_current_pid_tgid() >> 32;

if(mfutex_ctrl->target_pid == -1 && mfutex_ctrl->target_tgid == -1)//未指定目标进程或线程
return 0;
if((mfutex_ctrl->target_pid != -1 && pid!=mfutex_ctrl->target_pid)||
(mfutex_ctrl->target_tgid != -1 && tgid != mfutex_ctrl->target_tgid))//当前进程或线程非目标进程或线程
return 0;

u64 *lock_ptr;
u64 temp_lock_ptr;
u64 ts = bpf_ktime_get_ns();

/*1.找到锁的地址*/
struct proc_flag proc_flag = {};
proc_flag.pid = pid;
proc_flag.flag = FUTEX_FLAG;
lock_ptr = bpf_map_lookup_elem(&proc_unlock, &proc_flag);
if(!lock_ptr) return 0;
temp_lock_ptr = *lock_ptr;
bpf_map_delete_elem(&proc_unlock, &proc_flag);

/*2.传入rb*/
struct per_lock_event *e;
e = bpf_ringbuf_reserve(&mfutex_rb, sizeof(*e), 0);
if(!e)
return 0;
e->lock_ptr = temp_lock_ptr;
e->start_hold_time = bpf_ktime_get_ns();
e->type = FUTEX_FLAG;
bpf_ringbuf_submit(e, 0);
return 0;
}
21 changes: 16 additions & 5 deletions eBPF_Supermarket/CPU_Subsystem/eBPF_proc_image/include/proc_image.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
#define LAST_ARG (FULL_MAX_ARGS_ARR - ARGSIZE)

#define TASK_RUNNING 0x00000000
// #define MUTEX_FLAG 1
// #define RWLOCK_FLAG 2
// #define SPIN_FLAG 3
// #define RCU_FLAG 4
// #define FUTEX_FLAG 5
#define MUTEX_FLAG 1
#define RWLOCK_FLAG 2
#define SPIN_FLAG 3
#define RCU_FLAG 4
#define FUTEX_FLAG 5
typedef long long unsigned int u64;
typedef unsigned int u32;
#define MAX_STACK_DEPTH 128
Expand Down Expand Up @@ -134,6 +134,7 @@ struct per_request{//每条锁请求信息
u64 start_request_time;
u64 start_hold_time;
u64 wait_delay;
int cpu_id;//在哪个cpu上阻塞了
};

struct lock_record_key{
Expand All @@ -152,6 +153,16 @@ struct per_lock_event{
u64 start_hold_time,last_hold_delay;//持有锁的时间；
int cnt;//等待锁+持有锁的数量；
};
struct sys_futex_args {
u64 pad;
int __syscall_nr;
u32 * uaddr;
int op;
u32 val;
const struct __kernel_timespec * utime;
u32 * uaddr2;
u32 val3;
};

// keytime_image
struct kt_ctrl{
Expand Down
103 changes: 79 additions & 24 deletions eBPF_Supermarket/CPU_Subsystem/eBPF_proc_image/proc_image.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -534,32 +534,86 @@ static int print_mfutex(void *ctx, void *data,unsigned long data_sz)
time_t now = time(NULL);// 获取当前时间
struct tm *localTime = localtime(&now);// 将时间转换为本地时间结构
printf("%02d:%02d:%02d ",localTime->tm_hour, localTime->tm_min, localTime->tm_sec);
printf("%-12s 0x%x %-10d %-16llu %-11llu %-11lu | ",mfutex_type[e->type],e->lock_ptr,e->owner,e->start_hold_time,e->last_owner,e->last_hold_delay);

int err,record_lock_fd =bpf_map__fd(mfutex_skel->maps.record_lock);

struct record_lock_key lookup_key = {-1,-1}, next_key;
while(!bpf_map_get_next_key(record_lock_fd, &lookup_key, &next_key)){
// printf("next_key:%x target_ptr:%x\n",next_key,e->lock_ptr);
if(next_key.lock_ptr != e->lock_ptr) {
lookup_key = next_key;
continue;//不是目标锁，跳过
}
struct per_request per_request = {};
err = bpf_map_lookup_elem(record_lock_fd,&next_key,&per_request);
if (err < 0) {
// printf(stderr, "failed to lookup info: %d\n", err);
lookup_key = next_key;
continue;//不是目标锁，跳过
int type = e->type;
if(type == MUTEX_FLAG){
printf("%-12s 0x%-8x %-11llu %-11lu %-16llu %-10d\n",mfutex_type[e->type],e->lock_ptr,e->last_owner,e->last_hold_delay,e->start_hold_time,e->owner);
printf("%-81s","");
int err,record_lock_fd =bpf_map__fd(mfutex_skel->maps.record_lock);
struct record_lock_key lookup_key1 = {-1,-1}, next_key;
while(!bpf_map_get_next_key(record_lock_fd, &lookup_key1, &next_key)){
// printf("next_key:%x target_ptr:%x\n",next_key,e->lock_ptr);
if(next_key.lock_ptr != e->lock_ptr) {
lookup_key1 = next_key;
continue;//不是目标锁，跳过
}
struct per_request per_request = {};
err = bpf_map_lookup_elem(record_lock_fd,&next_key,&per_request);
if (err < 0) {
// printf(stderr, "failed to lookup info: %d\n", err);
lookup_key1 = next_key;
continue;//不是目标锁，跳过
}
float wait_delay = (e->start_hold_time-per_request.start_request_time)/1000000000.0;
if(wait_delay<=0||wait_delay>=10000)
printf("%d(NULL)|",per_request.pid);
else printf("%d(%ds)|",per_request.pid,((e->start_hold_time-per_request.start_request_time)/1000000000));
lookup_key1 = next_key;
}
}else if(type == FUTEX_FLAG){
printf("%-12s 0x%-8x %-11s %-11s %-16llu ",mfutex_type[e->type],e->lock_ptr,"NULL","NULL",e->start_hold_time);
int err,futex_wake_queue_fd =bpf_map__fd(mfutex_skel->maps.futex_wake_queue);
struct lock_record_key lookup_key1 = {-1,-1}, next_key;
while(!bpf_map_get_next_key(futex_wake_queue_fd, &lookup_key1, &next_key)){
// printf("next_key:%x target_ptr:%x\n",next_key,e->lock_ptr);
if(next_key.lock_ptr != e->lock_ptr) {
lookup_key1 = next_key;
continue;//不是目标锁，跳过
}
struct per_request per_request = {};
err = bpf_map_lookup_elem(futex_wake_queue_fd,&next_key,&per_request);
if (err < 0) {
// printf(stderr, "failed to lookup info: %d\n", err);
lookup_key1 = next_key;
continue;//不是目标锁，跳过
}

float wait_delay = (e->start_hold_time-per_request.start_request_time)/1000000000.0;
if(wait_delay<=0||wait_delay>=10000)
printf("%d(NULL)| ",per_request.pid);
else printf("%d(%ds)| ",per_request.pid,((e->start_hold_time-per_request.start_request_time)/1000000000));
err = bpf_map_delete_elem(futex_wake_queue_fd, &next_key);
if (err < 0) {
fprintf(stderr, "failed to cleanup infos: %d\n", err);
return -1;
}
lookup_key1 = next_key;
}
printf("\n%-81s","");
int err2,futex_wait_queue_fd =bpf_map__fd(mfutex_skel->maps.futex_wait_queue);
struct lock_record_key lookup_key2 = {-1,-1};
while(!bpf_map_get_next_key(futex_wait_queue_fd, &lookup_key2, &next_key)){
// printf("next_key:%x target_ptr:%x\n",next_key,e->lock_ptr);
if(next_key.lock_ptr != e->lock_ptr) {
lookup_key2 = next_key;
continue;//不是目标锁，跳过
}
struct per_request per_request = {};
err2 = bpf_map_lookup_elem(futex_wait_queue_fd,&next_key,&per_request);
if (err2 < 0) {
// printf(stderr2, "failed to lookup info: %d\n", err2);
lookup_key2 = next_key;
continue;//不是目标锁，跳过
}

float wait_delay = (e->start_hold_time-per_request.start_request_time)/1000000000.0;
if(wait_delay<=0||wait_delay>=10000)
printf("%d(NULL)| ",per_request.pid);
else printf("%d(%ds)| ",per_request.pid,((e->start_hold_time-per_request.start_request_time)/1000000000));
lookup_key2 = next_key;
}
float wait_delay = (e->start_hold_time-per_request.start_request_time)/1000000000.0;
if(wait_delay<=0||wait_delay>=10000)
printf("%d(NULL) | ",per_request.pid);
else printf("%d(%ds) | ",per_request.pid,((e->start_hold_time-per_request.start_request_time)/1000000000));
lookup_key = next_key;
printf("\n");
}
printf("\n");

// struct record_lock_key key;
// key.lock_ptr = e->lock_ptr;
// key.cnt = 1;
Expand Down Expand Up @@ -1020,7 +1074,8 @@ int main(int argc, char **argv)
fprintf(stderr, "Failed to create mfutex ring buffer\n");
goto cleanup;
}
printf("%-8s %-12s %-10s %-10s %-16s %-11s %-11s | %10s\n","TIME","LOCK_TYPE","LOCK_Addr","Holder","HoldTime","Last_Holder","Last_Delay","Wait_Proc");
printf("============================================ MFutex ============================================\n");
printf("%-8s %-12s %-10s %-11s %-11s %-16s %-15s\n","TIME","LOCK_TYPE","LOCK_Addr","Last_Holder","Last_Delay","HoldTime","Waker/Waiter");
}

if(env.enable_keytime){
Expand Down
Loading

0 comments on commit 8607cc5

Please sign in to comment.