From f76e46cf93c1ab0aac3748aea5c6dbeadeb712af Mon Sep 17 00:00:00 2001 From: the Date: Sat, 25 Nov 2023 16:02:47 +0800 Subject: [PATCH] abstract rte_ring api --- .gitignore | 1 + src/cmds.c | 6 +-- src/dp.c | 16 ++++---- src/grpc/Makefile | 38 +++++++++++++++++++ src/grpc/vrg_grcp_server.cpp | 17 +++++++++ src/grpc/vrg_grcp_server.h | 14 +++++++ src/uilts.c | 18 +-------- src/utils.h | 72 ++++++++++++++++++++++++++++++++++-- src/vrg.c | 11 +++++- src/vrg.h | 17 ++++----- 10 files changed, 167 insertions(+), 43 deletions(-) create mode 100644 src/grpc/Makefile create mode 100644 src/grpc/vrg_grcp_server.cpp create mode 100644 src/grpc/vrg_grcp_server.h diff --git a/.gitignore b/.gitignore index 98f24df..70d00f8 100755 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ build **/*.o **/vrg +**/vrg_northbound **/.libs unit_test/unit-tester \ No newline at end of file diff --git a/src/cmds.c b/src/cmds.c index 9fafba1..0a6f7a3 100644 --- a/src/cmds.c +++ b/src/cmds.c @@ -188,7 +188,7 @@ static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result, mail->type = IPC_EV_TYPE_CLI; mail->len = 1; //enqueue cli quit event to main thread - rte_ring_enqueue_burst(rte_ring,(void **)&mail,1,NULL); + vrg_ring_enqueue(rte_ring, (void **)&mail, 1); } cmdline_parse_token_string_t cmd_quit_quit = @@ -280,7 +280,7 @@ static void cmd_connect_parsed( void *parsed_result, mail->type = IPC_EV_TYPE_CLI; mail->len = sizeof(cli_to_main_msg_t); //enqueue cli quit event to main thread - rte_ring_enqueue_burst(rte_ring,(void **)&mail,1,NULL); + vrg_ring_enqueue(rte_ring, (void **)&mail, 1); } cmdline_parse_token_string_t cmd_connect_connect = @@ -349,7 +349,7 @@ static void cmd_dhcp_parsed( void *parsed_result, mail->type = IPC_EV_TYPE_CLI; mail->len = sizeof(cli_to_main_msg_t); //enqueue cli quit event to main thread - rte_ring_enqueue_burst(rte_ring,(void **)&mail,1,NULL); + vrg_ring_enqueue(rte_ring, (void **)&mail, 1); } cmdline_parse_token_string_t cmd_dhcp_dhcp = diff --git a/src/dp.c b/src/dp.c index e40c319..42f5018 100644 --- a/src/dp.c +++ b/src/dp.c @@ -204,7 +204,7 @@ int wan_recvd(void *arg) (mail + cp_recv_prod)->len = single_pkt->data_len; //enqueue eth_hdr single_pkt->data_len cur = (char *)(mail + cp_recv_prod); - rte_ring_enqueue_burst(rte_ring,(void **)&cur,1,NULL); + vrg_ring_enqueue(rte_ring, (void **)&cur, 1); cp_recv_prod++; if (cp_recv_prod >= 32) cp_recv_prod = 0; @@ -259,7 +259,7 @@ int wan_recvd(void *arg) break; case PROTO_TYPE_UDP: case PROTO_TYPE_TCP: - rte_ring_enqueue_burst(downlink_q,(void **)&single_pkt,1,NULL); + vrg_ring_enqueue(downlink_q, (void **)&single_pkt, 1); break; default: rte_pktmbuf_free(single_pkt); @@ -480,7 +480,7 @@ int lan_recvd(void *arg) } if (unlikely(vlan_header->next_proto == rte_cpu_to_be_16(FRAME_TYPE_ARP))) { /* We only reply arp request to us */ - rte_ring_enqueue_burst(gateway_q, (void **)&single_pkt, 1, NULL); + vrg_ring_enqueue(gateway_q, (void **)&single_pkt, 1); continue; } else if (unlikely(vlan_header->next_proto == rte_cpu_to_be_16(ETH_P_PPP_DIS) || (vlan_header->next_proto == rte_cpu_to_be_16(ETH_P_PPP_SES)))) { @@ -494,7 +494,7 @@ int lan_recvd(void *arg) else if (likely(vlan_header->next_proto == rte_cpu_to_be_16(FRAME_TYPE_IP))) { ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(single_pkt, unsigned char *) + sizeof(struct rte_ether_hdr) + sizeof(vlan_header_t)); if (unlikely(((ip_hdr->dst_addr << 8) ^ (vrg_ccb->lan_ip << 8)) == 0)) { - rte_ring_enqueue_burst(gateway_q, (void **)&single_pkt, 1, NULL); + vrg_ring_enqueue(gateway_q, (void **)&single_pkt, 1); continue; } single_pkt->l2_len = sizeof(struct rte_ether_hdr) + sizeof(vlan_header_t) + sizeof(pppoe_header_t) + sizeof(ppp_payload_t); @@ -568,7 +568,7 @@ int lan_recvd(void *arg) rte_pktmbuf_free(single_pkt); continue; } - rte_ring_enqueue_burst(uplink_q, (void **)&single_pkt, 1, NULL); + vrg_ring_enqueue(uplink_q, (void **)&single_pkt, 1); } else if (ip_hdr->next_proto_id == PROTO_TYPE_UDP) { if (unlikely(RTE_IS_IPV4_MCAST(rte_be_to_cpu_32(ip_hdr->dst_addr)))) { @@ -577,7 +577,7 @@ int lan_recvd(void *arg) } struct rte_udp_hdr *udp_hdr = (struct rte_udp_hdr *)(ip_hdr + 1); if (unlikely(udp_hdr->dst_port == rte_be_to_cpu_16(67))) { - rte_ring_enqueue_burst(gateway_q, (void **)&single_pkt, 1, NULL); + vrg_ring_enqueue(gateway_q, (void **)&single_pkt, 1); continue; } if (unlikely(!rte_is_same_ether_addr(ð_hdr->dst_addr, &vrg_ccb->nic_info.hsi_lan_mac))) { @@ -590,7 +590,7 @@ int lan_recvd(void *arg) rte_pktmbuf_free(single_pkt); continue; } - rte_ring_enqueue_burst(uplink_q, (void **)&single_pkt, 1, NULL); + vrg_ring_enqueue(uplink_q, (void **)&single_pkt, 1); } else { VRG_LOG(DBG, vrg_ccb->fp, NULL, NULL, "unknown L4 packet with protocol id %x recv on LAN port queue", ip_hdr->next_proto_id); @@ -782,7 +782,7 @@ static int lsi_event_callback(U16 port_id, enum rte_eth_event_type type, void *p mail->type = IPC_EV_TYPE_REG; mail->len = 1; //enqueue down event to main thread - rte_ring_enqueue_burst(rte_ring,(void **)&mail,1,NULL); + vrg_ring_enqueue(rte_ring, (void **)&mail, 1); return 0; } diff --git a/src/grpc/Makefile b/src/grpc/Makefile new file mode 100644 index 0000000..6667756 --- /dev/null +++ b/src/grpc/Makefile @@ -0,0 +1,38 @@ +############################################################ +# vRG northbound makefile +############################################################ + +###################################### +# Set variable +###################################### +CC = g++ +INCLUDE = +CFLAGS = $(INCLUDE) -Wall -g -O3 + +LDFLAGS = -lutils + +TARGET = vrg_northbound +SRC = $(wildcard *.cpp) +OBJ = $(SRC:.cpp=.o) + +.PHONY: $(TARGET) +all: $(TARGET) +###################################### +# Compile & Link +# Must use \tab key after new line +###################################### +$(TARGET): $(OBJ) + $(CC) $(CFLAGS) $(OBJ) -o $(TARGET) $(LDFLAGS) + +install: + cp $(TARGET) /usr/local/bin/$(TARGET) + +###################################### +# Clean +###################################### +clean: + rm -rf $(OBJ) $(TARGET) .libs + $(MAKE) -C $(TESTDIR) -f Makefile $@ + +uninstall: + rm -f /usr/local/bin/$(TARGET) diff --git a/src/grpc/vrg_grcp_server.cpp b/src/grpc/vrg_grcp_server.cpp new file mode 100644 index 0000000..aa2174b --- /dev/null +++ b/src/grpc/vrg_grcp_server.cpp @@ -0,0 +1,17 @@ +#include +#include "../vrg.h" + +VRG_t *vrg_ccb; + +void vrg_grpc_server_run() { + std::string server_address("0.0.0.0:50051"); + + std::cout << "Server listening on " << server_address << std::endl; + for(;;); //place holder for grpc server +} + +int main(int argc, char **argv) { + vrg_grpc_server_run(); + + return 0; +} \ No newline at end of file diff --git a/src/grpc/vrg_grcp_server.h b/src/grpc/vrg_grcp_server.h new file mode 100644 index 0000000..187ad53 --- /dev/null +++ b/src/grpc/vrg_grcp_server.h @@ -0,0 +1,14 @@ +#ifndef VRG_GRPC_SERVER_H +#define VRG_GRPC_SERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +int vrg_grpc_server_run(void *arg); + +#ifdef __cplusplus +} +#endif + +#endif // VRG_GRPC_SERVER_H \ No newline at end of file diff --git a/src/uilts.c b/src/uilts.c index 78930ff..f204969 100644 --- a/src/uilts.c +++ b/src/uilts.c @@ -1,11 +1,7 @@ #include #include -#include #include "utils.h" -#define BURST_SIZE 32 -extern struct rte_ring *rte_ring; - void get_all_lcore_id(struct lcore_map *lcore) { lcore->ctrl_thread = rte_get_next_lcore(rte_lcore_id(), 1, 0); @@ -15,17 +11,5 @@ void get_all_lcore_id(struct lcore_map *lcore) lcore->up_thread = rte_get_next_lcore(lcore->lan_thread, 1, 0); lcore->gateway_thread = rte_get_next_lcore(lcore->up_thread, 1, 0); lcore->timer_thread = rte_get_next_lcore(lcore->gateway_thread, 1, 0); -} - -int control_plane_dequeue(void **mail) -{ - U16 burst_size; - - for(;;) { - burst_size = rte_ring_dequeue_burst(rte_ring, mail, BURST_SIZE, NULL); - if (likely(burst_size == 0)) - continue; - break; - } - return burst_size; + lcore->northbound_thread = rte_get_next_lcore(lcore->timer_thread, 1, 0); } diff --git a/src/utils.h b/src/utils.h index 1956184..3c5d461 100644 --- a/src/utils.h +++ b/src/utils.h @@ -2,8 +2,20 @@ #define _UTILS_H_ #include +#include #include -#include "dbg.h" +#include "protocol.h" + +#define RING_BURST_SIZE 32 + +/** + * @brief msg between IF driver and daemon + */ +typedef struct { + U16 type; + U8 refp[ETH_JUMBO]; + int len; +} tVRG_MBX; /* only execution when condition is true */ #define VRG_ASSERT(cond, op, ret) do { \ @@ -14,7 +26,6 @@ static inline void *_vrg_malloc(size_t size, unsigned int aligned) { if (unlikely(size == 0)) { - VRG_LOG(ERR, NULL, NULL, NULL, "malloc size is 0"); return NULL; } return rte_malloc(NULL, size, aligned); @@ -22,7 +33,6 @@ static inline void *_vrg_malloc(size_t size, unsigned int aligned) { static inline void vrg_mfree(void *ptr) { if (unlikely(ptr == NULL)) { - VRG_LOG(ERR, NULL, NULL, NULL, "free ptr is NULL"); return; } rte_free(ptr); @@ -30,6 +40,60 @@ static inline void vrg_mfree(void *ptr) { #define vrg_malloc(type, size, aligned) (type *)_vrg_malloc(size, aligned) +/** + * vrg_ring_enqueue + * + * @brief + * vrg lockless ring enqueue, it will try to enqueue all mails + * @param ring + * ring pointer + * @param mails + * mail array + * @param enqueue_num + * mail amount + * @return + * void + */ +static inline void vrg_ring_enqueue(struct rte_ring *ring, void **mails, unsigned int enqueue_num) +{ + unsigned int burst_size = 0; + unsigned int rest_num = enqueue_num; + + for(;;) { + int rest_mails_index = enqueue_num - rest_num; + burst_size = rte_ring_enqueue_burst(ring, &mails[rest_mails_index], rest_num, NULL); + rest_num -= burst_size; + if (likely(rest_num == 0)) + break; + } + return; +} + +/** + * vrg_ring_dequeue + * + * @brief + * vrg lockless ring dequeue, it will return once there is a mail + * @param ring + * ring pointer + * @param mails + * mail array + * @return + * mail amount + */ +static inline int vrg_ring_dequeue(struct rte_ring *ring, void **mail) +{ + U16 burst_size; + + for(;;) { + burst_size = rte_ring_dequeue_burst(ring, mail, RING_BURST_SIZE, NULL); + if (likely(burst_size == 0)) + continue; + break; + } + return burst_size; +} + struct lcore_map { U8 ctrl_thread; U8 wan_thread; @@ -38,9 +102,9 @@ struct lcore_map { U8 up_thread; U8 gateway_thread; U8 timer_thread; + U8 northbound_thread; }; void get_all_lcore_id(struct lcore_map *lcore); -int control_plane_dequeue(void **mail); #endif \ No newline at end of file diff --git a/src/vrg.c b/src/vrg.c index c52292d..002a2ef 100644 --- a/src/vrg.c +++ b/src/vrg.c @@ -47,12 +47,12 @@ void link_disconnnect(struct rte_timer *tim, VRG_t *vrg_ccb) ***************************************************************/ int vrg_loop(VRG_t *vrg_ccb) { - tVRG_MBX *mail[BURST_SIZE]; + tVRG_MBX *mail[RING_BURST_SIZE]; U16 burst_size; U16 recv_type; for(;;) { - burst_size = control_plane_dequeue((void **)mail); + burst_size = vrg_ring_dequeue(rte_ring, (void **)mail); /* update the ring queue index between hsi_recvd() */ rte_atomic16_add(&cp_recv_cums,burst_size); if (rte_atomic16_read(&cp_recv_cums) > 32) @@ -168,6 +168,12 @@ int control_plane(VRG_t *vrg_ccb) return 0; } +int northbound(VRG_t *vrg_ccb) +{ + while(1); // place holder + return 0; +} + int vrg_start(int argc, char **argv) { int ret = rte_eal_init(argc, argv); @@ -260,6 +266,7 @@ int vrg_start(int argc, char **argv) rte_eal_remote_launch((lcore_function_t *)uplink, (void *)&vrg_ccb, vrg_ccb.lcore.up_thread); rte_eal_remote_launch((lcore_function_t *)gateway, (void *)&vrg_ccb, vrg_ccb.lcore.gateway_thread); rte_eal_remote_launch((lcore_function_t *)timer_loop, (void *)&vrg_ccb, vrg_ccb.lcore.timer_thread); + rte_eal_remote_launch((lcore_function_t *)northbound, (void *)&vrg_ccb, vrg_ccb.lcore.northbound_thread); cmdline_printf(vrg_ccb.cl, "vRG> type ? or help to show all available commands\n"); cmdline_interact(vrg_ccb.cl); diff --git a/src/vrg.h b/src/vrg.h index 7f57b21..790646f 100644 --- a/src/vrg.h +++ b/src/vrg.h @@ -1,6 +1,10 @@ #ifndef _VRG_H_ #define _VRG_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include #include #include @@ -46,16 +50,11 @@ typedef struct { struct rte_timer link; /* for physical link checking timer */ }__rte_cache_aligned VRG_t; -/** - * @brief msg between IF driver and daemon - */ -typedef struct { - U16 type; - U8 refp[ETH_JUMBO]; - int len; -} tVRG_MBX; - int vrg_start(int argc, char **argv); void vrg_interrupt(); +#ifdef __cplusplus +} +#endif + #endif \ No newline at end of file