diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..ff1b4378 Binary files /dev/null and b/.DS_Store differ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..d6128fd8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,76 @@ +{ + "files.associations": { + "vector": "cpp", + "chrono": "cpp", + "__bit_reference": "cpp", + "__bits": "cpp", + "__config": "cpp", + "__debug": "cpp", + "__errc": "cpp", + "__hash_table": "cpp", + "__locale": "cpp", + "__mutex_base": "cpp", + "__node_handle": "cpp", + "__nullptr": "cpp", + "__split_buffer": "cpp", + "__string": "cpp", + "__threading_support": "cpp", + "__tuple": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "codecvt": "cpp", + "compare": "cpp", + "complex": "cpp", + "concepts": "cpp", + "condition_variable": "cpp", + "csignal": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "exception": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "ios": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "locale": "cpp", + "memory": "cpp", + "mutex": "cpp", + "new": "cpp", + "numeric": "cpp", + "optional": "cpp", + "ostream": "cpp", + "queue": "cpp", + "random": "cpp", + "ratio": "cpp", + "sstream": "cpp", + "stack": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "typeinfo": "cpp", + "unordered_map": "cpp", + "variant": "cpp", + "algorithm": "cpp" + } +} diff --git a/meson.build b/meson.build index 7194a5cf..7db78c8b 100644 --- a/meson.build +++ b/meson.build @@ -19,6 +19,7 @@ add_global_arguments('-march=native', language: ['c', 'cpp']) notification_buf_size = get_option('notification_buf_size') enso_pipe_size = get_option('enso_pipe_size') latency_opt = get_option('latency_opt') +mock = get_option('mock') add_global_arguments(f'-D NOTIFICATION_BUF_SIZE=@notification_buf_size@', language: ['c', 'cpp']) @@ -29,6 +30,10 @@ if latency_opt add_global_arguments('-D LATENCY_OPT', language: ['c', 'cpp']) endif +if mock + add_global_arguments(f'-D MOCK', language: ['c', 'cpp']) +endif + subdir('software') subdir('docs') subdir('hardware') diff --git a/meson_options.txt b/meson_options.txt index b82c7227..92fda7e1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -6,3 +6,5 @@ option('enso_pipe_size', type: 'integer', min: 0, max: 32768, value: 32768, description: 'Buffer size used by each software enso pipe') option('latency_opt', type: 'boolean', value: false, description: 'Optimize for latency') +option('mock', type: 'boolean', value: false, + description: 'Build in mock mode. Does not require the hardware.') diff --git a/software/examples/capture.cpp b/software/examples/capture.cpp index 10ecac97..ff53cfcb 100644 --- a/software/examples/capture.cpp +++ b/software/examples/capture.cpp @@ -31,6 +31,7 @@ */ #include +#include #include #include diff --git a/software/examples/echo.cpp b/software/examples/echo.cpp index 4dd1dbda..a9f29ab0 100644 --- a/software/examples/echo.cpp +++ b/software/examples/echo.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +87,7 @@ void run_echo(uint32_t nb_queues, uint32_t core_id, continue; } + int num_pkts = 0; for (auto pkt : batch) { ++pkt[63]; // Increment payload. @@ -94,6 +96,7 @@ void run_echo(uint32_t nb_queues, uint32_t core_id, } ++(stats->nb_pkts); + num_pkts += 1; } uint32_t batch_length = batch.processed_bytes(); pipe->ConfirmBytes(batch_length); diff --git a/software/examples/l2_forward.cpp b/software/examples/l2_forward.cpp index 9a4e148f..20c6cd97 100644 --- a/software/examples/l2_forward.cpp +++ b/software/examples/l2_forward.cpp @@ -49,9 +49,11 @@ static volatile bool setup_done = false; // Adapted from DPDK's rte_mov64() and rte_memcpy() functions. static _enso_always_inline void mov64(uint8_t* dst, const uint8_t* src) { - __m512i zmm0; - zmm0 = _mm512_loadu_si512((const void*)src); - _mm512_storeu_si512((void*)dst, zmm0); + // __m512i zmm0; + // zmm0 = _mm512_loadu_si512((const void*)src); + // _mm512_storeu_si512((void*)dst, zmm0); + (void)dst; + (void)src; } static _enso_always_inline void memcpy_64_align(void* dst, const void* src, diff --git a/software/include/enso/helpers.h b/software/include/enso/helpers.h index 37f951b2..a14870e2 100644 --- a/software/include/enso/helpers.h +++ b/software/include/enso/helpers.h @@ -54,6 +54,8 @@ #include #include #include +#include +#include #include namespace enso { @@ -79,6 +81,26 @@ struct stats_t { uint64_t nb_pkts; } __attribute__((aligned(64))); +#ifdef MOCK +// RSS 5-tuple containing dst port, src port, dst ip, src ip, protocol +typedef std::tuple + ConfigTuple; + +// A hash function used to hash the config tuple +struct HashConfigTuple { + template + + size_t operator()(const std::tuple& x) const { + return std::get<0>(x) ^ std::get<1>(x) ^ std::get<2>(x) ^ std::get<3>(x) ^ + std::get<4>(x); + } +}; + +// Hash map containing bindings of configurations to enso pipe IDs +extern std::unordered_map config_hashmap; + +#endif + /** * @brief Returns RTT, in number of cycles, for a given packet. * @@ -124,6 +146,8 @@ void print_pkt_header(uint8_t* pkt); void print_buf(void* buf, const uint32_t nb_cache_lines); +int rss_hash_packet(uint8_t* pkt_buf, int mod); + int set_core_id(std::thread& thread, int core_id); void show_stats(const std::vector& thread_stats, diff --git a/software/include/enso/internals.h b/software/include/enso/internals.h index fdc0d9bf..54c33d78 100644 --- a/software/include/enso/internals.h +++ b/software/include/enso/internals.h @@ -75,7 +75,7 @@ struct __attribute__((__packed__)) RxNotification { }; struct __attribute__((__packed__)) TxNotification { - uint64_t signal; + uint64_t signal; // whether or not notification has been consumed by hardware uint64_t phys_addr; uint64_t length; // In bytes (up to 1MB). uint64_t pad[5]; @@ -114,6 +114,8 @@ struct NotificationBufPair { void* uio_mmap_bar2_addr; // UIO mmap address for BAR 2. }; +#ifdef MOCK + struct RxEnsoPipeInternal { uint32_t* buf; uint64_t buf_phys_addr; @@ -121,10 +123,26 @@ struct RxEnsoPipeInternal { uint32_t* buf_head_ptr; uint32_t rx_head; uint32_t rx_tail; + uint32_t rx_actual_tail; uint64_t phys_buf_offset; // Use to convert between phys and virt address. enso_pipe_id_t id; }; +#else + +struct RxEnsoPipeInternal { + uint32_t* buf; + uint64_t buf_phys_addr; + struct QueueRegs* regs; + uint32_t* buf_head_ptr; + uint32_t rx_head; + uint32_t rx_tail; + uint64_t phys_buf_offset; // Use to convert between phys and virt address. + enso_pipe_id_t id; +}; + +#endif + } // namespace enso #endif // SOFTWARE_INCLUDE_ENSO_INTERNALS_H_ diff --git a/software/meson.build b/software/meson.build index 2a0d3880..8d397a54 100644 --- a/software/meson.build +++ b/software/meson.build @@ -3,8 +3,16 @@ inc = include_directories('include') subdir('include') subdir('src') -enso_lib = library('enso', project_sources, install: true, - include_directories: inc) +if mock + pcap_dep = dependency('pcap', version : '>=1.0') + enso_lib = library('enso', project_sources, install: true, + include_directories: inc, dependencies: pcap_dep) + +else + enso_lib = library('enso', project_sources, install: true, + include_directories: inc) +endif + pkg_mod = import('pkgconfig') pkg_mod.generate(enso_lib) diff --git a/software/src/enso/config.cpp b/software/src/enso/config.cpp index 7ce4b548..51903106 100644 --- a/software/src/enso/config.cpp +++ b/software/src/enso/config.cpp @@ -43,8 +43,12 @@ #include #include +#include #include +#include +#include +#include "../mock_pcie.h" #include "../pcie.h" namespace enso { @@ -86,12 +90,48 @@ struct __attribute__((__packed__)) RateLimitConfig { /** * Sends configuration through a notification buffer. * + * NOTE: if mock enso pipe, then add config notification to global hash table + * of queue configurations. + * * @param notification_buf_pair The notification buffer pair to send the * configuration through. * @param config_notification The configuration notification to send. Must be * a config notification, i.e., signal >= 2. * @return 0 on success, -1 on failure. */ +#ifdef MOCK + +int send_config(struct NotificationBufPair* notification_buf_pair, + struct TxNotification* config_notification) { + (void)notification_buf_pair; + FlowTableConfig* config = (FlowTableConfig*)config_notification; + // reject anything that is not binding a configuration to a pipe + assert(config->config_id == FLOW_TABLE_CONFIG_ID); + + // Make sure it's a config notification. + if (config->signal < 2) { + return -1; + } + + // Check if the enso pipe ID is within the hashmap of enso pipes + if (enso_pipes_map.find(config->enso_pipe_id) == enso_pipes_map.end()) + return -2; + + // Adding to hash map + uint16_t dst_port = config->dst_port; + uint16_t src_port = config->src_port; + uint32_t dst_ip = config->dst_ip; + uint32_t src_ip = config->src_ip; + uint32_t protocol = config->protocol; + ConfigTuple tup = + std::make_tuple(dst_port, src_port, dst_ip, src_ip, protocol); + + config_hashmap[tup] = config->enso_pipe_id; + + return 0; +} + +#else int send_config(struct NotificationBufPair* notification_buf_pair, struct TxNotification* config_notification) { struct TxNotification* tx_buf = notification_buf_pair->tx_buf; @@ -134,6 +174,7 @@ int send_config(struct NotificationBufPair* notification_buf_pair, return 0; } +#endif int insert_flow_entry(struct NotificationBufPair* notification_buf_pair, uint16_t dst_port, uint16_t src_port, uint32_t dst_ip, diff --git a/software/src/enso/helpers.cpp b/software/src/enso/helpers.cpp index ea57d954..5d9fe858 100644 --- a/software/src/enso/helpers.cpp +++ b/software/src/enso/helpers.cpp @@ -43,8 +43,13 @@ #include #include #include + namespace enso { +#ifdef MOCK +std::unordered_map config_hashmap; +#endif + uint16_t get_bdf_from_pcie_addr(const std::string& pcie_addr) { uint32_t domain, bus, dev, func; uint16_t bdf = 0; @@ -132,6 +137,51 @@ void print_pkt_header(uint8_t* pkt) { } } +#ifdef MOCK +/** + * @brief Hashes a packet with RSS to determine which pipe it should be + * directed to. + * + * @param pkt_buf packet buffer. + * @param mod number of pipes + * @return Index of pipe + */ +int rss_hash_packet(uint8_t* pkt_buf, int mod) { + struct ether_header* l2_hdr = (struct ether_header*)pkt_buf; + struct iphdr* l3_hdr = (struct iphdr*)(l2_hdr + 1); + uint32_t src_ip = l3_hdr->saddr; + uint32_t dst_ip = l3_hdr->daddr; + uint8_t protocol = l3_hdr->protocol; + uint32_t src_port; + uint32_t dst_port; + switch (protocol) { + case IPPROTO_TCP: { + struct tcphdr* l4_hdr = (struct tcphdr*)(l3_hdr + 1); + src_port = l4_hdr->source; + dst_port = l4_hdr->dest; + break; + } + case IPPROTO_UDP: { + struct udphdr* l4_hdr = (struct udphdr*)(l3_hdr + 1); + src_port = l4_hdr->source; + dst_port = l4_hdr->dest; + break; + } + default: + break; + } + + // check if this configuration has already been bound + ConfigTuple tup(dst_port, src_port, dst_ip, src_ip, protocol); + if (config_hashmap.find(tup) != config_hashmap.end()) { + return config_hashmap[tup]; + } + + return (src_ip ^ dst_ip ^ protocol ^ src_port ^ dst_port) % mod; +} + +#endif + int set_core_id(std::thread& thread, int core_id) { cpu_set_t cpuset; CPU_ZERO(&cpuset); diff --git a/software/src/enso/ixy_helpers.cpp b/software/src/enso/ixy_helpers.cpp index 73939d81..cdce2557 100644 --- a/software/src/enso/ixy_helpers.cpp +++ b/software/src/enso/ixy_helpers.cpp @@ -48,6 +48,9 @@ namespace enso { +#ifdef MOCK +uint64_t virt_to_phys(void* virt) { return (uint64_t)virt; } +#else uint64_t virt_to_phys(void* virt) { long page_size = sysconf(_SC_PAGESIZE); int fd = open("/proc/self/pagemap", O_RDONLY); @@ -78,6 +81,7 @@ uint64_t virt_to_phys(void* virt) { return (uint64_t)((phy & 0x7fffffffffffffULL) * page_size + ((uintptr_t)virt) % page_size); } +#endif void* get_huge_page(const std::string& name, bool mirror) { int fd; diff --git a/software/src/enso/pipe.cpp b/software/src/enso/pipe.cpp index de5fe1b2..2dec72bb 100644 --- a/software/src/enso/pipe.cpp +++ b/software/src/enso/pipe.cpp @@ -50,6 +50,7 @@ #include #include +#include "../mock_pcie.h" #include "../pcie.h" namespace enso { @@ -104,13 +105,17 @@ TxPipe::~TxPipe() { int TxPipe::Init() noexcept { if (internal_buf_) { +#ifndef MOCK std::string name = "enso:tx_pipe_" + std::to_string(kId); + buf_ = (uint8_t*)get_huge_page(name, true); if (unlikely(!buf_)) { return -1; } +#else + buf_ = (uint8_t*)(malloc(MOCK_ENSO_PIPE_SIZE)); +#endif } - buf_phys_addr_ = virt_to_phys(buf_); return 0; diff --git a/software/src/meson.build b/software/src/meson.build index e3c9d0a1..8ffe85b5 100644 --- a/software/src/meson.build +++ b/software/src/meson.build @@ -7,6 +7,12 @@ project_libraries = [] subdir('enso') subdir('syscall_api') -project_sources += files( - 'pcie.cpp' -) +if mock + project_sources += files( + 'mock_pcie.cpp' + ) +else + project_sources += files( + 'pcie.cpp' + ) +endif diff --git a/software/src/mock_pcie.cpp b/software/src/mock_pcie.cpp new file mode 100644 index 00000000..181d831e --- /dev/null +++ b/software/src/mock_pcie.cpp @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2022, Carnegie Mellon University + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted (subject to the limitations in the disclaimer + * below) provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY + * THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "mock_pcie.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syscall_api/intel_fpga_pcie_api.hpp" + +namespace enso { + +struct PcapHandlerContext { + struct Packet** buf; + int buf_position; + uint32_t hugepage_offset; + pcap_t* pcap; +}; + +// Number of seconds passed +struct timeval ts; +// PCAP object +pcap_t* pd; +// PCAP dumper to send packets to +pcap_dumper_t* pdumper_out; +// Buffer storing all incoming packets +struct Packet* in_buf[MAX_NUM_PACKETS]; +// Index of network head: where the program can start reading from +uint32_t network_head; +// Index of pipe tail: where the program can start writing to +uint32_t network_tail; +int num_queues = 1; +int queue_assignments[MAX_NUM_PACKETS / MOCK_BATCH_SIZE]; +std::chrono::milliseconds time_last_recv_call; + +int num_queues_initialized = 0; + +std::chrono::milliseconds get_curr_time_millis() { + std::chrono::milliseconds ms = + duration_cast(system_clock::now().time_since_epoch()); + return ms; +} + +int notification_buf_init(uint32_t bdf, int32_t bar, int16_t core_id, + struct NotificationBufPair* notification_buf_pair, + enso_pipe_id_t nb_queues, + enso_pipe_id_t enso_pipe_id_offset) { + (void)bdf; + (void)bar; + (void)core_id; + (void)notification_buf_pair; + (void)enso_pipe_id_offset; + num_queues = nb_queues; + return 0; +} + +/** + * @brief Called every time a packet is processed when reading from a PCAP file, + * reads from network file and copies to enso pipe. + * + * @param user + * @param pkt_hdr + * @param pkt_bytes + */ +void pcap_pkt_handler(u_char* user, const struct pcap_pkthdr* pkt_hdr, + const u_char* pkt_bytes) { + struct PcapHandlerContext* context = (struct PcapHandlerContext*)user; + uint32_t len = pkt_hdr->len; + + // gets enso pipe from packet bytes + int enso_pipe_index = rss_hash_packet((u_char*)pkt_bytes, num_queues); + struct RxEnsoPipeInternal* enso_pipe = enso_pipes_vector[enso_pipe_index]; + + // get the number of bytes available in the enso pipe + uint32_t num_bytes_available; + if (enso_pipe->rx_actual_tail < enso_pipe->rx_head) { + num_bytes_available = enso_pipe->rx_head - enso_pipe->rx_actual_tail; + } else { + num_bytes_available = + MOCK_ENSO_PIPE_SIZE - (enso_pipe->rx_actual_tail - enso_pipe->rx_head); + } + + // packets must be cache-aligned: so get aligned length + uint16_t nb_flits = (len - 1) / 64 + 1; + uint16_t pkt_aligned_len = nb_flits * 64; + + // check if space available in enso pipe to store packet + if (num_bytes_available < pkt_aligned_len) { + return; + } + + // copy packet into enso pipe + uint32_t tail = (enso_pipe->rx_actual_tail * 64) % MOCK_ENSO_PIPE_SIZE; + memcpy(((u_char*)enso_pipe->buf) + tail, pkt_bytes, len); + enso_pipe->rx_actual_tail = + (enso_pipe->rx_actual_tail + pkt_aligned_len / 64) % MOCK_ENSO_PIPE_SIZE; + + network_tail += 1; + // if we hit the max num packets to read, break from loop + if (network_tail == MAX_NUM_PACKETS) pcap_breakloop(context->pcap); +} + +/** + * @brief Helper function for reading from the incoming network file. + * + * @return 0 on success, negative on error. + */ +int read_in_file() { + // reading from in file and storing in a buffer + char errbuf[PCAP_ERRBUF_SIZE]; + pcap_t* pcap = pcap_open_offline("in.pcap", errbuf); + if (pcap == NULL) { + std::cerr << "Error loading pcap file (" << errbuf << ")" << std::endl; + return 2; + } + struct PcapHandlerContext context; + context.pcap = pcap; + context.buf = in_buf; + // read up to 256 packets + int err; + if ((err = pcap_loop(pcap, 0, pcap_pkt_handler, (u_char*)&context)) < 0) { + std::cerr << "Error while reading pcap (" << pcap_geterr(pcap) << ")" + << std::endl; + return 3; + } + return 0; +} + +/** + * @brief Initializes the given receiving enso pipe as a mock. + * + * @param enso_pipe enso pipe to initialize. + * @param notification_buf_pair unused in the mock. + * @param enso_pipe_id + * @return int + */ +int enso_pipe_init(struct RxEnsoPipeInternal* enso_pipe, + struct NotificationBufPair* notification_buf_pair, + enso_pipe_id_t enso_pipe_id) { + (void)notification_buf_pair; + (void)enso_pipe_id; + + // Setting up rx enso pipe with mock buffer + enso_pipe->buf = (uint32_t*)malloc(MOCK_ENSO_PIPE_SIZE); + enso_pipe->buf_phys_addr = (uint64_t)enso_pipe->buf; + enso_pipe->rx_head = 0; + enso_pipe->rx_tail = 0; + enso_pipe->rx_actual_tail = 0; + enso_pipe->id = enso_pipe_id; + + // adding enso pipe to hashmap of all pipes and to vector + enso_pipes_map[enso_pipe->id] = enso_pipe; + enso_pipes_vector.push_back(enso_pipe); + + num_queues_initialized += 1; + + if (num_queues_initialized == num_queues) { + ts.tv_sec = 0; + ts.tv_usec = 0; + + network_head = 0; + network_tail = 0; + + time_last_recv_call = get_curr_time_millis(); + + // opening file to dump packets to that mimics the network. + pd = pcap_open_dead(DLT_EN10MB, 65535); + pdumper_out = pcap_dump_open(pd, "out.pcap"); + + if (read_in_file() < 0) return -1; + } + + return 0; +} + +/** + * @brief Consumes from the network and puts the received packets on the correct + * pipe, which is determined with RSS hashing and then gives the address of the + * buffer to the caller. + * + * @param enso_pipe + * @param notification_buf_pair + * @param buf + * @param peek + * @return _enso_always_inline + */ +static _enso_always_inline uint32_t +__consume_queue(struct RxEnsoPipeInternal* enso_pipe, + struct NotificationBufPair* notification_buf_pair, void** buf, + bool peek = false) { + (void)notification_buf_pair; + // Get upper limit on batch size to emulate target rate + std::chrono::milliseconds curr_time = get_curr_time_millis(); + float diff_seconds = (curr_time.count() - time_last_recv_call.count()) / 1000; + uint32_t upper_limit_size = (uint32_t)(TARGET_RATE * diff_seconds); + uint32_t upper_limit_size_aligned = ((upper_limit_size - 1) / 64 + 1) * 64; + + uint32_t enso_pipe_head = enso_pipe->rx_tail; + uint32_t enso_pipe_tail = enso_pipe->rx_actual_tail; + + *buf = ((u_char*)enso_pipe->buf) + enso_pipe_head; + + if (enso_pipe_head == enso_pipe_tail) { + return 0; + } + + uint32_t flit_aligned_size = + ((enso_pipe_tail - enso_pipe_head) % MOCK_ENSO_PIPE_SIZE) * 64; + + flit_aligned_size = std::min(upper_limit_size_aligned, flit_aligned_size); + + if (!peek) { + enso_pipe_head = (enso_pipe_head + flit_aligned_size / 64) % ENSO_PIPE_SIZE; + enso_pipe->rx_tail = enso_pipe_head; + } + + return flit_aligned_size; +} + +/** + * @brief Sends packets to the network file to be read by other programs. + * + * @param notification_buf_pair + * @param phys_addr + * @param len + * @return uint32_t + */ +uint32_t send_to_queue(struct NotificationBufPair* notification_buf_pair, + uint64_t phys_addr, uint32_t len) { + (void)notification_buf_pair; + + u_char* addr_buf = new u_char[len]; + memcpy(addr_buf, (uint8_t*)phys_addr, len); + + uint32_t processed_bytes = 0; + uint8_t* pkt = addr_buf; + + while (processed_bytes < len) { + // read header of each packet to get packet length + uint16_t pkt_len = enso::get_pkt_len(pkt); + // packets must be cache-aligned: so get aligned length + uint16_t nb_flits = (pkt_len - 1) / 64 + 1; + uint16_t pkt_aligned_len = nb_flits * 64; + + // Save packet to file using pcap + struct pcap_pkthdr pkt_hdr; + pkt_hdr.ts = ts; + pkt_hdr.len = pkt_len; + pkt_hdr.caplen = pkt_len; + ++(ts.tv_usec); + pcap_dump((u_char*)pdumper_out, &pkt_hdr, pkt); + + // moving packet forward by aligned length + pkt += pkt_aligned_len; + processed_bytes += pkt_aligned_len; + } + + return len; +} + +int dma_init(struct NotificationBufPair* notification_buf_pair, + struct RxEnsoPipeInternal* enso_pipe, unsigned socket_id, + unsigned nb_queues, uint32_t bdf, int32_t bar) { + (void)notification_buf_pair; + (void)enso_pipe; + (void)socket_id; + (void)bdf; + (void)bar; + num_queues = nb_queues; + return 0; +} + +static _enso_always_inline uint16_t +__get_new_tails(struct NotificationBufPair* notification_buf_pair) { + (void)notification_buf_pair; + return 0; +} + +uint16_t get_new_tails(struct NotificationBufPair* notification_buf_pair) { + return __get_new_tails(notification_buf_pair); +} + +void prefetch_pipe(struct RxEnsoPipeInternal* enso_pipe) { (void)enso_pipe; } + +uint32_t get_next_batch_from_queue( + struct RxEnsoPipeInternal* enso_pipe, + struct NotificationBufPair* notification_buf_pair, void** buf) { + return __consume_queue(enso_pipe, notification_buf_pair, buf); +} + +uint32_t peek_next_batch_from_queue( + struct RxEnsoPipeInternal* enso_pipe, + struct NotificationBufPair* notification_buf_pair, void** buf) { + return __consume_queue(enso_pipe, notification_buf_pair, buf, true); +} + +// Return next batch among all open sockets. +uint32_t get_next_batch(struct NotificationBufPair* notification_buf_pair, + struct SocketInternal* socket_entries, + int* enso_pipe_id, void** buf) { + (void)socket_entries; + (void)notification_buf_pair; + (void)enso_pipe_id; + (void)buf; + return 0; +} + +void advance_ring_buffer(struct RxEnsoPipeInternal* enso_pipe, size_t len) { + (void)enso_pipe; + (void)len; +} + +void fully_advance_ring_buffer(struct RxEnsoPipeInternal* enso_pipe) { + (void)enso_pipe; +} + +void advance_pipe(struct RxEnsoPipeInternal* enso_pipe, size_t len) { + uint32_t rx_pkt_head = enso_pipe->rx_head; + uint32_t nb_flits = ((uint64_t)len - 1) / 64 + 1; + rx_pkt_head = (rx_pkt_head + nb_flits) % MOCK_ENSO_PIPE_SIZE; + + enso_pipe->rx_head = rx_pkt_head; + + printf("advanced enso pipe head to %d\n", rx_pkt_head); +} + +void fully_advance_pipe(struct RxEnsoPipeInternal* enso_pipe) { + (void)enso_pipe; +} + +static _enso_always_inline uint32_t +__send_to_queue(struct NotificationBufPair* notification_buf_pair, + uint64_t phys_addr, uint32_t len) { + (void)phys_addr; + (void)notification_buf_pair; + (void)len; + return 0; +} + +uint32_t get_unreported_completions( + [[maybe_unused]] struct NotificationBufPair* notification_buf_pair) { + return 0; +} + +void update_tx_head(struct NotificationBufPair* notification_buf_pair) { + (void)notification_buf_pair; +} + +void notification_buf_free(struct NotificationBufPair* notification_buf_pair) { + (void)notification_buf_pair; +} + +void enso_pipe_free(struct RxEnsoPipeInternal* enso_pipe) { + (void)enso_pipe; + pcap_dump_close(pdumper_out); + pcap_close(pd); +} + +int dma_finish(struct SocketInternal* socket_entry) { + (void)socket_entry; + return 0; +} + +uint32_t get_enso_pipe_id_from_socket(struct SocketInternal* socket_entry) { + (void)socket_entry; + return 0; +} + +void print_fpga_reg(IntelFpgaPcieDev* dev, unsigned nb_regs) { + (void)dev; + (void)nb_regs; +} + +void print_stats(struct SocketInternal* socket_entry, bool print_global) { + (void)socket_entry; + (void)print_global; +} + +static _enso_always_inline int32_t +__get_next_enso_pipe_id(struct NotificationBufPair* notification_buf_pair) { + (void)notification_buf_pair; + printf("get next enso pipe\n"); + if (network_head == network_tail) return -1; + int id = + enso_pipes_map[queue_assignments[network_head / MOCK_BATCH_SIZE]]->id; + printf("next pipe: %u\n", id); + return id; +} + +int32_t get_next_enso_pipe_id( + struct NotificationBufPair* notification_buf_pair) { + return __get_next_enso_pipe_id(notification_buf_pair); +} + +} // namespace enso diff --git a/software/src/mock_pcie.h b/software/src/mock_pcie.h new file mode 100644 index 00000000..afc06369 --- /dev/null +++ b/software/src/mock_pcie.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2022, Carnegie Mellon University + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted (subject to the limitations in the disclaimer + * below) provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY + * THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT + * NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @file + * @brief Functions to initialize and interface directly with the mock PCIe + * device. + * + * @author Kaajal Gupta + */ + +#ifndef SOFTWARE_SRC_MOCK_PCIE_H_ +#define SOFTWARE_SRC_MOCK_PCIE_H_ + +#include +#include +#include +#include + +#include +#include + +#include "syscall_api/intel_fpga_pcie_api.hpp" + +namespace enso { + +#define MAX_NUM_PACKETS 512 +#define MOCK_BATCH_SIZE 16 +#define MOCK_ENSO_PIPE_SIZE 32768 +#define KILOBYTE 1000L +#define GIGABYTE (KILOBYTE * KILOBYTE * KILOBYTE) +#define TARGET_RATE (100 * GIGABYTE) + +struct Packet { + u_char* pkt_bytes; + uint32_t pkt_len; +}; + +typedef struct RxEnsoPipeInternal* MockEnsoPipe; + +/** + * @brief Hashmap of all enso pipes in mock + * + */ +static std::unordered_map enso_pipes_map; +static std::vector enso_pipes_vector; + +} // namespace enso + +#endif // SOFTWARE_SRC_MOCK_PCIE_H_ diff --git a/software/src/pcie.cpp b/software/src/pcie.cpp index e42f801c..e8948abb 100644 --- a/software/src/pcie.cpp +++ b/software/src/pcie.cpp @@ -460,18 +460,24 @@ __send_to_queue(struct NotificationBufPair* notification_buf_pair, uint64_t hugepage_base_addr = transf_addr & hugepage_mask; uint64_t hugepage_boundary = hugepage_base_addr + kBufPageSize; + // send all missing bytes while (missing_bytes > 0) { + // get free slots (i.e., slots in the enso pipe which do not currently have + // data) uint32_t free_slots = (notification_buf_pair->tx_head - tx_tail - 1) % kNotificationBufSize; - // Block until we can send. + // Block until we can send: continually move TX pipe head forward based + // on pipe's completion notifications while (unlikely(free_slots == 0)) { ++notification_buf_pair->tx_full_cnt; + // updating head to move past packets already sent update_tx_head(notification_buf_pair); free_slots = (notification_buf_pair->tx_head - tx_tail - 1) % kNotificationBufSize; } + // moving to notification struct TxNotification* tx_notification = tx_buf + tx_tail; uint32_t req_length = std::min(missing_bytes, (uint32_t)kMaxTransferLen); uint32_t missing_bytes_in_page = hugepage_boundary - transf_addr;