Feature/vfio platform

.gitignore

@@ -11,3 +11,7 @@ build/
 *.so
 *.user
 graph.dot
+
+.clang-format
+.cache/
+miob/

.gitlab-ci.yml

@@ -66,7 +66,7 @@ test:cppcheck:
                --error-exitcode=1
                --quiet
                --inline-suppr
-               --enable=warning,performance,portability,information,missingInclude
+               --enable=cppcoreguidelines,warning,performance,portability,information,missingInclude
                --std=c++11
                --suppress=noValidConfiguration
                -I include

.vscode/launch.json

@@ -8,7 +8,7 @@
             "name": "(gdb) Launch",
             "type": "cppdbg",
             "request": "launch",
-            "program": "${workspaceFolder}/build/src/villas-fpga-ctrl",
+            "program": "${workspaceFolder}/build/src/platform",
             "args": [
                 "-c", "${workspaceFolder}/etc/fpgas.json", "--connect", "\"2<->stdout\""
             ],

CMakeLists.txt

@@ -15,7 +15,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
 
 # Several CMake settings/defaults
 set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra")
 
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     set(TOPLEVEL_PROJECT ON)

etc/fpgas.json

@@ -1,6 +1,6 @@
 {
     "fpgas": {
-      "vc707": {
+      "zcu106": {
         "id": "10ee:7021",
         "slot": "0000:88:00.0",
         "do_reset": true,

include/villas/fpga/card.hpp

@@ -22,10 +22,10 @@ class Card
 {
 public:
         bool polling;
-
-        std::string name; // The name of the FPGA card
+        bool doReset;					// Reset VILLASfpga during startup?
+	int affinity;					// Affinity for MSI interrupts
+	std::string name;			// The name of the FPGA card
         std::shared_ptr<kernel::vfio::Container> vfioContainer;
-        std::shared_ptr<kernel::vfio::Device> vfioDevice;
 
         // Slave address space ID to access the PCIe address space from the
         // FPGA
@@ -52,6 +52,7 @@ class Card
 	std::map<MemoryManager::AddressSpaceId, std::shared_ptr<MemoryBlock>> memoryBlocksMapped;
 
         Logger logger;
+
 };
 
 } // namespace fpga

include/villas/fpga/card_parser.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "villas/exceptions.hpp"
+#include "villas/log.hpp"
+#include <jansson.h>
+#include <spdlog/logger.h>
+
+class CardParser {
+public:
+  std::shared_ptr<spdlog::logger> logger;
+
+  json_t *json_ips = nullptr;
+  json_t *json_paths = nullptr;
+  const char *pci_slot = nullptr;
+  const char *pci_id = nullptr;
+  int do_reset = 0;
+  int affinity = 0;
+  int polling = 0;
+  json_t *devices = nullptr;
+  std::vector<std::string> device_names;
+
+  CardParser(json_t *json_card) : logger(villas::logging.get("CardParser")) {
+      json_error_t err;
+      int ret = json_unpack_ex(
+          json_card, &err, 0,
+          "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o, s?: o }", "ips",
+          &json_ips, "affinity", &affinity, "do_reset", &do_reset, "slot",
+          &pci_slot, "id", &pci_id, "polling", &polling, "paths", &json_paths,
+          "devices", &devices);
+
+      if (ret != 0)
+        throw villas::ConfigError(json_card, err, "", "Failed to parse card");
+
+    // devices array parsing
+    size_t index;
+    json_t *value;
+    json_array_foreach(devices, index, value) {
+        auto str = json_string_value(value);
+        device_names.push_back(str);    
+    }
+  }
+};

include/villas/fpga/ip_loader.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include "villas/exceptions.hpp"
+#include "villas/log.hpp"
+#include <filesystem>
+#include <jansson.h>
+#include <spdlog/logger.h>
+
+using villas::ConfigError;
+
+class IpLoader {
+public:
+  std::shared_ptr<spdlog::logger> logger;
+
+
+  IpLoader(json_t* json_ips, const std::filesystem::path& searchPath): logger(villas::logging.get("IpParser")) {
+    // Load IPs from a separate json file
+    if (!json_is_string(json_ips)) {
+      logger->debug("FPGA IP cores config item is not a string.");
+      throw ConfigError(json_ips, "node-config-fpga-ips",
+                        "FPGA IP cores config item is not a string.");
+    }
+    if (!searchPath.empty()) {
+      std::filesystem::path json_ips_path =
+          searchPath / json_string_value(json_ips);
+      logger->debug("searching for FPGA IP cors config at {}", json_ips_path);
+      json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr);
+    }
+    if (json_ips == nullptr) {
+      json_ips =
+          json_load_file(json_string_value(json_ips), 0, nullptr);
+      logger->debug("searching for FPGA IP cors config at {}",
+                    json_string_value(json_ips));
+      if (json_ips == nullptr) {
+        throw ConfigError(json_ips, "node-config-fpga-ips",
+                          "Failed to find FPGA IP cores config");
+      }
+    }
+
+    if (not json_is_object(json_ips))
+      throw ConfigError(json_ips, "node-config-fpga-ips",
+                        "FPGA IP core list must be an object!");
+  }
+};

include/villas/fpga/ips/intc.hpp

@@ -27,25 +27,25 @@ class InterruptController : public Core {
 	virtual
 	bool init() override;
 
-	bool enableInterrupt(IrqMaskType mask, bool polling);
-	bool enableInterrupt(IrqPort irq, bool polling)
+	virtual bool enableInterrupt(IrqMaskType mask, bool polling);
+	virtual bool enableInterrupt(IrqPort irq, bool polling)
 	{
 		return enableInterrupt(1 << irq.num, polling);
 	}
 
-	bool disableInterrupt(IrqMaskType mask);
-	bool disableInterrupt(IrqPort irq)
+	virtual bool disableInterrupt(IrqMaskType mask);
+	virtual bool disableInterrupt(IrqPort irq)
 	{
 		return disableInterrupt(1 << irq.num);
 	}
 
-	int waitForInterrupt(int irq);
-	int waitForInterrupt(IrqPort irq)
+	virtual int waitForInterrupt(int irq);
+	virtual int waitForInterrupt(IrqPort irq)
 	{
 		return waitForInterrupt(irq.num);
 	}
 
-private:
+protected:
 
 	static constexpr char registerMemory[] = "reg0";
 

include/villas/fpga/ips/platform_intc.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <villas/fpga/ips/intc.hpp>
+
+class PlatformInterruptController
+    : public villas::fpga::ip::InterruptController {
+public:
+  bool init() override;
+
+  bool enableInterrupt(InterruptController::IrqMaskType mask,
+                       bool polling) override;
+  bool enableInterrupt(IrqPort irq, bool polling) override;
+};

include/villas/fpga/ips/zynq.hpp

@@ -0,0 +1,78 @@
+/** Zynq VFIO connector node
+ *
+ * Author: Pascal Bauer <[email protected]>
+ * SPDX-FileCopyrightText: 2017 Steffen Vogel <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ *********************************************************************************/
+
+#pragma once
+
+#include <xilinx/xaxis_switch.h>
+
+#include <villas/fpga/node.hpp>
+
+namespace villas {
+namespace fpga {
+namespace ip {
+
+class Zynq : public Core {
+public:
+	friend class ZynqFactory;
+
+	virtual
+	bool init() override;
+
+private:
+	static constexpr char axiInterface[] = "M_AXI";
+	static constexpr char pcieMemory[] = "BAR0";
+
+	struct AxiBar {
+		uintptr_t base;
+		size_t size;
+		uintptr_t translation;
+	};
+
+	struct PciBar {
+		uintptr_t translation;
+	};
+
+	std::map<std::string, AxiBar> axiToPcieTranslations;
+	std::map<std::string, PciBar> pcieToAxiTranslations;
+};
+
+class ZynqFactory : CoreFactory {
+
+public:
+	virtual
+	std::string getName() const
+	{
+		return "pcie";
+	}
+
+	virtual
+	std::string getDescription() const
+	{
+		return "Custom platform vfio connector";
+	}
+
+private:
+	virtual
+	Vlnv getCompatibleVlnv() const
+	{
+		return Vlnv("xilinx.com:ip:zynq_ultra_ps_e:");
+	}
+
+	// Create a concrete IP instance
+	Core* make() const
+	{
+		return new Zynq;
+	};
+
+protected:
+	virtual
+	void parse(Core &, json_t *) override;
+};
+
+} /* namespace ip */
+} /* namespace fpga */
+} /* namespace villas */

include/villas/fpga/pcie_card.hpp

@@ -61,13 +61,15 @@ class PCIeCard : public Card {
 	{ }
 
 public:	// TODO: make this private
-	bool doReset;					// Reset VILLASfpga during startup?
-	int affinity;					// Affinity for MSI interrupts
+	std::shared_ptr<kernel::pci::Device> pdev;	// PCI device handle
+	std::shared_ptr<kernel::vfio::Device> vfioDevice; //? Only used by intc
 
-        std::shared_ptr<kernel::pci::Device> pdev; // PCI device handle
-
-      protected:
-        Logger getLogger() const { return villas::logging.get(name); }
+protected:
+	Logger
+	getLogger() const
+	{
+		return villas::logging.get(name);
+	}
 };
 
 class PCIeCardFactory : public plugin::Plugin {

include/villas/fpga/platform_card.hpp

@@ -0,0 +1,44 @@
+/** 
+ * Author: Pascal Henry Bauer <[email protected]>
+ * Based on the work of: Steffen Vogel <[email protected]> and Daniel Krebs <[email protected]>
+ *
+ * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC
+ * SPDX-License-Identifier: Apache-2.0
+ *********************************************************************************/
+
+#pragma once
+
+#include <villas/fpga/card.hpp>
+#include <filesystem>
+#include <vector>
+
+namespace villas {
+namespace fpga {
+
+class PlatformCard : public Card
+{
+public:
+  PlatformCard(std::shared_ptr<kernel::vfio::Container> vfioContainer,
+               std::vector<std::string> device_names);
+
+  ~PlatformCard(){};
+
+  std::vector<std::shared_ptr<kernel::vfio::Device>> devices;
+
+  void connectVFIOtoIPS();
+  bool mapMemoryBlock(const std::shared_ptr<MemoryBlock> block) override;
+
+private:    
+
+};
+
+class PlatformCardFactory
+{
+public:
+        static std::list<std::shared_ptr<Card>> make(json_t *json, 
+                                        std::shared_ptr<kernel::vfio::Container> vc,
+                                        const std::filesystem::path& searchPath);
+};
+
+} /* namespace fpga */
+} /* namespace villas */

lib/CMakeLists.txt

@@ -8,6 +8,7 @@ set(SOURCES
 	vlnv.cpp
 	card.cpp
 	pcie_card.cpp
+	platform_card.cpp
 	core.cpp
 	node.cpp
 	utils.cpp
@@ -23,10 +24,12 @@ set(SOURCES
 	ips/gpio.cpp
 	ips/intc.cpp
 	ips/pcie.cpp
+	ips/platform_intc.cpp
 	ips/rtds.cpp
 	ips/switch.cpp
 	ips/timer.cpp
 	ips/i2c.cpp
+	ips/zynq.cpp
 
 	ips/rtds2gpu/rtds2gpu.cpp
 	ips/rtds2gpu/xrtds2gpu.c