Rearchitecture initial commit. Most of StorageCluster & Volume spec now in header files

Author: kennymalac

.gitignore

@@ -1,3 +1,6 @@
+# emacs
+*~
+
 # Prerequisites
 *.d
 

CMakeLists.txt

@@ -4,7 +4,7 @@ set(PROJECT_VERSION 0.0.1)
 
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 set(CMAKE_INCLUDE_CURRENT_DIR_IN_INTERFACE ON)
-set(CMAKE_CXX_COMPILER "/usr/lib64/llvm/8/bin/clang++")
+set(CMAKE_CXX_COMPILER "clang++")
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -38,10 +38,17 @@ set(SOURCES
   src/middlewares/FileStorage/storage.hpp
   src/middlewares/FileStorage/storage.hpp
   src/middlewares/FileStorage/storage.cpp
-  src/middlewares/FileStorage/haystack.hpp
-  src/middlewares/FileStorage/haystack.cpp
+#  src/middlewares/FileStorage/haystack.hpp
+#  src/middlewares/FileStorage/haystack.cpp
   src/middlewares/FileStorage/filesystem.hpp
   src/middlewares/FileStorage/filesystem.cpp
+  src/middlewares/Volume/marshaller.hpp
+  src/middlewares/Volume/volume.hpp
+  src/middlewares/Volume/services.hpp
+  src/middlewares/StorageCluster/request.hpp
+  src/middlewares/StorageCluster/response.hpp
+  src/middlewares/StorageCluster/services.hpp
+  src/middlewares/StorageCluster/storagecluster.hpp
   src/master.hpp
   src/master.cpp
   src/services.hpp
@@ -60,9 +67,11 @@ set(TEST_SOURCES
 include_directories("${PROJECT_SOURCE_DIR}")
 
 #add_subdirectory(src)
+include_directories(/usr/local/include/asio)
 add_library(TinyCDN_Base SHARED ${SOURCES})
 add_library(stdc++fs UNKNOWN IMPORTED)
-set_property(TARGET stdc++fs PROPERTY IMPORTED_LOCATION "/usr/lib64/gcc/x86_64-pc-linux-gnu/7.3.0/libstdc++fs.a")
+set_property(TARGET stdc++fs PROPERTY IMPORTED_LOCATION "/usr/lib/gcc/x86_64-linux-gnu/8/libstdc++fs.a")
+
 #target_link_libraries(TinyCDN_Base PUBLIC)
 add_executable(Run src/main.cpp)
 target_link_libraries(TinyCDN_Base stdc++fs)
@@ -79,7 +88,6 @@ add_executable(Test ${TEST_SOURCES})
 target_link_libraries(Test TinyCDN_Base Catch stdc++fs)
 add_test(NAME TestBase COMMAND Test)
 
-
 option(BUILD_CFFI "Generate the shared library" ON)
 if(BUILD_CFFI)
   add_library(tinyCDN SHARED src/middlewares/file_interop.h src/interop.c src/interop.cpp)

goals.org

@@ -3,6 +3,12 @@
 * TODO remove META file necessity (buckets should have uuid)
 * TODO volume node/master node distinction
 * TODO spawnCDN should load from configuration for storage nodes
+* TODO make FileBucketRegistry and VolumeRegistry use same base class
+** converter changes
+*** convertInput -> deserialize
+*** convertField -> deserializeField
+*** convertToValue -> serialize
+* TODO make Volume an abstract class like FileStorage?
 * TODO Dockerfiles for master node and storage node
 * TODO replication strategy class (geolocation, backup, & )
 * TODO owned filebuckets

src/hashing.hpp

@@ -4,7 +4,7 @@
 #include <cstring>
 #include <sstream>
 
-namespace TinyCDN {
+namespace TinyCDN::Utility::Hashing {
 
 class PseudoRandomHexFactory {
 public:
@@ -31,7 +31,6 @@ class PseudoRandomHexFactory {
   static constexpr char hex[17]{"0123456789ABCDEF"};
 };
 
-
 //! Wrapper type for fixed-size bitset with hex conversion to/from a string
 template <int fixedSize>
 class Id {
@@ -59,6 +58,10 @@ class Id {
     return *this;
   }
 
+  bool operator==(const Id<fixedSize> &other) const {
+    return this->_value == other->value();
+  }
+
   friend std::ostream& operator<< (std::ostream &out, const Id<fixedSize> &id) {
     out << std::hex << id.value().to_ulong();
     return out;
@@ -69,17 +72,17 @@ class Id {
     return out;
   }
 
-  std::string str() {
+  std::string str() const {
     std::stringstream hex;
     hex << std::hex << _value.to_ulong();
     return hex.str();
   }
 
-  const char* c_str() {
+  const char* c_str() const {
     return this->str().c_str();
   }
 
-  inline std::bitset<fixedSize> value() noexcept {
+  inline std::bitset<fixedSize> value() const noexcept {
     return _value;
   }
 
@@ -95,6 +98,15 @@ class Id {
   std::bitset<fixedSize> _value;
 };
 
+class IdHasher {
+public:
+  template <int fixedSize>
+  std::size_t operator()(const Id<fixedSize> &id) const {
+    const std::bitset<fixedSize> id2 = id.value();
+    return std::hash<std::bitset<fixedSize>>()(id2);
+  }
+};
+
 using UUID = Id<128>;
 
 class UUID4Factory {

src/middlewares/StorageCluster/interop.h

@@ -0,0 +1 @@
+#pragma once

src/middlewares/StorageCluster/request.hpp

@@ -0,0 +1 @@
+#pragma once

src/middlewares/StorageCluster/response.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "storagecluster.hpp"
+
+namespace TinyCDN::Middleware::StorageCluster {
+
+struct StorageFileUploadingSession {
+};
+
+struct StorageFileHostingSession {
+};
+}

src/middlewares/StorageCluster/services.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+
+#include "../Volume/volume.hpp"
+#include "request.hpp"
+#include "response.hpp"
+
+struct StorageFileHostingService;
+struct StorageFileUploadingService;
+
+namespace TinyCDN::Middleware::StorageCluster {
+using namespace TinyCDN::Middleware::Volume;
+
+struct MasterRequest;
+struct MasterResponse;
+class StorageClusterRequest;
+class StorageClusterResponse;
+
+/*!
+ * \brief A single "core" allocated with available storage.
+ * Hosts a virtual volume and receives commands from Master node to store/retrieve files to/from filebuckets, or replicate filebuckets, etc.
+Gets a authentication key from the Master node that gives permission to store/retrieve a file.
+ */
+class StorageClusterNode {
+public:
+  std::unique_ptr<VirtualVolume> virtualVolume;
+
+  std::unique_ptr<StorageFileHostingService> getHostingService();
+  std::unique_ptr<StorageFileUploadingService> getUploadingService();
+
+  // std::map<SessionId, StorageFileHostingSession>
+p
+  // HTTP frontend: implement the following
+  // parseStorageClusterMasterRequest(std::string message); -> StorageClusterRequest
+  // parseStorageClusterClientRequest(std::string message); -> StorageClusterRequest
+
+  // TODO: Use Command pattern
+  StorageClusterResponse receiveMasterCommand(StorageClusterRequest request);
+  void sendMasterCommand(MasterRequest request);
+  // NOTE: The client does not need TCP wrappers, client communication going from HTTP -> CFFI
+  // StorageClusterResponse receiveClientCommand(StorageClusterRequest request);
+  // StorageClusterResponse sendClientCommand(StorageClusterRequest request);
+
+private:
+  // TODO: object pool of services
+  /*
+    This is where we can create the most threads.
+    Each service will exist in its own thread.
+    The service shares access to the virtual volume's volumemanager and access to the lookup of storagevolumes
+  */
+  std::unique_ptr<StorageFileUploadingService> uploadingService;
+  std::unique_ptr<StorageFileHostingService> hostingService;
+
+  // TODO pool of services... see above
+  std::mutex uploadServiceMutex;
+  std::mutex hostingServiceMutex;
+  std::mutex virtualVolumeMutex;
+
+  // TODO: networking
+  // masterSocket
+};
+}

src/middlewares/StorageCluster/storagecluster.hpp

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "volume.hpp"
+
+namespace TinyCDN::Middleware::Volume {
+
+//! A Volume CSV gets converted into this POD and subsequently this data is assigned to a Volume instance
+struct VolumeParams {
+  VolumeId id;
+  uintmax_t size;
+
+  VolumeParams() : size(0) {};
+  VolumeParams(VolumeId id, uintmax_t size) : id(id), size(size) {};
+};
+
+//! Converts volumes from a CSV into a POD into an object instance
+struct VolumeCSVMarshaller {
+  std::unique_ptr<VolumeParams> params;
+
+  //! Takes a Volume field and assigns it to its deduced conversion value
+  auto deserializeField(std::string field, std::string value);
+
+  //! Creates a Volume instance by taking params and creating a Volume instance from it
+  template <typename T>
+  std::unique_ptr<T> deserialize();
+
+  //! Convenience helper method for generating fieldName=
+  inline std::string assignmentToken(std::string fieldName) {
+    return fieldName + "=";
+  }
+
+  //! Resets the converter by emptying the current VolumeParams
+  inline void reset() {
+    params = std::make_unique<VolumeParams>();
+  }
+
+  inline VolumeCSVMarshaller() {
+    params = std::make_unique<VolumeParams>();
+  }
+};
+
+// JSON because this may be read by other programs and will not grow very large
+struct VirtualVolumeJsonMarshaller  {
+  // TODO decide what JSON library to use
+};
+}

src/middlewares/Volume/marshaller.hpp

@@ -0,0 +1,3 @@
+// class VolumeStorageService;
+namespace TinyCDN::Middleware::Volume {
+}

src/middlewares/Volume/services.hpp

@@ -0,0 +1,119 @@
+#pragma once
+
+#include <vector>
+#include <memory>
+#include <shared_mutex>
+#include <unordered_map>
+#include <optional>
+
+#include "../../utility.hpp"
+#include "../../hashing.hpp"
+
+using namespace TinyCDN::Utility::Exceptions;
+using namespace TinyCDN::Utility::Hashing;
+
+namespace TinyCDN::Middleware::Volume {
+
+using TinyCDN::Utility::Size;
+
+using FileBucketId = Id<64>;
+using VolumeId = Id<32>;
+
+/*!
+ * \brief Abstract Static size blob
+*/
+class Volume {
+public:
+  // TODO remove this? StorageVolumeManager needs this, not Volume itself
+  const VolumeId id;
+  //! The maximum allowed size
+  Size size;
+
+  // virtual void resize(t size) = 0;
+  virtual void destroy() = 0;
+
+  // bool hasBucketId(FileBucketId id);
+
+  inline Volume(VolumeId id, Size size) : id(id), size(size) {};
+};
+
+/*! Volume capable of storing files.
+ * FileBuckets are assigned volumes based on location, config, etc.
+ A StorageVolume will be assigned many different buckets from a VirtualVolume, but if a bucket does not populate, it can "spill over" into another StorageVolume if necessary.
+ At some point, every now and then the volume can reallocate space by reclaiming lost volume space. (?)
+ A StorageVolume is not be tied to the storage requirements of its buckets.
+ StorageVolumes are replicated at a lower level than buckets. Buckets with default replication settings will be replicated as part of a volume manager's replication strategy.
+*/
+template <typename storageType>
+class StorageVolume : Volume {
+public:
+  inline void resize() {
+    throw new NotImplementedException();
+  }
+
+  inline void destroy() {
+    throw new NotImplementedException();
+  }
+
+  //! A file storage driver that provides methods to retrieve, modify, and delete files
+  std::unique_ptr<storageType> storage;
+
+  inline StorageVolume(Size size, fs::path location, bool preallocated)
+    : Volume(id, size) {
+    storage = std::make_unique<storageType>(size, location, preallocated);
+
+  }
+  // TODO delete default, volume constructors
+};
+
+//! All StorageVolume types
+using AnyStorageVolume = std::variant<StorageVolume<FileStorage::FilesystemStorage>>;
+// Could be any StorageVolume instance, or a non-existent value
+// Could use std::optional, but wrapping variant would make std::visit less usable
+using MaybeAnyStorageVolume = std::variant<std::monostate, StorageVolume<FileStorage::FilesystemStorage>>;
+
+//class BackupVolume : Volume;
+
+/*!
+ * \brief Responsible for allocating, replicating, and removing volumes as necessary
+ */
+class StorageVolumeManager {
+public:
+  uintmax_t getSize();
+  //! Checks if size is going DOWN, if so, will replicate elsewhere?
+  void setSize(uintmax_t size);
+
+  std::unique_ptr<AnyStorageVolume> getStorageVolume(VolumeId id);
+  template <typename T>
+  std::unique_ptr<StorageVolume<T>> createStorageVolume(fs::path location);
+  // replicateVolume
+  void removeVolume(VolumeId id);
+
+  inline StorageVolumeManager(uintmax_t size) : size(size) {}
+
+private:
+  uintmax_t size;
+
+  std::unordered_map<VolumeId, std::unique_ptr<MaybeAnyStorageVolume>, IdHasher> volumes;
+  std::unordered_map<VolumeId, std::shared_mutex, IdHasher> volumeMutexes;
+};
+
+/*!
+ * \brief Volume that can be of a very large indeterminate size and manages many volumes within it.
+ * The simple difference between a VirtualVolume and a StorageVolume is that a virtual volume is an alias to a single Volume that manages many volumes internally to efficently store large amounts of files.
+ A FileBucket references a single virtual volume instead of the set of StorageVolumes that store its files.
+*/
+class VirtualVolume : Volume {
+public:
+  fs::path location;
+  std::optional<std::vector<VolumeId>> getVolumeIds();
+  //! Also modifies storage volume manager's size
+  void setSize(uintmax_t size);
+  StorageVolumeManager storageVolumeManager;
+
+private:
+  uintmax_t size;
+  std::unordered_map<FileBucketId, std::vector<VolumeId>, IdHasher> fbVolDb;
+
+};
+}