Build time improvements

cpp/arcticdb/CMakeLists.txt

@@ -208,10 +208,13 @@ set(arcticdb_srcs
         entity/serialized_key.hpp
         entity/type_conversion.hpp
         entity/types.hpp
+        entity/types_proto.hpp
         entity/types-inl.hpp
         entity/variant_key.hpp
         entity/versioned_item.hpp
         entity/descriptor_item.hpp
+        entity/field_collection.hpp
+        entity/field_collection_proto.hpp
         log/log.hpp
         log/trace.hpp
         pipeline/column_mapping.hpp
@@ -371,11 +374,13 @@ set(arcticdb_srcs
         column_store/string_pool.cpp
         entity/data_error.cpp
         entity/field_collection.cpp
+        entity/field_collection_proto.cpp
         entity/key.cpp
         entity/merge_descriptors.cpp
         entity/metrics.cpp
         entity/performance_tracing.cpp
         entity/types.cpp
+        entity/types_proto.cpp
         log/log.cpp
         pipeline/column_stats.cpp
         pipeline/frame_slice.cpp

cpp/arcticdb/async/tasks.hpp

@@ -22,6 +22,7 @@
 #include <arcticdb/pipeline/frame_slice.hpp>
 #include <arcticdb/processing/processing_unit.hpp>
 #include <arcticdb/util/constructors.hpp>
+#include <arcticdb/codec/codec.hpp>
 
 #include <type_traits>
 

cpp/arcticdb/column_store/chunked_buffer.cpp

@@ -7,6 +7,8 @@
 
 #include <arcticdb/column_store/chunked_buffer.hpp>
 
+#include <optional>
+
 namespace arcticdb {
 
 template <size_t BlockSize>

cpp/arcticdb/column_store/memory_segment.hpp

@@ -8,18 +8,11 @@
 #pragma once
 
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/cursor.hpp>
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/util/preconditions.hpp>
-#include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/column_store/memory_segment_impl.hpp>
 
-#include <google/protobuf/message.h>
-#include <google/protobuf/any.h>
-#include <google/protobuf/any.pb.h>
-
 namespace arcticdb {
 
 /*

cpp/arcticdb/column_store/memory_segment_impl.cpp

@@ -8,10 +8,28 @@
 #include <arcticdb/column_store/memory_segment_impl.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
 #include <arcticdb/entity/type_utils.hpp>
-#include <arcticdb/stream/index.hpp>
-#include <arcticdb/util/format_date.hpp>
+
+#include <google/protobuf/any.h>
+#include <google/protobuf/any.pb.h>
+
 
 namespace arcticdb {
+
+    SegmentInMemoryImpl::SegmentInMemoryImpl() = default;
+
+    SegmentInMemoryImpl::SegmentInMemoryImpl( const StreamDescriptor& desc, size_t expected_column_size, bool presize, bool allow_sparse)
+        : descriptor_(std::make_shared<StreamDescriptor>(StreamDescriptor{ desc.id(), desc.index() }))
+        , allow_sparse_(allow_sparse)
+    {
+        on_descriptor_change(desc, expected_column_size, presize, allow_sparse);
+    }
+
+    SegmentInMemoryImpl::~SegmentInMemoryImpl()
+    {
+        ARCTICDB_TRACE(log::version(), "Destroying segment in memory");
+    }
+
+
 // Append any columns that exist both in this segment and in the 'other' segment onto the
 // end of the column in this segment. Any columns that exist in this segment but not in the
 // one being appended will be default-valued (or sparse) in this segment. Any columns in the
@@ -598,4 +616,23 @@ void SegmentInMemoryImpl::set_timeseries_descriptor(TimeseriesDescriptor&& tsd)
     set_metadata(std::move(any));
 }
 
+void SegmentInMemoryImpl::set_metadata(google::protobuf::Any&& meta) {
+    util::check_arg(!metadata_, "Cannot override previously set metadata");
+    if (meta.ByteSizeLong())
+        metadata_ = std::make_unique<google::protobuf::Any>(std::move(meta));
+}
+
+void SegmentInMemoryImpl::override_metadata(google::protobuf::Any&& meta) {
+    if (meta.ByteSizeLong())
+        metadata_ = std::make_unique<google::protobuf::Any>(std::move(meta));
+}
+
+bool SegmentInMemoryImpl::has_metadata() const {
+    return static_cast<bool>(metadata_);
+}
+
+const google::protobuf::Any* SegmentInMemoryImpl::metadata() const {
+    return metadata_.get();
+}
+
 }

cpp/arcticdb/column_store/memory_segment_impl.hpp

@@ -8,7 +8,6 @@
 #pragma once
 
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/cursor.hpp>
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/util/offset_string.hpp>
 #include <arcticdb/util/preconditions.hpp>
@@ -18,18 +17,16 @@
 #include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/column_store/column_map.hpp>
-#include <arcticdb/pipeline/string_pool_utils.hpp>
-#include <arcticdb/util/format_date.hpp>
-#include <arcticdb/stream/index.hpp>
-#include <arcticdb/util/hash.hpp>
 #include <arcticdb/entity/stream_descriptor.hpp>
 
-#include <google/protobuf/message.h>
-#include <google/protobuf/any.h>
-#include <google/protobuf/any.pb.h>
 #include <boost/iterator/iterator_facade.hpp>
 #include <folly/container/Enumerate.h>
 
+namespace google::protobuf
+{
+    class Any;
+}
+
 namespace arcticdb {
 
 class SegmentInMemoryImpl;
@@ -56,14 +53,6 @@ inline void check_output_bitset(const arcticdb::util::BitSet& output,
 }
 } // namespace anon
 
-inline bool operator==(const Field::Proto& left, const Field::Proto& right) {
-    google::protobuf::util::MessageDifferencer diff;
-    return diff.Compare(left, right);
-}
-
-inline bool operator<(const Field::Proto& left, const Field::Proto& right) {
-    return left.name() < right.name();
-}
 
 class SegmentInMemoryImpl {
 public:
@@ -380,21 +369,15 @@ class SegmentInMemoryImpl {
 
 
 
-    SegmentInMemoryImpl() = default;
+    SegmentInMemoryImpl();
 
     explicit SegmentInMemoryImpl(
-        const StreamDescriptor &desc,
+        const StreamDescriptor& desc,
         size_t expected_column_size,
         bool presize,
-        bool allow_sparse) :
-        descriptor_(std::make_shared<StreamDescriptor>(StreamDescriptor{desc.id(), desc.index()})),
-        allow_sparse_(allow_sparse) {
-        on_descriptor_change(desc, expected_column_size, presize, allow_sparse);
-    }
+        bool allow_sparse);
 
-    ~SegmentInMemoryImpl() {
-        ARCTICDB_TRACE(log::version(), "Destroying segment in memory");
-    }
+    ~SegmentInMemoryImpl();
 
     iterator begin() { return iterator{this}; }
 
@@ -734,24 +717,11 @@ class SegmentInMemoryImpl {
 
     StringPool &string_pool() { return *string_pool_; } //TODO protected
 
-    void set_metadata(google::protobuf::Any &&meta) {
-        util::check_arg(!metadata_, "Cannot override previously set metadata");
-        if (meta.ByteSizeLong())
-            metadata_ = std::make_unique<google::protobuf::Any>(std::move(meta));
-    }
-
-    void override_metadata(google::protobuf::Any &&meta) {
-        if (meta.ByteSizeLong())
-            metadata_ = std::make_unique<google::protobuf::Any>(std::move(meta));
-    }
+    void set_metadata(google::protobuf::Any&& meta);
+    void override_metadata(google::protobuf::Any&& meta);
+    bool has_metadata() const;
 
-    bool has_metadata() {
-        return static_cast<bool>(metadata_);
-    }
-
-    const google::protobuf::Any *metadata() const {
-        return metadata_.get();
-    }
+    const google::protobuf::Any* metadata() const;
 
     bool is_index_sorted() const;
 

cpp/arcticdb/entity/atom_key.hpp

@@ -10,10 +10,9 @@
 #include <arcticdb/entity/key.hpp>
 #include <arcticdb/entity/types.hpp>
 #include <arcticdb/entity/index_range.hpp>
-#include <arcticdb/util/string_utils.hpp>
 #include <variant>
+#include <optional>
 #include <fmt/format.h>
-#include <string_view>
 
 namespace arcticdb::entity {
 

cpp/arcticdb/entity/field_collection.hpp

@@ -10,6 +10,7 @@
 #include <arcticdb/column_store/chunked_buffer.hpp>
 #include <util/cursored_buffer.hpp>
 #include <util/buffer.hpp>
+#include <entity/types.hpp>
 #include <arcticdb/column_store/column_data.hpp>
 
 namespace arcticdb {
@@ -191,13 +192,6 @@ class FieldCollection {
     }
 };
 
-inline FieldCollection fields_from_proto(const arcticdb::proto::descriptors::StreamDescriptor& desc) {
-    FieldCollection output;
-    for(const auto& field : desc.fields())
-        output.add_field(type_desc_from_proto(field.type_desc()), field.name());
-
-    return output;
-}
 
 template <typename RangeType>
 FieldCollection fields_from_range(const RangeType& fields) {
@@ -208,12 +202,5 @@ FieldCollection fields_from_range(const RangeType& fields) {
     return output;
 }
 
-inline void proto_from_fields(const FieldCollection& fields, arcticdb::proto::descriptors::StreamDescriptor& desc) {
-    for(const auto& field : fields) {
-        auto new_field = desc.add_fields();
-        new_field->MergeFrom(field_proto(field.type().data_type(), field.type().dimension(), field.name()));
-    }
-}
-
 
 } //namespace arcticdb

cpp/arcticdb/entity/field_collection_proto.cpp

@@ -0,0 +1,30 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#include <arcticdb/entity/field_collection_proto.hpp>
+
+namespace arcticdb {
+
+
+FieldCollection fields_from_proto(const arcticdb::proto::descriptors::StreamDescriptor& desc) {
+    FieldCollection output;
+    for (const auto& field : desc.fields())
+        output.add_field(type_desc_from_proto(field.type_desc()), field.name());
+
+    return output;
+}
+
+
+void proto_from_fields(const FieldCollection& fields, arcticdb::proto::descriptors::StreamDescriptor& desc) {
+    for (const auto& field : fields) {
+        auto new_field = desc.add_fields();
+        new_field->MergeFrom(field_proto(field.type().data_type(), field.type().dimension(), field.name()));
+    }
+}
+
+
+}

cpp/arcticdb/entity/field_collection_proto.hpp

@@ -0,0 +1,19 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+
+#pragma once
+
+#include <arcticdb/entity/field_collection.hpp>
+#include <arcticdb/entity/types_proto.hpp>
+
+namespace arcticdb {
+
+FieldCollection fields_from_proto(const arcticdb::proto::descriptors::StreamDescriptor& desc);
+void proto_from_fields(const FieldCollection& fields, arcticdb::proto::descriptors::StreamDescriptor& desc);
+
+
+} //namespace arcticdb

cpp/arcticdb/entity/serialized_key.hpp

@@ -17,6 +17,7 @@
 #include <arcticdb/stream/index.hpp>
 #include <arcticdb/util/buffer.hpp>
 #include <arcticdb/util/cursored_buffer.hpp>
+#include <arcticdb/util/string_utils.hpp>
 
 namespace arcticdb::entity {
 

cpp/arcticdb/entity/stream_descriptor.hpp

@@ -7,9 +7,14 @@
 
 #pragma once
 
-#include <arcticdb/entity/field_collection.hpp>
+
+#include <proto/arcticc/pb2/proto/descriptors.pb.h>
+#include <google/protobuf/util/message_differencer.h>
 #include <folly/gen/Base.h>
 
+#include <arcticdb/entity/field_collection_proto.hpp>
+#include <arcticdb/util/variant.hpp>
+
 namespace arcticdb::entity {
 
 struct StreamDescriptor {