Added some benchmarks

benchmarks/dataframe_read_large_file.cc

@@ -0,0 +1,97 @@
+/*
+Copyright (c) 2019-2026, Hossein Moein
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+* Neither the name of Hossein Moein and/or the DataFrame nor the
+  names of its contributors may be used to endorse or promote products
+  derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL Hossein Moein BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <DataFrame/DataFrame.h>
+
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <ctime>
+#include <iostream>
+
+using namespace hmdf;
+using namespace std::chrono;
+
+// A DataFrame with long type index
+//
+using MyDataFrame = StdDataFrame<long>;
+
+// -----------------------------------------------------------------------------
+
+int main(int, char *[]) {
+
+    /*
+    std::ofstream   stream;
+
+    stream.open("Large_File.csv");
+
+    stream << "INDEX:200000000:<long>,COL1:200000000:<uint>,COL2:200000000:<int>,COL3:200000000:<int>,COL4:200000000:<int>,COL5:200000000:<ulong>,COL6:200000000:<int>,COL7:200000000:<ulong>\n";
+
+    std::srand(std::time(nullptr));
+    for (std::size_t i = 0; i < 200000000; ++i)  {
+        stream << i << ','
+               << std::rand() << ','
+               << std::rand() << ','
+               << std::rand() << ','
+               << std::rand() << ','
+               << std::rand() << ','
+               << std::rand() << ','
+               << std::rand() << '\n';
+    }
+    stream.close();
+    */
+
+    MyDataFrame df;
+
+    const auto  start = std::chrono::high_resolution_clock::now();
+
+    try  {
+        df.read("Large_File.csv", io_format::csv2);
+    }
+    catch (const DataFrameError &ex)  {
+        std::cout << ex.what() << std::endl;
+    }
+
+    const auto  end = std::chrono::high_resolution_clock::now();
+
+    std::cout << "Column Length: "
+              << df.get_column<int>("COL6").size() << '\n';
+    std::cout
+        << "Reading Took: "
+        << double(duration_cast<microseconds>(end - start).count()) / 1000000.0
+        << " seconds\n";
+
+    return (0);
+}
+
+// -----------------------------------------------------------------------------
+
+// Local Variables:
+// mode:C++
+// tab-width:4
+// c-basic-offset:4
+// End:

benchmarks/pandas_read_large_file.py

@@ -0,0 +1,20 @@
+import datetime
+import pandas as pd
+
+# ------------------------------------------------------------------------------
+
+start = datetime.datetime.now()
+df = df = pd.read_csv("Large_File.csv")
+end = datetime.datetime.now()
+
+print(f"Column Length: {len(df['COL6:200000000:<int>'])}")
+print(f"Reading Took: {(end - start).seconds}.{(end - start).microseconds}")
+
+# ------------------------------------------------------------------------------
+
+# Local Variables:
+# mode:Python
+# tab-width:4
+# c-basic-offset:4
+# End:
+

benchmarks/polars_read_large_file.py

@@ -0,0 +1,20 @@
+import datetime
+import polars as pl
+
+# ------------------------------------------------------------------------------
+
+
+start = datetime.datetime.now()
+df = pl.read_csv("Large_File.csv")
+end = datetime.datetime.now()
+
+print(f"Column Length: {df.select(pl.count()).item()}")
+print(f"Reading Took: {(end - start).seconds}.{(end - start).microseconds}")
+
+# ------------------------------------------------------------------------------
+
+# Local Variables:
+# mode:Python
+# tab-width:4
+# c-basic-offset:4
+# End:

include/DataFrame/Utils/Utils.h

@@ -347,7 +347,7 @@ shift_left(V &vec, std::size_t n)  {
 
 // ----------------------------------------------------------------------------
 
-template<typename STR, std::size_t SIZ = 64 * 1024>
+template<typename STR, std::size_t SIZ = 256 * 1024>
 struct  IOStreamOpti  {
 
     IOStreamOpti (STR &stream, const char *file_name, bool binary = false)

src/CommonMakefile.mk

@@ -26,6 +26,7 @@ SRCS = ../test/dataframe_tester.cc \
        ../test/gen_rand_tester.cc \
        ../test/allocator_tester.cc \
        ../test/linkedin_benchmark.cc \
+       ../test/dataframe_read_large_file.cc \
        Utils/DateTime.cc
 
 HEADERS = $(LOCAL_INCLUDE_DIR)/DataFrame/Vectors/HeteroVector.h \
@@ -93,6 +94,7 @@ TARGETS += $(TARGET_LIB) \
            $(LOCAL_BIN_DIR)/vectors_tester \
            $(LOCAL_BIN_DIR)/allocator_tester \
            $(LOCAL_BIN_DIR)/linkedin_benchmark \
+           $(LOCAL_BIN_DIR)/dataframe_read_large_file \
            $(LOCAL_BIN_DIR)/vector_ptr_view_tester \
            $(LOCAL_BIN_DIR)/meta_prog_tester \
            $(LOCAL_BIN_DIR)/date_time_tester \
@@ -197,6 +199,10 @@ LINKEDIN_BENCHMARK_OBJ = $(LOCAL_OBJ_DIR)/linkedin_benchmark.o
 $(LOCAL_BIN_DIR)/linkedin_benchmark: $(TARGET_LIB) $(LINKEDIN_BENCHMARK_OBJ)
 	$(CXX) -o $@ $(LINKEDIN_BENCHMARK_OBJ) $(LIBS)
 
+DATAFRAME_READ_LARGE_FILE_OBJ = $(LOCAL_OBJ_DIR)/dataframe_read_large_file.o
+$(LOCAL_BIN_DIR)/dataframe_read_large_file: $(TARGET_LIB) $(DATAFRAME_READ_LARGE_FILE_OBJ)
+	$(CXX) -o $@ $(DATAFRAME_READ_LARGE_FILE_OBJ) $(LIBS)
+
 VECTOR_PTR_VIEW_TESTER_OBJ = $(LOCAL_OBJ_DIR)/vector_ptr_view_tester.o
 $(LOCAL_BIN_DIR)/vector_ptr_view_tester: $(TARGET_LIB) $(VECTOR_PTR_VIEW_TESTER_OBJ)
 	$(CXX) -o $@ $(VECTOR_PTR_VIEW_TESTER_OBJ) $(LIBS)
@@ -223,20 +229,23 @@ clean:
           $(DATE_TIME_TESTER_OBJ) $(VECTOR_PTR_VIEW_TESTER_OBJ) \
           $(GEN_RAND_TESTER_OBJ) $(META_PROG_OBJ) \
           $(DATAFRAME_PERFORMANCE_OBJ) $(DATAFRAME_TESTER_OBJ_2) \
-          $(DATAFRAME_TESTER_OBJ_3) $(DATAFRAME_TESTER_OBJ_4) $(HELLO_WORLD_OBJ) \
-          $(DATAFRAME_PERFORMANCE_2_OBJ) $(DATAFRAME_THREAD_SAFTY_OBJ) \
-          $(DATAFRAME_TESTER_SCHEMA_OBJ) $(ALLOCATOR_TESTER_OBJ) \
-          $(LINKEDIN_BENCHMARK_OBJ)
+          $(DATAFRAME_TESTER_OBJ_3) $(DATAFRAME_TESTER_OBJ_4) \
+          $(HELLO_WORLD_OBJ) $(DATAFRAME_PERFORMANCE_2_OBJ) \
+          $(DATAFRAME_THREAD_SAFTY_OBJ) $(DATAFRAME_TESTER_SCHEMA_OBJ) \
+          $(ALLOCATOR_TESTER_OBJ) $(LINKEDIN_BENCHMARK_OBJ) \
+          $(DATAFRAME_READ_LARGE_FILE_OBJ)
 
 clobber:
 	rm -f $(LIB_OBJS) $(TARGETS) $(DATAFRAME_TESTER_OBJ) $(VECTORS_TESTER_OBJ) \
           $(DATE_TIME_TESTER_OBJ) $(VECTOR_PTR_VIEW_TESTER_OBJ) \
           $(GEN_RAND_TESTER_OBJ) $(DATAFRAME_PERFORMACE_OBJ) \
           $(DATAFRAME_TESTER_OBJ_2) $(DATAFRAME_THREAD_SAFTY_OBJ) \
-          $(DATAFRAME_TESTER_OBJ_3) $(DATAFRAME_TESTER_OBJ_4) $(HELLO_WORLD_OBJ) \
-          $(DATAFRAME_TESTER_SCHEMA_OBJ) $(ALLOCATOR_TESTER_OBJ) \
-          $(DATAFRAME_PERFORMANCE_OBJ) $(DATAFRAME_PERFORMANCE_2_OBJ) \
-          $(META_PROG_OBJ) $(LINKEDIN_BENCHMARK_OBJ)
+          $(DATAFRAME_TESTER_OBJ_3) $(DATAFRAME_TESTER_OBJ_4) \
+          $(HELLO_WORLD_OBJ) $(DATAFRAME_TESTER_SCHEMA_OBJ) \
+          $(ALLOCATOR_TESTER_OBJ) $(DATAFRAME_PERFORMANCE_OBJ) \
+          $(DATAFRAME_PERFORMANCE_2_OBJ) \
+          $(META_PROG_OBJ) $(LINKEDIN_BENCHMARK_OBJ) \
+          $(DATAFRAME_READ_LARGE_FILE_OBJ)
 
 install_lib:
 	cp -pf $(TARGET_LIB) $(PROJECT_LIB_DIR)/.