From aa71a8399fbdcd1b71463cfe96f5a428bf3f9f2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=99=93=E7=AE=80?= <lxj193371@alibaba-inc.com>
Date: Wed, 11 Sep 2024 14:41:30 +0800
Subject: [PATCH 01/15] vc init

---
 examples/analytical_apps/flags.cc             |   2 +
 examples/analytical_apps/flags.h              |   2 +
 .../analytical_apps/pagerank/pagerank_vc.h    | 126 +++++++++
 .../pagerank/pagerank_vc_context.h            |  68 +++++
 examples/analytical_apps/run_app.cc           |   3 +
 examples/analytical_apps/run_app_vc.h         |  81 ++++++
 grape/app/vc_app_base.h                       |  58 +++++
 grape/fragment/basic_efile_fragment_loader.h  |   1 -
 grape/fragment/basic_vc_fragment_loader.h     | 115 +++++++++
 grape/fragment/immutable_edgecut_fragment.h   |   2 +-
 grape/fragment/immutable_vertexcut_fragment.h | 170 ++++++++++++
 grape/fragment/loader.h                       |  13 +
 grape/fragment/vc_fragment_loader.h           | 119 +++++++++
 grape/grape.h                                 |   3 +
 grape/parallel/vc_message_manager.h           | 244 ++++++++++++++++++
 grape/utils/vertex_array.h                    |  36 +++
 grape/vertex_map/partitioner.h                |  76 ++++++
 grape/worker/vc_worker.h                      | 117 +++++++++
 18 files changed, 1234 insertions(+), 2 deletions(-)
 create mode 100644 examples/analytical_apps/pagerank/pagerank_vc.h
 create mode 100644 examples/analytical_apps/pagerank/pagerank_vc_context.h
 create mode 100644 examples/analytical_apps/run_app_vc.h
 create mode 100644 grape/app/vc_app_base.h
 create mode 100644 grape/fragment/basic_vc_fragment_loader.h
 create mode 100644 grape/fragment/immutable_vertexcut_fragment.h
 create mode 100644 grape/fragment/vc_fragment_loader.h
 create mode 100644 grape/parallel/vc_message_manager.h
 create mode 100644 grape/worker/vc_worker.h

diff --git a/examples/analytical_apps/flags.cc b/examples/analytical_apps/flags.cc
index e2872d08..e6d9a93c 100644
--- a/examples/analytical_apps/flags.cc
+++ b/examples/analytical_apps/flags.cc
@@ -61,6 +61,8 @@ DEFINE_string(serialization_prefix, "",
 DEFINE_int32(app_concurrency, -1, "concurrency of application");
 DEFINE_int32(load_concurrency, 1, "concurrency of loading graph");
 
+DEFINE_bool(vc, false, "whether to use vertex-cut storage.");
+
 DEFINE_string(lb, "cta",
               "Load balancing policy, these options can be used: "
               " none, cta, cm, wm, strict");
diff --git a/examples/analytical_apps/flags.h b/examples/analytical_apps/flags.h
index 59a55e6e..6f60d0b7 100644
--- a/examples/analytical_apps/flags.h
+++ b/examples/analytical_apps/flags.h
@@ -53,5 +53,7 @@ DECLARE_string(serialization_prefix);
 DECLARE_int32(app_concurrency);
 DECLARE_int32(load_concurrency);
 
+DECLARE_bool(vc);
+
 DECLARE_string(lb);
 #endif  // EXAMPLES_ANALYTICAL_APPS_FLAGS_H_
diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
new file mode 100644
index 00000000..b5520993
--- /dev/null
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -0,0 +1,126 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_H_
+#define EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_H_
+
+#include <grape/grape.h>
+
+#include "pagerank/pagerank_vc_context.h"
+
+namespace grape {
+
+template <typename T>
+struct NumericSum {
+  static T init() { return 0; }
+
+  static void aggregate(T& a, const T& b) { a += b; }
+};
+
+template <typename FRAG_T>
+class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
+                   public ParallelEngine,
+                   public Communicator {
+  using vertex_t = Vertex<typename FRAG_T::oid_t>;
+
+ public:
+  INSTALL_VC_WORKER(PageRankVC<FRAG_T>, PageRankVCContext<FRAG_T>, FRAG_T)
+
+  void PEval(const fragment_t& frag, context_t& ctx,
+             message_manager_t& messages) {
+    if (ctx.max_round <= 0) {
+      return;
+    }
+
+    ctx.step = 0;
+    ctx.graph_vnum = frag.GetTotalVerticesNum();
+
+    {
+      typename fragment_t::template both_vertex_array_t<int> degree(
+          frag.GetVertices(), 0);
+      for (auto& e : frag.GetEdges()) {
+        ++degree[vertex_t(e.src)];
+        ++degree[vertex_t(e.dst)];
+      }
+
+      messages.GatherMasterVertices<fragment_t, int, NumericSum<int>>(
+          frag, degree, ctx.master_degree);
+    }
+
+    int64_t dangling_vnum_local = 0;
+    double p = 1.0 / ctx.graph_vnum;
+    for (auto v : frag.GetMasterVertices()) {
+      if (ctx.master_degree[v] == 0) {
+        ++dangling_vnum_local;
+        ctx.master_result[v] = p;
+      } else {
+        ctx.master_result[v] = p / ctx.master_degree[v];
+      }
+    }
+
+    Sum(dangling_vnum_local, ctx.total_dangling_vnum);
+    ctx.dangling_sum = p * ctx.total_dangling_vnum;
+
+    messages.ScatterMasterVertices<fragment_t, double>(frag, ctx.master_result,
+                                                       ctx.curr_result);
+  }
+
+  void IncEval(const fragment_t& frag, context_t& ctx,
+               message_manager_t& messages) {
+    ++ctx.step;
+
+    double base = (1.0 - ctx.delta) / ctx.graph_vnum +
+                  ctx.delta * ctx.dangling_sum / ctx.graph_vnum;
+    ctx.dangling_sum = base * ctx.total_dangling_vnum;
+
+    if (ctx.step != ctx.max_round) {
+      for (auto& e : frag.GetEdges()) {
+        ctx.next_result[vertex_t(e.dst)] += ctx.curr_result[vertex_t(e.src)];
+        ctx.next_result[vertex_t(e.src)] += ctx.curr_result[vertex_t(e.dst)];
+      }
+
+      messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
+          frag, ctx.next_result, ctx.master_result);
+      for (auto v : frag.GetMasterVertices()) {
+        if (ctx.master_degree[v] > 0) {
+          ctx.master_result[v] =
+              (base + ctx.delta * ctx.master_result[v]) / ctx.master_degree[v];
+        } else {
+          ctx.master_result[v] = base;
+        }
+      }
+
+      messages.ScatterMasterVertices<fragment_t, double>(
+          frag, ctx.master_result, ctx.curr_result);
+    } else {
+      for (auto& e : frag.GetEdges()) {
+        ctx.next_result[vertex_t(e.dst)] += ctx.curr_result[vertex_t(e.src)];
+        ctx.next_result[vertex_t(e.src)] += ctx.curr_result[vertex_t(e.dst)];
+      }
+
+      messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
+          frag, ctx.next_result, ctx.master_result);
+      for (auto v : frag.GetMasterVertices()) {
+        ctx.master_result[v] = ctx.master_result[v] * ctx.delta + base;
+      }
+
+      messages.ForceTerminate();
+    }
+  }
+};
+
+}  // namespace grape
+
+#endif  // EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_H_
diff --git a/examples/analytical_apps/pagerank/pagerank_vc_context.h b/examples/analytical_apps/pagerank/pagerank_vc_context.h
new file mode 100644
index 00000000..41ad6737
--- /dev/null
+++ b/examples/analytical_apps/pagerank/pagerank_vc_context.h
@@ -0,0 +1,68 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
+#define EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
+
+namespace grape {
+
+template <typename FRAG_T>
+class PageRankVCContext {
+  using oid_t = typename FRAG_T::oid_t;
+
+ public:
+  const FRAG_T& fragment() { return fragment_; }
+
+  explicit PageRankVCContext(const FRAG_T& fragment) : fragment_(fragment) {
+    curr_result.Init(fragment.GetVertices());
+    next_result.Init(fragment.GetVertices());
+    master_result.Init(fragment.GetMasterVertices());
+    master_degree.Init(fragment.GetMasterVertices());
+  }
+
+  void Init(VCMessageManager& messages, double delta, int max_round) {
+    this->delta = delta;
+    this->max_round = max_round;
+    step = 0;
+  }
+
+  void Output(std::ostream& os) {
+    auto& frag = this->fragment();
+    auto master_vertices = frag.GetMasterVertices();
+    for (auto v : master_vertices) {
+      os << v.GetValue() << " " << std::scientific << std::setprecision(15)
+         << master_result[v] << std::endl;
+    }
+  }
+
+  typename FRAG_T::template both_vertex_array_t<double> curr_result;
+  typename FRAG_T::template both_vertex_array_t<double> next_result;
+  typename FRAG_T::template vertex_array_t<double> master_result;
+  typename FRAG_T::template vertex_array_t<int> master_degree;
+
+  int64_t total_dangling_vnum = 0;
+  int64_t graph_vnum;
+  int step = 0;
+  int max_round = 0;
+  double delta = 0;
+
+  double dangling_sum = 0.0;
+
+  const FRAG_T& fragment_;
+};
+
+}  // namespace grape
+
+#endif  // EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
diff --git a/examples/analytical_apps/run_app.cc b/examples/analytical_apps/run_app.cc
index 7bdaa19d..7af79bd8 100644
--- a/examples/analytical_apps/run_app.cc
+++ b/examples/analytical_apps/run_app.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <glog/logging.h>
 
 #include "run_app_opt.h"
+#include "run_app_vc.h"
 
 int main(int argc, char* argv[]) {
   FLAGS_stderrthreshold = 0;
@@ -41,6 +42,8 @@ int main(int argc, char* argv[]) {
   std::string name = FLAGS_application;
   if (FLAGS_opt) {
     grape::RunOpt();
+  } else if (FLAGS_vc) {
+    grape::RunVC();
   } else {
     if (name.find("sssp") != std::string::npos) {
       grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
new file mode 100644
index 00000000..77439a7b
--- /dev/null
+++ b/examples/analytical_apps/run_app_vc.h
@@ -0,0 +1,81 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef EXAMPLES_ANALYTICAL_APPS_RUN_APP_VC_H_
+#define EXAMPLES_ANALYTICAL_APPS_RUN_APP_VC_H_
+
+#include "grape/fragment/immutable_vertexcut_fragment.h"
+#include "grape/fragment/loader.h"
+#include "pagerank/pagerank_vc.h"
+#include "run_app.h"
+
+namespace grape {
+
+template <template <class> class APP_T, typename... Args>
+void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
+                      const ParallelEngineSpec& spec, Args... args) {
+  timer_next("load graph");
+  LoadGraphSpec graph_spec = DefaultLoadGraphSpec();
+  if (FLAGS_deserialize) {
+    graph_spec.set_deserialize(true, FLAGS_serialization_prefix);
+  } else if (FLAGS_serialize) {
+    graph_spec.set_serialize(true, FLAGS_serialization_prefix);
+  }
+
+  using FRAG_T = ImmutableVertexcutFragment<int64_t, uint32_t, grape::EmptyType,
+                                            grape::EmptyType>;
+  std::shared_ptr<FRAG_T> fragment = LoadVertexcutGraph<FRAG_T>(
+      FLAGS_efile, FLAGS_vertex_num, comm_spec, graph_spec);
+  using AppType = APP_T<FRAG_T>;
+  auto app = std::make_shared<AppType>();
+  DoQuery<FRAG_T, AppType, Args...>(fragment, app, comm_spec, spec, out_prefix,
+                                    args...);
+}
+
+void RunVC() {
+  CommSpec comm_spec;
+  comm_spec.Init(MPI_COMM_WORLD);
+
+  bool is_coordinator = comm_spec.worker_id() == kCoordinatorRank;
+  timer_start(is_coordinator);
+
+  // FIXME: no barrier apps. more manager? or use a dynamic-cast.
+  std::string out_prefix = FLAGS_out_prefix;
+  auto spec = MultiProcessSpec(comm_spec, __AFFINITY__);
+  if (FLAGS_app_concurrency != -1) {
+    spec.thread_num = FLAGS_app_concurrency;
+    if (__AFFINITY__) {
+      if (spec.cpu_list.size() >= spec.thread_num) {
+        spec.cpu_list.resize(spec.thread_num);
+      } else {
+        uint32_t num_to_append = spec.thread_num - spec.cpu_list.size();
+        for (uint32_t i = 0; i < num_to_append; ++i) {
+          spec.cpu_list.push_back(spec.cpu_list[i]);
+        }
+      }
+    }
+  }
+  std::string name = FLAGS_application;
+  if (name == "pagerank") {
+    CreateAndQueryVC<PageRankVC>(comm_spec, out_prefix, spec, FLAGS_pr_d,
+                                 FLAGS_pr_mr);
+  } else {
+    LOG(FATAL) << "No avaiable application named [" << name << "].";
+  }
+}
+
+}  // namespace grape
+
+#endif  // EXAMPLES_ANALYTICAL_APPS_RUN_APP_VC_H_
diff --git a/grape/app/vc_app_base.h b/grape/app/vc_app_base.h
new file mode 100644
index 00000000..a52b92a1
--- /dev/null
+++ b/grape/app/vc_app_base.h
@@ -0,0 +1,58 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_APP_VC_APP_BASE_H_
+#define GRAPE_APP_VC_APP_BASE_H_
+
+#include <memory>
+
+#include "grape/types.h"
+#include "grape/worker/vc_worker.h"
+
+namespace grape {
+
+class VCMessageManager;
+
+template <typename FRAG_T, typename CONTEXT_T,
+          typename MESSAGE_MANAGER_T = VCMessageManager>
+class VCAppBase {
+ public:
+  using message_manager_t = MESSAGE_MANAGER_T;
+
+  VCAppBase() = default;
+  virtual ~VCAppBase() = default;
+
+  virtual void PEval(const FRAG_T& graph, CONTEXT_T& context,
+                     message_manager_t& messages) = 0;
+
+  virtual void IncEval(const FRAG_T& graph, CONTEXT_T& context,
+                       message_manager_t& messages) = 0;
+};
+
+#define INSTALL_VC_WORKER(APP_T, CONTEXT_T, FRAG_T)               \
+ public:                                                          \
+  using fragment_t = FRAG_T;                                      \
+  using context_t = CONTEXT_T;                                    \
+  using message_manager_t = grape::VCMessageManager;              \
+  using worker_t = grape::VCWorker<APP_T, message_manager_t>;     \
+  virtual ~APP_T() {}                                             \
+  static std::shared_ptr<worker_t> CreateWorker(                  \
+      std::shared_ptr<APP_T> app, std::shared_ptr<FRAG_T> frag) { \
+    return std::shared_ptr<worker_t>(new worker_t(app, frag));    \
+  }
+
+}  // namespace grape
+
+#endif  // GRAPE_APP_VC_APP_BASE_H_
diff --git a/grape/fragment/basic_efile_fragment_loader.h b/grape/fragment/basic_efile_fragment_loader.h
index c7ab6c6a..8bdb54ee 100644
--- a/grape/fragment/basic_efile_fragment_loader.h
+++ b/grape/fragment/basic_efile_fragment_loader.h
@@ -137,7 +137,6 @@ class BasicEFileFragmentLoader : public BasicFragmentLoaderBase<FRAG_T> {
               make_index_sequence<2>{});
         }
       } else {
-        std::atomic<size_t> idx(0);
         std::vector<std::vector<internal_oid_t>> vertices(concurrency_);
         std::vector<std::vector<internal_oid_t>> vertices_mat(concurrency_ *
                                                               concurrency_);
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
new file mode 100644
index 00000000..a51e3b45
--- /dev/null
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -0,0 +1,115 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
+#define GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
+
+namespace grape {
+
+template <typename FRAG_T>
+class BasicVCFragmentLoader {
+  using fragment_t = FRAG_T;
+  using oid_t = typename fragment_t::oid_t;
+  using vid_t = typename fragment_t::vid_t;
+  using vdata_t = typename fragment_t::vdata_t;
+  using edata_t = typename fragment_t::edata_t;
+  using edge_t = typename fragment_t::edge_t;
+  using vertices_t = typename fragment_t::vertices_t;
+
+ public:
+  explicit BasicVCFragmentLoader(const CommSpec& comm_spec, int64_t vnum)
+      : comm_spec_(comm_spec), partitioner_(comm_spec.fnum(), vnum) {
+    comm_spec_.Dup();
+
+    edges_to_frag_.resize(comm_spec_.fnum());
+    for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+      int worker_id = comm_spec_.FragToWorker(fid);
+      edges_to_frag_[fid].Init(comm_spec_.comm(), edge_tag, 4096000);
+      edges_to_frag_[fid].SetDestination(worker_id, fid);
+      if (worker_id == comm_spec_.worker_id()) {
+        edges_to_frag_[fid].DisableComm();
+      }
+    }
+    edge_recv_thread_ =
+        std::thread(&BasicVCFragmentLoader::edgeRecvRoutine, this);
+    recv_thread_running_ = true;
+  }
+
+  void AddEdge(const oid_t& src, const oid_t& dst, const edata_t& data) {
+    fid_t fid = partitioner_.get_edge_partition(src, dst);
+    edges_to_frag_[fid].Emplace(src, dst, data);
+  }
+
+  void ConstructFragment(std::shared_ptr<fragment_t>& fragment) {
+    for (auto& e : edges_to_frag_) {
+      e.Flush();
+    }
+    edge_recv_thread_.join();
+    recv_thread_running_ = false;
+
+    MPI_Barrier(comm_spec_.comm());
+    got_edges_.emplace_back(
+        std::move(edges_to_frag_[comm_spec_.fid()].buffers()));
+    edges_to_frag_[comm_spec_.fid()].Clear();
+
+    std::vector<edge_t> edges;
+    size_t edge_num = 0;
+    for (auto& buffer : got_edges_) {
+      edge_num += buffer.size();
+    }
+    edges.reserve(edge_num);
+    for (auto& buffer : got_edges_) {
+      foreach_rval(buffer, [&edges](oid_t&& src, oid_t&& dst, edata_t&& data) {
+        edges.emplace_back(src, dst, data);
+      });
+    }
+
+    fragment.reset(new fragment_t());
+    fragment->Init(comm_spec_, vnum_, std::move(edges));
+  }
+
+ private:
+  void edgeRecvRoutine() {
+    ShuffleIn<oid_t, oid_t, edata_t> data_in;
+    data_in.Init(comm_spec_.fnum(), comm_spec_.comm(), edge_tag);
+    fid_t dst_fid;
+    int src_worker_id;
+    while (!data_in.Finished()) {
+      src_worker_id = data_in.Recv(dst_fid);
+      if (src_worker_id == -1) {
+        break;
+      }
+      CHECK_EQ(dst_fid, comm_spec_.fid());
+      got_edges_.emplace_back(std::move(data_in.buffers()));
+      data_in.Clear();
+    }
+  }
+
+  CommSpec comm_spec_;
+  int64_t vnum_;
+  VCPartitioner<int64_t> partitioner_;
+
+  std::vector<ShuffleBufferTuple<oid_t, oid_t, edata_t>> got_edges_;
+
+  std::vector<ShuffleOut<oid_t, oid_t, edata_t>> edges_to_frag_;
+  std::thread edge_recv_thread_;
+  bool recv_thread_running_;
+
+  static constexpr int edge_tag = 6;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
diff --git a/grape/fragment/immutable_edgecut_fragment.h b/grape/fragment/immutable_edgecut_fragment.h
index 5482440c..1ce5a158 100644
--- a/grape/fragment/immutable_edgecut_fragment.h
+++ b/grape/fragment/immutable_edgecut_fragment.h
@@ -424,7 +424,7 @@ class ImmutableEdgecutFragment
       std::vector<std::thread> threads;
       for (int i = 0; i < concurrency; ++i) {
         threads.emplace_back(
-            [&, this](int tid) {
+            [&](int tid) {
               size_t batch = (edges.size() + concurrency - 1) / concurrency;
               size_t begin = std::min(batch * tid, edges.size());
               size_t end = std::min(begin + batch, edges.size());
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
new file mode 100644
index 00000000..9d74bc4f
--- /dev/null
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -0,0 +1,170 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_FRAGMENT_IMMUTABLE_VERTEXCUT_FRAGMENT_H_
+#define GRAPE_FRAGMENT_IMMUTABLE_VERTEXCUT_FRAGMENT_H_
+
+#include <glog/logging.h>
+
+namespace grape {
+
+template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T>
+class ImmutableVertexcutFragment {};
+
+template <typename VID_T>
+class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
+ public:
+  using oid_t = int64_t;
+  using vid_t = VID_T;
+  using edata_t = EmptyType;
+  using vdata_t = EmptyType;
+  using vertices_t = VertexRange<oid_t>;
+  using both_vertices_t = DualVertexRange<oid_t>;
+  using edge_t = Edge<oid_t, edata_t>;
+
+  template <typename T>
+  using vertex_array_t = VertexArray<vertices_t, T>;
+
+  template <typename T>
+  using both_vertex_array_t = VertexArray<both_vertices_t, T>;
+
+  ImmutableVertexcutFragment() = default;
+  ~ImmutableVertexcutFragment() = default;
+
+  void Init(const CommSpec& comm_spec, int64_t vnum,
+            std::vector<edge_t>&& edges) {
+    fid_ = comm_spec.fid();
+    fnum_ = comm_spec.fnum();
+
+    partitioner_.init(comm_spec.fnum(), vnum);
+    src_vertices_ = partitioner_.get_src_vertices(fid_);
+    dst_vertices_ = partitioner_.get_dst_vertices(fid_);
+    master_vertices_ = partitioner_.get_master_vertices(fid_);
+
+    int64_t src_begin = src_vertices_.begin_value();
+    int64_t src_end = src_vertices_.end_value();
+    int64_t dst_begin = dst_vertices_.begin_value();
+    int64_t dst_end = dst_vertices_.end_value();
+
+    if (src_end < dst_begin) {
+      vertices_ = both_vertices_t(src_begin, src_end, dst_begin, dst_end);
+    } else if (dst_end < src_begin) {
+      vertices_ = both_vertices_t(dst_begin, dst_end, src_begin, src_end);
+    } else {
+      vertices_ = both_vertices_t(0, 0, std::min(src_begin, dst_begin),
+                                  std::max(src_end, dst_end));
+    }
+
+    edges_ = std::move(edges);
+  }
+
+  const vertices_t& GetSourceVertices() const { return src_vertices_; }
+  const vertices_t& GetDestinationVertices() const { return dst_vertices_; }
+  const both_vertices_t& GetVertices() const { return vertices_; }
+
+  const vertices_t& GetMasterVertices() const { return master_vertices_; }
+
+  const std::vector<edge_t>& GetEdges() const { return edges_; }
+
+  fid_t fid() const { return fid_; }
+  fid_t fnum() const { return fnum_; }
+
+  template <typename IOADAPTOR_T>
+  void Serialize(const std::string& prefix) {
+    char fbuf[1024];
+    snprintf(fbuf, sizeof(fbuf), kSerializationFilenameFormat, prefix.c_str(),
+             fid_);
+
+    auto io_adaptor =
+        std::unique_ptr<IOADAPTOR_T>(new IOADAPTOR_T(std::string(fbuf)));
+    io_adaptor->Open("wb");
+
+    InArchive arc;
+    arc << fid_ << fnum_ << edges_.size();
+    arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_;
+    CHECK(io_adaptor->WriteArchive(arc));
+    arc.Clear();
+
+    partitioner_.serialize(io_adaptor);
+
+    if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
+      LOG(INFO) << "is pod";
+      if (!edges_.empty()) {
+        io_adaptor->Write(edges_.data(), edges_.size() * sizeof(edge_t));
+      }
+    } else {
+      LOG(FATAL) << "is not pod";
+
+      arc << edges_;
+      if (!arc.Empty()) {
+        CHECK(io_adaptor->WriteArchive(arc));
+        arc.Clear();
+      }
+    }
+  }
+
+  template <typename IOADAPTOR_T>
+  void Deserialize(const CommSpec& comm_spec, const std::string& prefix) {
+    char fbuf[1024];
+    snprintf(fbuf, sizeof(fbuf), kSerializationFilenameFormat, prefix.c_str(),
+             comm_spec.fid());
+    auto io_adaptor =
+        std::unique_ptr<IOADAPTOR_T>(new IOADAPTOR_T(std::string(fbuf)));
+    io_adaptor->Open();
+
+    OutArchive arc;
+    size_t edge_num;
+    CHECK(io_adaptor->ReadArchive(arc));
+    arc >> fid_ >> fnum_ >> edge_num;
+    CHECK_EQ(fid_, comm_spec.fid());
+    CHECK_EQ(fnum_, comm_spec.fnum());
+    arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_;
+
+    partitioner_.deserialize(io_adaptor);
+
+    if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
+      LOG(INFO) << "is pod";
+      edges_.resize(edge_num);
+      if (edge_num > 0) {
+        CHECK(io_adaptor->Read(edges_.data(), edge_num * sizeof(edge_t)));
+      }
+    } else {
+      LOG(FATAL) << "is not pod";
+      arc.Clear();
+      CHECK(io_adaptor->ReadArchive(arc));
+      arc >> edges_;
+    }
+  }
+
+  int64_t GetTotalVerticesNum() const {
+    return partitioner_.get_total_vertices_num();
+  }
+
+  const VCPartitioner<int64_t>& GetPartitioner() const { return partitioner_; }
+
+ private:
+  fid_t fid_;
+  fid_t fnum_;
+  VCPartitioner<int64_t> partitioner_;
+  vertices_t src_vertices_;
+  vertices_t dst_vertices_;
+  both_vertices_t vertices_;
+  vertices_t master_vertices_;
+  std::vector<edge_t> edges_;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_FRAGMENT_IMMUTABLE_VERTEXCUT_FRAGMENT_H_
diff --git a/grape/fragment/loader.h b/grape/fragment/loader.h
index d2abe7af..7b9048fa 100644
--- a/grape/fragment/loader.h
+++ b/grape/fragment/loader.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "grape/fragment/ev_fragment_loader.h"
 #include "grape/fragment/ev_fragment_mutator.h"
+#include "grape/fragment/vc_fragment_loader.h"
 #include "grape/io/local_io_adaptor.h"
 
 namespace grape {
@@ -66,6 +67,18 @@ static std::shared_ptr<FRAG_T> LoadGraphAndMutate(
   return mutator.MutateFragment(delta_efile, delta_vfile, ret, spec.directed);
 }
 
+template <typename FRAG_T, typename IOADAPTOR_T = LocalIOAdaptor,
+          typename LINE_PARSER_T =
+              TSVLineParser<typename FRAG_T::oid_t, typename FRAG_T::vdata_t,
+                            typename FRAG_T::edata_t>>
+static std::shared_ptr<FRAG_T> LoadVertexcutGraph(
+    const std::string& efile, int64_t vnum, const CommSpec& comm_spec,
+    const LoadGraphSpec& spec = DefaultLoadGraphSpec()) {
+  std::unique_ptr<VCFragmentLoader<FRAG_T, IOADAPTOR_T, LINE_PARSER_T>> loader(
+      new VCFragmentLoader<FRAG_T, IOADAPTOR_T, LINE_PARSER_T>(comm_spec));
+  return loader->LoadFragment(vnum, efile, spec);
+}
+
 }  // namespace grape
 
 #endif  // GRAPE_FRAGMENT_LOADER_H_
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
new file mode 100644
index 00000000..146dcf76
--- /dev/null
+++ b/grape/fragment/vc_fragment_loader.h
@@ -0,0 +1,119 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_FRAGMENT_VC_FRAGMENT_LOADER_H_
+#define GRAPE_FRAGMENT_VC_FRAGMENT_LOADER_H_
+
+#include "grape/fragment/basic_vc_fragment_loader.h"
+
+namespace grape {
+
+template <typename FRAG_T, typename IOADAPTOR_T = LocalIOAdaptor,
+          typename LINE_PARSER_T =
+              TSVLineParser<typename FRAG_T::oid_t, typename FRAG_T::vdata_t,
+                            typename FRAG_T::edata_t>>
+class VCFragmentLoader {
+  using fragment_t = FRAG_T;
+  using oid_t = typename fragment_t::oid_t;
+  using vid_t = typename fragment_t::vid_t;
+  using vdata_t = typename fragment_t::vdata_t;
+  using edata_t = typename fragment_t::edata_t;
+
+  using vertices_t = typename fragment_t::vertices_t;
+
+  using io_adaptor_t = IOADAPTOR_T;
+  using line_parser_t = LINE_PARSER_T;
+
+ public:
+  explicit VCFragmentLoader(const CommSpec& comm_spec)
+      : comm_spec_(comm_spec), basic_fragment_loader_(nullptr) {}
+  ~VCFragmentLoader() = default;
+
+  std::shared_ptr<fragment_t> LoadFragment(int64_t vnum,
+                                           const std::string& efile,
+                                           const LoadGraphSpec& spec) {
+    std::shared_ptr<fragment_t> fragment(nullptr);
+    if (spec.deserialize) {
+      fragment = std::make_shared<fragment_t>();
+      fragment->template Deserialize<IOADAPTOR_T>(comm_spec_,
+                                                  spec.deserialization_prefix);
+      return fragment;
+    }
+
+    basic_fragment_loader_ = std::unique_ptr<BasicVCFragmentLoader<fragment_t>>(
+        new BasicVCFragmentLoader<fragment_t>(comm_spec_, vnum));
+
+    auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
+    io_adaptor->SetPartialRead(comm_spec_.worker_id(), comm_spec_.worker_num());
+    io_adaptor->Open();
+
+    std::string line;
+    edata_t e_data;
+    oid_t src, dst;
+
+    double t0 = -grape::GetCurrentTime();
+    size_t lineNo = 0;
+    while (io_adaptor->ReadLine(line)) {
+      ++lineNo;
+      if (lineNo % 1000000 == 0) {
+        VLOG(10) << "[worker-" << comm_spec_.worker_id() << "][efile] "
+                 << lineNo;
+      }
+      if (line.empty() || line[0] == '#')
+        continue;
+
+      try {
+        line_parser_.LineParserForEFile(line, src, dst, e_data);
+      } catch (std::exception& e) {
+        VLOG(1) << e.what();
+        continue;
+      }
+
+      basic_fragment_loader_->AddEdge(src, dst, e_data);
+    }
+    io_adaptor->Close();
+
+    MPI_Barrier(comm_spec_.comm());
+    t0 += grape::GetCurrentTime();
+    if (comm_spec_.worker_id() == 0) {
+      VLOG(1) << "finished reading edges inputs, time: " << t0 << " s";
+    }
+
+    double t2 = -grape::GetCurrentTime();
+    basic_fragment_loader_->ConstructFragment(fragment);
+    MPI_Barrier(comm_spec_.comm());
+    t2 += grape::GetCurrentTime();
+
+    if (comm_spec_.worker_id() == 0) {
+      VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
+    }
+
+    if (spec.serialize) {
+      fragment->template Serialize<IOADAPTOR_T>(spec.serialization_prefix);
+    }
+
+    return fragment;
+  }
+
+ private:
+  CommSpec comm_spec_;
+
+  std::unique_ptr<BasicVCFragmentLoader<fragment_t>> basic_fragment_loader_;
+  line_parser_t line_parser_;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_FRAGMENT_VC_FRAGMENT_LOADER_H_
diff --git a/grape/grape.h b/grape/grape.h
index 08d594df..1a04165d 100644
--- a/grape/grape.h
+++ b/grape/grape.h
@@ -20,15 +20,18 @@ limitations under the License.
 #include "grape/app/batch_shuffle_app_base.h"
 #include "grape/app/context_base.h"
 #include "grape/app/parallel_app_base.h"
+#include "grape/app/vc_app_base.h"
 #include "grape/app/vertex_data_context.h"
 #include "grape/app/void_context.h"
 #include "grape/parallel/auto_parallel_message_manager.h"
 #include "grape/parallel/batch_shuffle_message_manager.h"
 #include "grape/parallel/default_message_manager.h"
 #include "grape/parallel/parallel_message_manager.h"
+#include "grape/parallel/vc_message_manager.h"
 #include "grape/utils/atomic_ops.h"
 #include "grape/utils/vertex_array.h"
 #include "grape/utils/vertex_set.h"
+#include "grape/worker/vc_worker.h"
 #include "grape/worker/worker.h"
 
 #ifdef __CUDACC__
diff --git a/grape/parallel/vc_message_manager.h b/grape/parallel/vc_message_manager.h
new file mode 100644
index 00000000..985529f5
--- /dev/null
+++ b/grape/parallel/vc_message_manager.h
@@ -0,0 +1,244 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
+#define GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
+
+namespace grape {
+
+class VCMessageManager : public MessageManagerBase {
+ public:
+  VCMessageManager() : comm_(NULL_COMM) {}
+  ~VCMessageManager() override {
+    if (ValidComm(comm_)) {
+      MPI_Comm_free(&comm_);
+    }
+  }
+
+  void Init(MPI_Comm comm) override {
+    MPI_Comm_dup(comm, &comm_);
+    comm_spec_.Init(comm_);
+    fid_ = comm_spec_.fid();
+    fnum_ = comm_spec_.fnum();
+
+    force_terminate_ = false;
+    terminate_info_.Init(fnum_);
+
+    round_ = 0;
+
+    sent_size_ = 0;
+  }
+
+  void Start() override {}
+
+  void StartARound() override {}
+
+  void FinishARound() override {}
+
+  bool ToTerminate() override { return force_terminate_; }
+
+  void Finalize() override {}
+
+  void ForceContinue() override {}
+
+  void ForceTerminate(const std::string& terminate_info = "") override {
+    force_terminate_ = true;
+    terminate_info_.info[comm_spec_.fid()] = terminate_info;
+  }
+
+  const TerminateInfo& GetTerminateInfo() const override {
+    return terminate_info_;
+  }
+
+  size_t GetMsgSize() const override { return sent_size_; }
+
+  template <typename GRAPH_T, typename MESSAGE_T, typename AGGR_T>
+  void GatherMasterVertices(
+      const GRAPH_T& frag,
+      const typename GRAPH_T::template both_vertex_array_t<MESSAGE_T>& input,
+      typename GRAPH_T::template vertex_array_t<MESSAGE_T>& output) {
+    if (!std::is_pod<MESSAGE_T>::value) {
+      LOG(FATAL) << "not implemented for non-POD type";
+    }
+
+    const auto& partitioner = frag.GetPartitioner();
+    const auto& input_range = input.GetVertexRange();
+    std::vector<MPI_Request> requests;
+    for (fid_t i = 1; i < fnum_; ++i) {
+      fid_t dst_fid = (fid_ + i) % fnum_;
+      auto target_vertices = partitioner.get_master_vertices(dst_fid);
+      if (target_vertices.IsSubsetOf(input_range)) {
+        const MESSAGE_T* input_data = &input[*target_vertices.begin()];
+        size_t input_size = target_vertices.size();
+        sync_comm::isend_buffer(input_data, input_size, dst_fid, 0, comm_,
+                                requests);
+        sent_size_ += input_size * sizeof(MESSAGE_T);
+      } else {
+        CHECK(!input_range.OverlapWith(target_vertices));
+      }
+    }
+
+    std::vector<std::vector<MESSAGE_T>> recv_buffers;
+    const auto& output_range = output.GetVertexRange();
+    size_t output_size = output_range.size();
+    for (fid_t i = 1; i < fnum_; ++i) {
+      fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
+      auto src_vertices = partitioner.get_src_vertices(src_fid);
+      if (output_range.IsSubsetOf(src_vertices)) {
+        recv_buffers.emplace_back(output_size);
+        sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
+                                src_fid, 0, comm_, requests);
+        continue;
+      } else {
+        CHECK(!output_range.OverlapWith(src_vertices));
+      }
+      auto dst_vertices = partitioner.get_dst_vertices(src_fid);
+      if (output_range.IsSubsetOf(dst_vertices)) {
+        recv_buffers.emplace_back(output_size);
+        sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
+                                src_fid, 0, comm_, requests);
+        continue;
+      } else {
+        CHECK(!output_range.OverlapWith(dst_vertices));
+      }
+    }
+
+    MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+
+    MESSAGE_T* output_data = &output[*output_range.begin()];
+    std::vector<std::thread> threads;
+    int thread_num = std::thread::hardware_concurrency();
+    size_t chunk_size =
+        std::min<size_t>((output_size + thread_num - 1) / thread_num, 4096);
+    std::atomic<size_t> offset(0);
+    for (int i = 0; i < thread_num; ++i) {
+      threads.emplace_back([&]() {
+        if (output_range.IsSubsetOf(input_range)) {
+          const MESSAGE_T* to_self = &input[*output_range.begin()];
+          while (true) {
+            size_t begin = std::min(offset.fetch_add(chunk_size), output_size);
+            size_t end = std::min(begin + chunk_size, output_size);
+            if (begin == end) {
+              break;
+            }
+            while (begin < end) {
+              output_data[begin] = AGGR_T::init();
+              AGGR_T::aggregate(output_data[begin], to_self[begin]);
+              for (auto& recv_buffer : recv_buffers) {
+                AGGR_T::aggregate(output_data[begin], recv_buffer[begin]);
+              }
+              ++begin;
+            }
+          }
+        } else {
+          CHECK(!output_range.OverlapWith(input_range));
+          while (true) {
+            size_t begin = std::min(offset.fetch_add(chunk_size), output_size);
+            size_t end = std::min(begin + chunk_size, output_size);
+            if (begin == end) {
+              break;
+            }
+            while (begin < end) {
+              output_data[begin] = AGGR_T::init();
+              for (auto& recv_buffer : recv_buffers) {
+                AGGR_T::aggregate(output_data[begin], recv_buffer[begin]);
+              }
+              ++begin;
+            }
+          }
+        }
+      });
+    }
+    for (auto& thrd : threads) {
+      thrd.join();
+    }
+  }
+
+  template <typename GRAPH_T, typename MESSAGE_T>
+  void ScatterMasterVertices(
+      const GRAPH_T& frag,
+      const typename GRAPH_T::template vertex_array_t<MESSAGE_T>& input,
+      typename GRAPH_T::template both_vertex_array_t<MESSAGE_T>& output) {
+    if (!std::is_pod<MESSAGE_T>::value) {
+      LOG(FATAL) << "not implemented for non-POD type";
+    }
+    const auto& input_range = input.GetVertexRange();
+    const MESSAGE_T* input_data = &input[*input_range.begin()];
+    size_t input_size = input_range.size();
+
+    const auto& partitioner = frag.GetPartitioner();
+    std::vector<MPI_Request> requests;
+    for (fid_t i = 1; i < fnum_; ++i) {
+      fid_t dst_fid = (fid_ + i) % fnum_;
+      auto dst_vertices = partitioner.get_dst_vertices(dst_fid);
+      if (input_range.IsSubsetOf(dst_vertices)) {
+        sync_comm::isend_buffer(input_data, input_size, dst_fid, 0, comm_,
+                                requests);
+        sent_size_ += input_size * sizeof(MESSAGE_T);
+        continue;
+      } else {
+        CHECK(!input_range.OverlapWith(dst_vertices));
+      }
+      auto src_vertices = partitioner.get_src_vertices(dst_fid);
+      if (input_range.IsSubsetOf(src_vertices)) {
+        sync_comm::isend_buffer(input_data, input_size, dst_fid, 0, comm_,
+                                requests);
+        sent_size_ += input_size * sizeof(MESSAGE_T);
+        continue;
+      } else {
+        CHECK(!input_range.OverlapWith(src_vertices));
+      }
+    }
+
+    const auto& output_range = output.GetVertexRange();
+    for (fid_t i = 1; i < fnum_; ++i) {
+      fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
+      auto master_vertices = partitioner.get_master_vertices(src_fid);
+      if (master_vertices.IsSubsetOf(output_range)) {
+        MESSAGE_T* output_data = &output[*master_vertices.begin()];
+        size_t output_size = master_vertices.size();
+        sync_comm::recv_buffer(output_data, output_size, src_fid, 0, comm_);
+      } else {
+        CHECK(!master_vertices.OverlapWith(output_range));
+      }
+    }
+
+    if (input_range.IsSubsetOf(partitioner.get_master_vertices(fid_))) {
+      MESSAGE_T* output_data = &output[*input_range.begin()];
+      std::copy(input_data, input_data + input_size, output_data);
+    } else {
+      CHECK(!input_range.OverlapWith(partitioner.get_master_vertices(fid_)));
+    }
+
+    MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+  }
+
+ private:
+  fid_t fid_;
+  fid_t fnum_;
+  CommSpec comm_spec_;
+
+  MPI_Comm comm_;
+  int round_;
+
+  size_t sent_size_;
+
+  bool force_terminate_;
+  TerminateInfo terminate_info_;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
diff --git a/grape/utils/vertex_array.h b/grape/utils/vertex_array.h
index e3ee90bc..5bc7fe54 100644
--- a/grape/utils/vertex_array.h
+++ b/grape/utils/vertex_array.h
@@ -121,6 +121,9 @@ bool operator==(Vertex<T> const& lhs, Vertex<T> const& rhs) {
   return lhs.GetValue() == rhs.GetValue();
 }
 
+template <typename T>
+class DualVertexRange;
+
 template <typename T>
 class VertexRange {
  public:
@@ -211,6 +214,18 @@ class VertexRange {
     return begin_ <= v.GetValue() && v.GetValue() < end_;
   }
 
+  inline bool IsSubsetOf(const VertexRange<T>& rhs) const {
+    return rhs.begin_ <= begin_ && end_ <= rhs.end_;
+  }
+
+  bool IsSubsetOf(const DualVertexRange<T>& rhs) const;
+
+  inline bool OverlapWith(const VertexRange<T>& rhs) const {
+    return begin_ < rhs.end_ && end_ > rhs.begin_;
+  }
+
+  bool OverlapWith(const DualVertexRange<T>& rhs) const;
+
   inline friend InArchive& operator<<(InArchive& in_archive,
                                       const VertexRange<T>& range) {
     in_archive << range.begin_ << range.end_;
@@ -335,6 +350,10 @@ class DualVertexRange {
            (tail_begin_ <= v.GetValue() && v.GetValue() < tail_end_);
   }
 
+  bool OverlapWith(const VertexRange<VID_T>& rhs) const {
+    return head().OverlapWith(rhs) || tail().OverlapWith(rhs);
+  }
+
   VID_T size() const {
     return (head_end_ - head_begin_) + (tail_end_ - tail_begin_);
   }
@@ -512,6 +531,13 @@ class VertexArray<DualVertexRange<VID_T>, T> {
     tail_.SetValue(value);
   }
 
+  const DualVertexRange<VID_T>& GetVertexRange() const {
+    return DualVertexRange<VID_T>(head_.GetVertexRange().begin_value(),
+                                  head_.GetVertexRange().end_value(),
+                                  tail_.GetVertexRange().begin_value(),
+                                  tail_.GetVertexRange().end_value());
+  }
+
  private:
   void initMid() { mid_ = head_.GetVertexRange().end_value(); }
 
@@ -520,6 +546,16 @@ class VertexArray<DualVertexRange<VID_T>, T> {
   VID_T mid_;
 };
 
+template <typename T>
+bool VertexRange<T>::IsSubsetOf(const DualVertexRange<T>& rhs) const {
+  return IsSubsetOf(rhs.head()) || IsSubsetOf(rhs.tail());
+}
+
+template <typename T>
+bool VertexRange<T>::OverlapWith(const DualVertexRange<T>& rhs) const {
+  return OverlapWith(rhs.head()) || OverlapWith(rhs.tail());
+}
+
 }  // namespace grape
 
 namespace std {
diff --git a/grape/vertex_map/partitioner.h b/grape/vertex_map/partitioner.h
index 82a3cb50..bffdd0f7 100644
--- a/grape/vertex_map/partitioner.h
+++ b/grape/vertex_map/partitioner.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <string>
 #include "grape/io/io_adaptor_base.h"
+#include "grape/utils/vertex_array.h"
 
 namespace grape {
 
@@ -264,6 +265,81 @@ std::unique_ptr<IPartitioner<OID_T>> deserialize_partitioner(
   return partitioner;
 }
 
+template <typename OID_T>
+struct VCPartitioner {};
+
+template <>
+struct VCPartitioner<int64_t> {
+  using vertices_t = VertexRange<int64_t>;
+
+ public:
+  VCPartitioner() {}
+  VCPartitioner(fid_t fnum, int64_t vnum) { init(fnum, vnum); }
+
+  void init(fid_t fnum, int64_t vnum) {
+    fnum_ = fnum;
+    vnum_ = vnum;
+
+    fnum_sr_ = std::sqrt(fnum_);
+    CHECK_EQ(fnum_sr_ * fnum_sr_, fnum_);
+
+    vchunk_ = (vnum_ + fnum_sr_ - 1) / fnum_sr_;
+  }
+
+  fid_t get_edge_partition(const int64_t& src, const int64_t& dst) const {
+    return coord_to_fid(get_vertex_coord(src), get_vertex_coord(dst));
+  }
+
+  fid_t get_vertex_coord(const int64_t& v) const { return v / vchunk_; }
+
+  fid_t coord_to_fid(const fid_t& src, const fid_t& dst) const {
+    return src * fnum_sr_ + dst;
+  }
+
+  fid_t get_src_coord(fid_t fid) const { return fid / fnum_sr_; }
+
+  fid_t get_dst_coord(fid_t fid) const { return fid % fnum_sr_; }
+
+  vertices_t get_src_vertices(fid_t fid) const {
+    fid_t src_coord = get_src_coord(fid);
+    return vertices_t(src_coord * vchunk_,
+                      std::min(src_coord + vchunk_, vnum_));
+  }
+  vertices_t get_dst_vertices(fid_t fid) const {
+    fid_t dst_coord = get_dst_coord(fid);
+    return vertices_t(dst_coord * vchunk_,
+                      std::min(dst_coord + vchunk_, vnum_));
+  }
+
+  vertices_t get_master_vertices(fid_t fid) const {
+    int64_t master_vchunk = (vnum_ + fnum_ - 1) / fnum_;
+    return vertices_t(fid * master_vchunk,
+                      std::min((fid + 1) * master_vchunk, vnum_));
+  }
+
+  int64_t get_total_vertices_num() const { return vnum_; }
+
+  template <typename IOADAPTOR_T>
+  void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
+    InArchive arc;
+    arc << fnum_ << fnum_sr_ << vnum_ << vchunk_;
+    CHECK(writer->WriteArchive(arc));
+  }
+
+  template <typename IOADAPTOR_T>
+  void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
+    OutArchive arc;
+    CHECK(reader->ReadArchive(arc));
+    arc >> fnum_ >> fnum_sr_ >> vnum_ >> vchunk_;
+  }
+
+ private:
+  fid_t fnum_;
+  fid_t fnum_sr_;
+  int64_t vnum_;
+  int64_t vchunk_;
+};
+
 }  // namespace grape
 
 #include <iostream>
diff --git a/grape/worker/vc_worker.h b/grape/worker/vc_worker.h
new file mode 100644
index 00000000..6840341e
--- /dev/null
+++ b/grape/worker/vc_worker.h
@@ -0,0 +1,117 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_VC_WORKER_WORKER_H_
+#define GRAPE_VC_WORKER_WORKER_H_
+
+#include "grape/app/vc_app_base.h"
+#include "grape/communication/communicator.h"
+#include "grape/parallel/parallel_engine.h"
+#include "grape/worker/comm_spec.h"
+
+namespace grape {
+
+template <typename APP_T, typename MESSAGE_MANAGER_T>
+class VCWorker {
+ public:
+  using fragment_t = typename APP_T::fragment_t;
+  using context_t = typename APP_T::context_t;
+
+  using message_manager_t = MESSAGE_MANAGER_T;
+
+  VCWorker(std::shared_ptr<APP_T> app, std::shared_ptr<fragment_t> graph)
+      : app_(app), context_(nullptr), fragment_(graph) {}
+  ~VCWorker() = default;
+
+  void Init(const CommSpec& comm_spec,
+            const ParallelEngineSpec& pe_spec = DefaultParallelEngineSpec()) {
+    auto& graph = *fragment_;
+
+    comm_spec_ = comm_spec;
+    MPI_Barrier(comm_spec_.comm());
+    if (context_ == nullptr) {
+      context_ = std::make_shared<context_t>(graph);
+    }
+
+    messages_.Init(comm_spec_.comm());
+
+    InitParallelEngine(app_, pe_spec);
+    InitCommunicator(app_, comm_spec_.comm());
+  }
+
+  void Finalize() {}
+
+  template <class... Args>
+  void Query(Args&&... args) {
+    double t = GetCurrentTime();
+
+    context_->Init(messages_, std::forward<Args>(args)...);
+    auto& graph = context_->fragment();
+
+    messages_.Start();
+
+    messages_.StartARound();
+
+    app_->PEval(graph, *context_, messages_);
+
+    messages_.FinishARound();
+
+    if (comm_spec_.worker_id() == kCoordinatorRank) {
+      VLOG(1) << "[Coordinator]: Finished PEval, time: " << GetCurrentTime() - t
+              << " sec";
+    }
+
+    int step = 1;
+
+    while (!messages_.ToTerminate()) {
+      t = GetCurrentTime();
+      messages_.StartARound();
+
+      app_->IncEval(graph, *context_, messages_);
+
+      messages_.FinishARound();
+
+      if (comm_spec_.worker_id() == kCoordinatorRank) {
+        VLOG(1) << "[Coordinator]: Finished IncEval - " << step
+                << ", time: " << GetCurrentTime() - t << " sec";
+      }
+      ++step;
+    }
+
+    MPI_Barrier(comm_spec_.comm());
+
+    messages_.Finalize();
+  }
+
+  std::shared_ptr<context_t> GetContext() { return context_; }
+
+  const TerminateInfo& GetTerminateInfo() const {
+    return messages_.GetTerminateInfo();
+  }
+
+  void Output(std::ostream& os) { context_->Output(os); }
+
+ private:
+  std::shared_ptr<APP_T> app_;
+  std::shared_ptr<context_t> context_;
+  std::shared_ptr<fragment_t> fragment_;
+  message_manager_t messages_;
+
+  CommSpec comm_spec_;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_VC_WORKER_WORKER_H_

From 80c0aaff7c0ee38c45fffe4a962dd5924b3bcdc8 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 12 Sep 2024 14:46:51 +0800
Subject: [PATCH 02/15] ok

---
 .../analytical_apps/pagerank/pagerank_vc.h    | 139 ++++++++++--
 .../pagerank/pagerank_vc_context.h            |  25 ++
 examples/analytical_apps/run_app.cc           |  16 +-
 examples/analytical_apps/run_app.h            |  57 +----
 examples/analytical_apps/run_app_vc.h         |  10 +-
 examples/analytical_apps/utils.h              |  86 +++++++
 grape/fragment/basic_vc_fragment_loader.h     | 214 ++++++++++++++++--
 grape/fragment/immutable_vertexcut_fragment.h |  63 +++++-
 grape/fragment/vc_fragment_loader.h           |   8 +-
 grape/parallel/parallel_engine.h              |  22 ++
 grape/parallel/vc_message_manager.h           |  25 +-
 grape/utils/hp_allocator.h                    |   2 +
 grape/utils/memory_inspector.h                |  57 +++++
 grape/utils/vertex_array.h                    |  12 +-
 grape/vertex_map/partitioner.h                |  17 +-
 15 files changed, 644 insertions(+), 109 deletions(-)
 create mode 100644 examples/analytical_apps/utils.h
 create mode 100644 grape/utils/memory_inspector.h

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index b5520993..a7bd7245 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -48,33 +48,70 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
     ctx.graph_vnum = frag.GetTotalVerticesNum();
 
     {
+      ctx.t0 -= GetCurrentTime();
       typename fragment_t::template both_vertex_array_t<int> degree(
           frag.GetVertices(), 0);
-      for (auto& e : frag.GetEdges()) {
-        ++degree[vertex_t(e.src)];
-        ++degree[vertex_t(e.dst)];
+      // allocate degree array for both src and dst vertices
+      MemoryInspector::GetInstance().allocate(frag.GetVertices().size() *
+                                              sizeof(int));
+      int bucket_num = frag.GetBucketNum();
+      int concurrency = thread_num();
+      if (bucket_num < (concurrency / 2)) {
+        ForEach(
+            frag.GetEdges().begin(), frag.GetEdges().end(),
+            [&](int tid, const typename fragment_t::edge_t& e) {
+              atomic_add(degree[vertex_t(e.src)], 1);
+              atomic_add(degree[vertex_t(e.dst)], 1);
+            },
+            4096);
+      } else {
+        ForEach(0, bucket_num,
+                [&degree, &frag, bucket_num](int tid, int bucket_id) {
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
+                      degree[vertex_t(e.dst)] += 1;
+                    }
+                  }
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
+                      degree[vertex_t(e.src)] += 1;
+                    }
+                  }
+                });
       }
+      ctx.t0 += GetCurrentTime();
 
       messages.GatherMasterVertices<fragment_t, int, NumericSum<int>>(
           frag, degree, ctx.master_degree);
+      // deallocate degree array for both src and dst vertices
+      MemoryInspector::GetInstance().deallocate(frag.GetVertices().size() *
+                                                sizeof(int));
     }
 
-    int64_t dangling_vnum_local = 0;
     double p = 1.0 / ctx.graph_vnum;
-    for (auto v : frag.GetMasterVertices()) {
+    int64_t dangling_vnum_local = 0;
+    ctx.t2 -= GetCurrentTime();
+    std::vector<int64_t> dangling_vnum_local_vec(thread_num(), 0);
+    ForEach(frag.GetMasterVertices(), [&](int tid, vertex_t v) {
       if (ctx.master_degree[v] == 0) {
-        ++dangling_vnum_local;
+        ++dangling_vnum_local_vec[tid];
         ctx.master_result[v] = p;
       } else {
         ctx.master_result[v] = p / ctx.master_degree[v];
       }
+    });
+    for (auto x : dangling_vnum_local_vec) {
+      dangling_vnum_local += x;
     }
+    ctx.t2 += GetCurrentTime();
 
     Sum(dangling_vnum_local, ctx.total_dangling_vnum);
     ctx.dangling_sum = p * ctx.total_dangling_vnum;
 
+    ctx.t7 -= GetCurrentTime();
     messages.ScatterMasterVertices<fragment_t, double>(frag, ctx.master_result,
                                                        ctx.curr_result);
+    ctx.t7 += GetCurrentTime();
   }
 
   void IncEval(const fragment_t& frag, context_t& ctx,
@@ -85,36 +122,106 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
                   ctx.delta * ctx.dangling_sum / ctx.graph_vnum;
     ctx.dangling_sum = base * ctx.total_dangling_vnum;
 
+    ctx.t11 -= GetCurrentTime();
+    ForEach(frag.GetVertices(),
+            [&ctx](int tid, vertex_t v) { ctx.next_result[v] = 0; });
+    ctx.t11 += GetCurrentTime();
+
     if (ctx.step != ctx.max_round) {
-      for (auto& e : frag.GetEdges()) {
-        ctx.next_result[vertex_t(e.dst)] += ctx.curr_result[vertex_t(e.src)];
-        ctx.next_result[vertex_t(e.src)] += ctx.curr_result[vertex_t(e.dst)];
+      ctx.t4 -= GetCurrentTime();
+
+      int bucket_num = frag.GetBucketNum();
+      int concurrency = thread_num();
+      if (bucket_num < (concurrency / 2)) {
+        ForEach(
+            frag.GetEdges().begin(), frag.GetEdges().end(),
+            [&ctx](int tid, const typename fragment_t::edge_t& e) {
+              atomic_add(ctx.next_result[vertex_t(e.dst)],
+                         ctx.curr_result[vertex_t(e.src)]);
+              atomic_add(ctx.next_result[vertex_t(e.src)],
+                         ctx.curr_result[vertex_t(e.dst)]);
+            },
+            4096);
+      } else {
+        ForEach(0, bucket_num,
+                [&ctx, &frag, bucket_num](int tid, int bucket_id) {
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
+                      ctx.next_result[vertex_t(e.dst)] +=
+                          ctx.curr_result[vertex_t(e.src)];
+                    }
+                  }
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
+                      ctx.next_result[vertex_t(e.src)] +=
+                          ctx.curr_result[vertex_t(e.dst)];
+                    }
+                  }
+                });
       }
 
+      ctx.t4 += GetCurrentTime();
+
       messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
           frag, ctx.next_result, ctx.master_result);
-      for (auto v : frag.GetMasterVertices()) {
+
+      ctx.t2 -= GetCurrentTime();
+      ForEach(frag.GetMasterVertices(), [&ctx, base](int tid, vertex_t v) {
         if (ctx.master_degree[v] > 0) {
           ctx.master_result[v] =
               (base + ctx.delta * ctx.master_result[v]) / ctx.master_degree[v];
         } else {
           ctx.master_result[v] = base;
         }
-      }
+      });
+      ctx.t2 += GetCurrentTime();
 
+      ctx.t7 -= GetCurrentTime();
       messages.ScatterMasterVertices<fragment_t, double>(
           frag, ctx.master_result, ctx.curr_result);
+      ctx.t7 += GetCurrentTime();
     } else {
-      for (auto& e : frag.GetEdges()) {
-        ctx.next_result[vertex_t(e.dst)] += ctx.curr_result[vertex_t(e.src)];
-        ctx.next_result[vertex_t(e.src)] += ctx.curr_result[vertex_t(e.dst)];
+      ctx.t4 -= GetCurrentTime();
+      int bucket_num = frag.GetBucketNum();
+      int concurrency = thread_num();
+      if (bucket_num < (concurrency / 2)) {
+        ForEach(
+            frag.GetEdges().begin(), frag.GetEdges().end(),
+            [&ctx](int tid, const typename fragment_t::edge_t& e) {
+              atomic_add(ctx.next_result[vertex_t(e.dst)],
+                         ctx.curr_result[vertex_t(e.src)]);
+              atomic_add(ctx.next_result[vertex_t(e.src)],
+                         ctx.curr_result[vertex_t(e.dst)]);
+            },
+            4096);
+      } else {
+        ForEach(0, bucket_num,
+                [&ctx, &frag, bucket_num](int tid, int bucket_id) {
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
+                      ctx.next_result[vertex_t(e.dst)] +=
+                          ctx.curr_result[vertex_t(e.src)];
+                    }
+                  }
+                  for (int i = 0; i < bucket_num; ++i) {
+                    for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
+                      ctx.next_result[vertex_t(e.src)] +=
+                          ctx.curr_result[vertex_t(e.dst)];
+                    }
+                  }
+                });
       }
 
+      ctx.t4 += GetCurrentTime();
+
       messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
           frag, ctx.next_result, ctx.master_result);
-      for (auto v : frag.GetMasterVertices()) {
+
+      ctx.t2 -= GetCurrentTime();
+      ForEach(frag.GetMasterVertices(), [&ctx, base](int tid, vertex_t v) {
         ctx.master_result[v] = ctx.master_result[v] * ctx.delta + base;
-      }
+      });
+      ctx.t2 += GetCurrentTime();
 
       messages.ForceTerminate();
     }
diff --git a/examples/analytical_apps/pagerank/pagerank_vc_context.h b/examples/analytical_apps/pagerank/pagerank_vc_context.h
index 41ad6737..11bf766c 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc_context.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc_context.h
@@ -16,6 +16,10 @@ limitations under the License.
 #ifndef EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
 #define EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
 
+#include "grape/utils/memory_inspector.h"
+
+#include <iomanip>
+
 namespace grape {
 
 template <typename FRAG_T>
@@ -30,6 +34,15 @@ class PageRankVCContext {
     next_result.Init(fragment.GetVertices());
     master_result.Init(fragment.GetMasterVertices());
     master_degree.Init(fragment.GetMasterVertices());
+
+    MemoryInspector::GetInstance().allocate(fragment.GetVertices().size() *
+                                            sizeof(double));
+    MemoryInspector::GetInstance().allocate(fragment.GetVertices().size() *
+                                            sizeof(double));
+    MemoryInspector::GetInstance().allocate(
+        fragment.GetMasterVertices().size() * sizeof(double));
+    MemoryInspector::GetInstance().allocate(
+        fragment.GetMasterVertices().size() * sizeof(int));
   }
 
   void Init(VCMessageManager& messages, double delta, int max_round) {
@@ -45,6 +58,12 @@ class PageRankVCContext {
       os << v.GetValue() << " " << std::scientific << std::setprecision(15)
          << master_result[v] << std::endl;
     }
+
+    LOG(INFO) << "[frag-" << frag.fid() << "]: init degree: " << t0 << " s, "
+              << "calc master result: " << t2 << " s, "
+              << "propogate: " << t4 << " s, "
+              << "scatter: " << t7 << " s, "
+              << "reset next result: " << t11 << " s";
   }
 
   typename FRAG_T::template both_vertex_array_t<double> curr_result;
@@ -61,6 +80,12 @@ class PageRankVCContext {
   double dangling_sum = 0.0;
 
   const FRAG_T& fragment_;
+
+  double t0 = 0;   // init degree
+  double t2 = 0;   // calc master result
+  double t4 = 0;   // propogate
+  double t7 = 0;   // scatter
+  double t11 = 0;  // reset next result
 };
 
 }  // namespace grape
diff --git a/examples/analytical_apps/run_app.cc b/examples/analytical_apps/run_app.cc
index 7af79bd8..f1c445b2 100644
--- a/examples/analytical_apps/run_app.cc
+++ b/examples/analytical_apps/run_app.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include "run_app.h"
+// #include "run_app.h"
 
 #include <gflags/gflags.h>
 #include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 
-#include "run_app_opt.h"
+// #include "run_app_opt.h"
 #include "run_app_vc.h"
 
 int main(int argc, char* argv[]) {
@@ -41,15 +41,15 @@ int main(int argc, char* argv[]) {
 
   std::string name = FLAGS_application;
   if (FLAGS_opt) {
-    grape::RunOpt();
+    // grape::RunOpt();
   } else if (FLAGS_vc) {
     grape::RunVC();
   } else {
-    if (name.find("sssp") != std::string::npos) {
-      grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
-    } else {
-      grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
-    }
+    // if (name.find("sssp") != std::string::npos) {
+    //   grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
+    // } else {
+    //   grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
+    // }
   }
 
   grape::Finalize();
diff --git a/examples/analytical_apps/run_app.h b/examples/analytical_apps/run_app.h
index 94e17f78..9f244d5b 100644
--- a/examples/analytical_apps/run_app.h
+++ b/examples/analytical_apps/run_app.h
@@ -59,67 +59,12 @@ limitations under the License.
 #include "sssp/sssp.h"
 #include "sssp/sssp_auto.h"
 #include "timer.h"
+#include "utils.h"
 #include "wcc/wcc.h"
 #include "wcc/wcc_auto.h"
 
-#ifndef __AFFINITY__
-#define __AFFINITY__ false
-#endif
-
 namespace grape {
 
-void Init() {
-  if (FLAGS_deserialize && FLAGS_serialization_prefix.empty()) {
-    LOG(FATAL) << "Please assign a serialization prefix.";
-  } else if (FLAGS_efile.empty()) {
-    LOG(FATAL) << "Please assign input edge file.";
-  }
-
-  if (!FLAGS_out_prefix.empty() && access(FLAGS_out_prefix.c_str(), 0) != 0) {
-    mkdir(FLAGS_out_prefix.c_str(), 0777);
-  }
-
-  InitMPIComm();
-  CommSpec comm_spec;
-  comm_spec.Init(MPI_COMM_WORLD);
-  if (comm_spec.worker_id() == kCoordinatorRank) {
-    VLOG(1) << "Workers of libgrape-lite initialized.";
-  }
-}
-
-void Finalize() {
-  FinalizeMPIComm();
-  VLOG(1) << "Workers finalized.";
-}
-
-template <typename FRAG_T, typename APP_T, typename... Args>
-void DoQuery(std::shared_ptr<FRAG_T> fragment, std::shared_ptr<APP_T> app,
-             const CommSpec& comm_spec, const ParallelEngineSpec& spec,
-             const std::string& out_prefix, Args... args) {
-  timer_next("load application");
-  auto worker = APP_T::CreateWorker(app, fragment);
-  worker->Init(comm_spec, spec);
-  MPI_Barrier(comm_spec.comm());
-  timer_next("run algorithm");
-  worker->Query(std::forward<Args>(args)...);
-  timer_next("print output");
-  if (!out_prefix.empty()) {
-    std::ofstream ostream;
-    std::string output_path =
-        grape::GetResultFilename(out_prefix, fragment->fid());
-    ostream.open(output_path);
-    worker->Output(ostream);
-    ostream.close();
-    worker->Finalize();
-    VLOG(1) << "Worker-" << comm_spec.worker_id()
-            << " finished: " << output_path;
-  } else {
-    worker->Finalize();
-    VLOG(1) << "Worker-" << comm_spec.worker_id() << " finished without output";
-  }
-  timer_end();
-}
-
 template <typename FRAG_T, typename APP1_T, typename APP2_T, typename... Args>
 void DoDualQuery(std::shared_ptr<FRAG_T> fragment, std::shared_ptr<APP1_T> app1,
                  std::shared_ptr<APP2_T> app2, const CommSpec& comm_spec,
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index 77439a7b..bd062863 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -19,7 +19,7 @@ limitations under the License.
 #include "grape/fragment/immutable_vertexcut_fragment.h"
 #include "grape/fragment/loader.h"
 #include "pagerank/pagerank_vc.h"
-#include "run_app.h"
+#include "utils.h"
 
 namespace grape {
 
@@ -38,10 +38,18 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
                                             grape::EmptyType>;
   std::shared_ptr<FRAG_T> fragment = LoadVertexcutGraph<FRAG_T>(
       FLAGS_efile, FLAGS_vertex_num, comm_spec, graph_spec);
+  VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after loading graph: "
+          << MemoryInspector::GetInstance().GetCurrentMemoryUsage()
+          << " GB, peak: "
+          << MemoryInspector::GetInstance().GetPeakMemoryUsage() << " GB";
   using AppType = APP_T<FRAG_T>;
   auto app = std::make_shared<AppType>();
   DoQuery<FRAG_T, AppType, Args...>(fragment, app, comm_spec, spec, out_prefix,
                                     args...);
+  VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after query: "
+          << MemoryInspector::GetInstance().GetCurrentMemoryUsage()
+          << " GB, peak: "
+          << MemoryInspector::GetInstance().GetPeakMemoryUsage() << " GB";
 }
 
 void RunVC() {
diff --git a/examples/analytical_apps/utils.h b/examples/analytical_apps/utils.h
new file mode 100644
index 00000000..79a7f3f5
--- /dev/null
+++ b/examples/analytical_apps/utils.h
@@ -0,0 +1,86 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef EXAMPLES_ANALYTICAL_APPS_UTILS_H_
+#define EXAMPLES_ANALYTICAL_APPS_UTILS_H_
+
+#include <glog/logging.h>
+
+#include <grape/grape.h>
+
+#include "flags.h"
+#include "timer.h"
+
+#ifndef __AFFINITY__
+#define __AFFINITY__ false
+#endif
+
+namespace grape {
+
+void Init() {
+  if (FLAGS_deserialize && FLAGS_serialization_prefix.empty()) {
+    LOG(FATAL) << "Please assign a serialization prefix.";
+  } else if (FLAGS_efile.empty()) {
+    LOG(FATAL) << "Please assign input edge file.";
+  }
+
+  if (!FLAGS_out_prefix.empty() && access(FLAGS_out_prefix.c_str(), 0) != 0) {
+    mkdir(FLAGS_out_prefix.c_str(), 0777);
+  }
+
+  InitMPIComm();
+  CommSpec comm_spec;
+  comm_spec.Init(MPI_COMM_WORLD);
+  if (comm_spec.worker_id() == kCoordinatorRank) {
+    VLOG(1) << "Workers of libgrape-lite initialized.";
+  }
+}
+
+void Finalize() {
+  FinalizeMPIComm();
+  VLOG(1) << "Workers finalized.";
+}
+
+template <typename FRAG_T, typename APP_T, typename... Args>
+void DoQuery(std::shared_ptr<FRAG_T> fragment, std::shared_ptr<APP_T> app,
+             const CommSpec& comm_spec, const ParallelEngineSpec& spec,
+             const std::string& out_prefix, Args... args) {
+  timer_next("load application");
+  auto worker = APP_T::CreateWorker(app, fragment);
+  worker->Init(comm_spec, spec);
+  MPI_Barrier(comm_spec.comm());
+  timer_next("run algorithm");
+  worker->Query(std::forward<Args>(args)...);
+  timer_next("print output");
+  if (!out_prefix.empty()) {
+    std::ofstream ostream;
+    std::string output_path =
+        grape::GetResultFilename(out_prefix, fragment->fid());
+    ostream.open(output_path);
+    worker->Output(ostream);
+    ostream.close();
+    worker->Finalize();
+    VLOG(1) << "Worker-" << comm_spec.worker_id()
+            << " finished: " << output_path;
+  } else {
+    worker->Finalize();
+    VLOG(1) << "Worker-" << comm_spec.worker_id() << " finished without output";
+  }
+  timer_end();
+}
+
+}  // namespace grape
+
+#endif  // EXAMPLES_ANALYTICAL_APPS_UTILS_H_
\ No newline at end of file
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
index a51e3b45..68d67e93 100644
--- a/grape/fragment/basic_vc_fragment_loader.h
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
 #define GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
 
+#include "grape/utils/memory_inspector.h"
+
 namespace grape {
 
 template <typename FRAG_T>
@@ -28,23 +30,51 @@ class BasicVCFragmentLoader {
   using edge_t = typename fragment_t::edge_t;
   using vertices_t = typename fragment_t::vertices_t;
 
+  static constexpr size_t shuffle_out_size = 4096000;
+
  public:
-  explicit BasicVCFragmentLoader(const CommSpec& comm_spec, int64_t vnum)
-      : comm_spec_(comm_spec), partitioner_(comm_spec.fnum(), vnum) {
+  explicit BasicVCFragmentLoader(const CommSpec& comm_spec, int64_t vnum,
+                                 int load_concurrency, int bucket_num)
+      : comm_spec_(comm_spec),
+        partitioner_(comm_spec.fnum(), vnum),
+        bucket_num_(bucket_num),
+        vnum_(vnum),
+        load_concurrency_(load_concurrency) {
     comm_spec_.Dup();
-
     edges_to_frag_.resize(comm_spec_.fnum());
     for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
       int worker_id = comm_spec_.FragToWorker(fid);
-      edges_to_frag_[fid].Init(comm_spec_.comm(), edge_tag, 4096000);
+      edges_to_frag_[fid].Init(comm_spec_.comm(), edge_tag, shuffle_out_size);
       edges_to_frag_[fid].SetDestination(worker_id, fid);
       if (worker_id == comm_spec_.worker_id()) {
         edges_to_frag_[fid].DisableComm();
       }
     }
+
+    // allocate shuffle out buffers
+    MemoryInspector::GetInstance().allocate(
+        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
+        shuffle_out_size);
     edge_recv_thread_ =
         std::thread(&BasicVCFragmentLoader::edgeRecvRoutine, this);
     recv_thread_running_ = true;
+
+    auto src_vertices = partitioner_.get_src_vertices(comm_spec_.fid());
+    src_start_ = src_vertices.begin_value();
+    src_chunk_ = (src_vertices.size() + bucket_num_ - 1) / bucket_num_;
+
+    auto dst_vertices = partitioner_.get_dst_vertices(comm_spec_.fid());
+    dst_start_ = dst_vertices.begin_value();
+    dst_chunk_ = (dst_vertices.size() + bucket_num_ - 1) / bucket_num_;
+  }
+
+  ~BasicVCFragmentLoader() {
+    if (recv_thread_running_) {
+      for (auto& e : edges_to_frag_) {
+        e.Flush();
+      }
+      edge_recv_thread_.join();
+    }
   }
 
   void AddEdge(const oid_t& src, const oid_t& dst, const edata_t& data) {
@@ -62,22 +92,165 @@ class BasicVCFragmentLoader {
     MPI_Barrier(comm_spec_.comm());
     got_edges_.emplace_back(
         std::move(edges_to_frag_[comm_spec_.fid()].buffers()));
+
+    size_t got_edges_memory_usage = 0;
+    for (auto& buf : got_edges_) {
+      got_edges_memory_usage +=
+          buf.size() * (sizeof(oid_t) * 2 + sizeof(edata_t));
+    }
+    // allocate edges recv buffer
+    MemoryInspector::GetInstance().allocate(got_edges_memory_usage);
+
     edges_to_frag_[comm_spec_.fid()].Clear();
+    // deallocate shuffle out buffers
+    MemoryInspector::GetInstance().deallocate(
+        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
+        shuffle_out_size);
 
-    std::vector<edge_t> edges;
+    std::vector<size_t> bucket_edge_num(bucket_num_ * bucket_num_, 0);
+    {
+      std::vector<std::vector<size_t>> thread_local_bucket_edge_num(
+          load_concurrency_);
+      std::vector<std::thread> scan_threads;
+      for (int i = 0; i < load_concurrency_; ++i) {
+        scan_threads.emplace_back(
+            [this, &bucket_edge_num, &thread_local_bucket_edge_num](int tid) {
+              auto& vec = thread_local_bucket_edge_num[tid];
+              vec.clear();
+              vec.resize(bucket_num_ * bucket_num_, 0);
+
+              for (auto& buffer : got_edges_) {
+                size_t size = buffer.size();
+                size_t chunk =
+                    (size + load_concurrency_ - 1) / load_concurrency_;
+                size_t start = std::min(tid * chunk, size);
+                size_t end = std::min(start + chunk, size);
+                range_foreach_helper(
+                    buffer, start, end,
+                    [&vec, this](const oid_t& src, const oid_t& dst) {
+                      ++vec[edgeBucketId(src, dst)];
+                    },
+                    make_index_sequence<2>{});
+              }
+            },
+            i);
+      }
+      for (auto& thrd : scan_threads) {
+        thrd.join();
+      }
+      for (auto& vec : thread_local_bucket_edge_num) {
+        for (size_t i = 0; i < bucket_edge_num.size(); ++i) {
+          bucket_edge_num[i] += vec[i];
+        }
+      }
+    }
+
+    std::vector<std::atomic<size_t>> bucket_edge_cursor(bucket_num_ *
+                                                        bucket_num_);
+    std::vector<size_t> bucket_edge_offset;
     size_t edge_num = 0;
-    for (auto& buffer : got_edges_) {
-      edge_num += buffer.size();
+    for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
+      bucket_edge_cursor[i].store(edge_num);
+      bucket_edge_offset.emplace_back(edge_num);
+      edge_num += bucket_edge_num[i];
     }
-    edges.reserve(edge_num);
-    for (auto& buffer : got_edges_) {
-      foreach_rval(buffer, [&edges](oid_t&& src, oid_t&& dst, edata_t&& data) {
-        edges.emplace_back(src, dst, data);
-      });
+    bucket_edge_offset.emplace_back(edge_num);
+
+    std::vector<edge_t> edges;
+    edges.resize(edge_num);
+    // allocate edges buffer
+    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edge_num);
+
+    {
+      static constexpr size_t thread_local_cache_size = 4096;
+      // allocate thread local cache
+      MemoryInspector::GetInstance().allocate(
+          sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
+          load_concurrency_);
+      std::vector<std::thread> insert_threads;
+      for (int i = 0; i < load_concurrency_; ++i) {
+        insert_threads.emplace_back(
+            [this, &bucket_edge_cursor, &edges](int tid) {
+              std::vector<std::vector<edge_t>> thread_local_edges(bucket_num_ *
+                                                                  bucket_num_);
+
+              for (auto& buffer : got_edges_) {
+                size_t size = buffer.size();
+                size_t chunk =
+                    (size + load_concurrency_ - 1) / load_concurrency_;
+                size_t start = std::min(tid * chunk, size);
+                size_t end = std::min(start + chunk, size);
+                range_foreach_rval(
+                    buffer, start, end,
+                    [&, this](oid_t&& src, oid_t&& dst, edata_t&& data) {
+                      int idx = edgeBucketId(src, dst);
+                      thread_local_edges[idx].emplace_back(
+                          std::move(src), std::move(dst), std::move(data));
+
+                      if (thread_local_edges[idx].size() ==
+                          thread_local_cache_size) {
+                        size_t cursor = bucket_edge_cursor[idx].fetch_add(
+                            thread_local_cache_size);
+                        std::copy(thread_local_edges[idx].begin(),
+                                  thread_local_edges[idx].end(),
+                                  edges.begin() + cursor);
+                        thread_local_edges[idx].clear();
+                      }
+                    });
+              }
+
+              for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
+                size_t cursor = bucket_edge_cursor[i].fetch_add(
+                    thread_local_edges[i].size());
+                std::copy(thread_local_edges[i].begin(),
+                          thread_local_edges[i].end(), edges.begin() + cursor);
+              }
+            },
+            i);
+      }
+      for (auto& thrd : insert_threads) {
+        thrd.join();
+      }
+      // deallocate thread local cache
+      MemoryInspector::GetInstance().deallocate(
+          sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
+          load_concurrency_);
+      got_edges_.clear();
+      // deallocate edges recv buffer
+      MemoryInspector::GetInstance().deallocate(got_edges_memory_usage);
+    }
+
+    {
+      std::atomic<int> d2_bucket_idx(0);
+      int d2_bucket_num = bucket_num_ * bucket_num_;
+      std::vector<std::thread> sort_threads;
+      for (int i = 0; i < load_concurrency_; ++i) {
+        sort_threads.emplace_back(
+            [this, &d2_bucket_idx, d2_bucket_num, &edges,
+             &bucket_edge_offset](int tid) {
+              while (true) {
+                int idx = d2_bucket_idx.fetch_add(1);
+                if (idx >= d2_bucket_num) {
+                  break;
+                }
+                std::sort(edges.begin() + bucket_edge_offset[idx],
+                          edges.begin() + bucket_edge_offset[idx + 1],
+                          [](const edge_t& a, const edge_t& b) {
+                            return a.src < b.src ||
+                                   (a.src == b.src && a.dst < b.dst);
+                          });
+              }
+            },
+            i);
+      }
+      for (auto& thrd : sort_threads) {
+        thrd.join();
+      }
     }
 
     fragment.reset(new fragment_t());
-    fragment->Init(comm_spec_, vnum_, std::move(edges));
+    fragment->Init(comm_spec_, vnum_, std::move(edges), bucket_num_,
+                   std::move(bucket_edge_offset));
   }
 
  private:
@@ -97,9 +270,22 @@ class BasicVCFragmentLoader {
     }
   }
 
+  int edgeBucketId(const oid_t& src, const oid_t& dst) const {
+    int src_bucket_id = (src - src_start_) / src_chunk_;
+    int dst_bucket_id = (dst - dst_start_) / dst_chunk_;
+    return src_bucket_id * bucket_num_ + dst_bucket_id;
+  }
+
   CommSpec comm_spec_;
+  VCPartitioner<oid_t> partitioner_;
+  int bucket_num_;
+  oid_t src_start_;
+  oid_t dst_start_;
+  oid_t src_chunk_;
+  oid_t dst_chunk_;
+
   int64_t vnum_;
-  VCPartitioner<int64_t> partitioner_;
+  int load_concurrency_;
 
   std::vector<ShuffleBufferTuple<oid_t, oid_t, edata_t>> got_edges_;
 
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index 9d74bc4f..c27cb45a 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -16,13 +16,37 @@ limitations under the License.
 #ifndef GRAPE_FRAGMENT_IMMUTABLE_VERTEXCUT_FRAGMENT_H_
 #define GRAPE_FRAGMENT_IMMUTABLE_VERTEXCUT_FRAGMENT_H_
 
+#include <atomic>
+
 #include <glog/logging.h>
 
+#include "grape/graph/edge.h"
+#include "grape/types.h"
+#include "grape/utils/memory_inspector.h"
+#include "grape/utils/vertex_array.h"
+#include "grape/vertex_map/partitioner.h"
+#include "grape/worker/comm_spec.h"
+
 namespace grape {
 
+// #define USE_EDGE_ARRAY
+
 template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T>
 class ImmutableVertexcutFragment {};
 
+template <typename T>
+struct IteratorPair {
+  IteratorPair() : begin_(nullptr), end_(nullptr) {}
+  IteratorPair(const T* begin, const T* end) : begin_(begin), end_(end) {}
+
+  const T* begin() const { return begin_; }
+
+  const T* end() const { return end_; }
+
+  const T* begin_;
+  const T* end_;
+};
+
 template <typename VID_T>
 class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
  public:
@@ -41,10 +65,11 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   using both_vertex_array_t = VertexArray<both_vertices_t, T>;
 
   ImmutableVertexcutFragment() = default;
-  ~ImmutableVertexcutFragment() = default;
+  ~ImmutableVertexcutFragment() {}
 
   void Init(const CommSpec& comm_spec, int64_t vnum,
-            std::vector<edge_t>&& edges) {
+            std::vector<edge_t>&& edges, int bucket_num,
+            std::vector<size_t>&& bucket_edge_offsets) {
     fid_ = comm_spec.fid();
     fnum_ = comm_spec.fnum();
 
@@ -67,7 +92,15 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
                                   std::max(src_end, dst_end));
     }
 
+#ifdef USE_EDGE_ARRAY
+    edges_.resize(edges.size());
+    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edges_.size());
+    std::copy(edges.begin(), edges.end(), edges_.begin());
+#else
     edges_ = std::move(edges);
+#endif
+    bucket_num_ = bucket_num;
+    bucket_edge_offsets_ = std::move(bucket_edge_offsets);
   }
 
   const vertices_t& GetSourceVertices() const { return src_vertices_; }
@@ -76,7 +109,11 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
 
   const vertices_t& GetMasterVertices() const { return master_vertices_; }
 
+#ifdef USE_EDGE_ARRAY
+  const Array<edge_t, Allocator<edge_t>>& GetEdges() const { return edges_; }
+#else
   const std::vector<edge_t>& GetEdges() const { return edges_; }
+#endif
 
   fid_t fid() const { return fid_; }
   fid_t fnum() const { return fnum_; }
@@ -94,6 +131,7 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
     InArchive arc;
     arc << fid_ << fnum_ << edges_.size();
     arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_;
+    arc << bucket_num_ << bucket_edge_offsets_;
     CHECK(io_adaptor->WriteArchive(arc));
     arc.Clear();
 
@@ -131,6 +169,7 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
     CHECK_EQ(fid_, comm_spec.fid());
     CHECK_EQ(fnum_, comm_spec.fnum());
     arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_;
+    arc >> bucket_num_ >> bucket_edge_offsets_;
 
     partitioner_.deserialize(io_adaptor);
 
@@ -146,6 +185,8 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
       CHECK(io_adaptor->ReadArchive(arc));
       arc >> edges_;
     }
+
+    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edges_.size());
   }
 
   int64_t GetTotalVerticesNum() const {
@@ -154,6 +195,18 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
 
   const VCPartitioner<int64_t>& GetPartitioner() const { return partitioner_; }
 
+  IteratorPair<edge_t> GetEdgesOfBucket(int src_bucket_id,
+                                        int dst_bucket_id) const {
+    int idx = src_bucket_id * bucket_num_ + dst_bucket_id;
+    if (static_cast<size_t>(idx) >= bucket_edge_offsets_.size()) {
+      return IteratorPair<edge_t>();
+    }
+    return IteratorPair<edge_t>(edges_.data() + bucket_edge_offsets_[idx],
+                                edges_.data() + bucket_edge_offsets_[idx + 1]);
+  }
+
+  int GetBucketNum() const { return bucket_num_; }
+
  private:
   fid_t fid_;
   fid_t fnum_;
@@ -162,7 +215,13 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   vertices_t dst_vertices_;
   both_vertices_t vertices_;
   vertices_t master_vertices_;
+#ifdef USE_EDGE_ARRAY
+  Array<edge_t, Allocator<edge_t>> edges_;
+#else
   std::vector<edge_t> edges_;
+#endif
+  int bucket_num_;
+  std::vector<size_t> bucket_edge_offsets_;
 };
 
 }  // namespace grape
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
index 146dcf76..a03ae2a1 100644
--- a/grape/fragment/vc_fragment_loader.h
+++ b/grape/fragment/vc_fragment_loader.h
@@ -53,7 +53,9 @@ class VCFragmentLoader {
     }
 
     basic_fragment_loader_ = std::unique_ptr<BasicVCFragmentLoader<fragment_t>>(
-        new BasicVCFragmentLoader<fragment_t>(comm_spec_, vnum));
+        new BasicVCFragmentLoader<fragment_t>(
+            comm_spec_, vnum, spec.load_concurrency,
+            std::thread::hardware_concurrency()));
 
     auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
     io_adaptor->SetPartialRead(comm_spec_.worker_id(), comm_spec_.worker_num());
@@ -99,6 +101,10 @@ class VCFragmentLoader {
     if (comm_spec_.worker_id() == 0) {
       VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
     }
+    VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
+            << MemoryInspector::GetInstance().GetCurrentMemoryUsage() << " GB"
+            << ", peak: " << MemoryInspector::GetInstance().GetPeakMemoryUsage()
+            << " GB";
 
     if (spec.serialize) {
       fragment->template Serialize<IOADAPTOR_T>(spec.serialization_prefix);
diff --git a/grape/parallel/parallel_engine.h b/grape/parallel/parallel_engine.h
index 70f9e34b..69dd0894 100644
--- a/grape/parallel/parallel_engine.h
+++ b/grape/parallel/parallel_engine.h
@@ -42,6 +42,28 @@ class ParallelEngine {
 
   inline ThreadPool& GetThreadPool() { return thread_pool_; }
 
+  template <typename ITER_FUNC_T>
+  inline void ForEach(int from, int to, const ITER_FUNC_T& iter_func) {
+    std::atomic<int> cur(from);
+
+    std::vector<std::future<void>> results(thread_num_);
+    for (uint32_t tid = 0; tid < thread_num_; ++tid) {
+      results[tid] = thread_pool_.enqueue(
+          [&cur, &iter_func, to](int tid) {
+            while (true) {
+              int got = cur.fetch_add(1);
+              if (got >= to) {
+                break;
+              }
+              iter_func(tid, got);
+            }
+          },
+          tid);
+    }
+
+    thread_pool_.WaitEnd(results);
+  }
+
   /**
    * @brief Iterate on vertexes of a VertexRange concurrently.
    *
diff --git a/grape/parallel/vc_message_manager.h b/grape/parallel/vc_message_manager.h
index 985529f5..0f13a6b7 100644
--- a/grape/parallel/vc_message_manager.h
+++ b/grape/parallel/vc_message_manager.h
@@ -49,7 +49,10 @@ class VCMessageManager : public MessageManagerBase {
 
   bool ToTerminate() override { return force_terminate_; }
 
-  void Finalize() override {}
+  void Finalize() override {
+    LOG(INFO) << "[frag-" << fid_ << "] gather comm: " << t0_gather_comm_
+              << ", gather calc: " << t1_gather_calc_;
+  }
 
   void ForceContinue() override {}
 
@@ -73,6 +76,7 @@ class VCMessageManager : public MessageManagerBase {
       LOG(FATAL) << "not implemented for non-POD type";
     }
 
+    t0_gather_comm_ -= GetCurrentTime();
     const auto& partitioner = frag.GetPartitioner();
     const auto& input_range = input.GetVertexRange();
     std::vector<MPI_Request> requests;
@@ -86,7 +90,9 @@ class VCMessageManager : public MessageManagerBase {
                                 requests);
         sent_size_ += input_size * sizeof(MESSAGE_T);
       } else {
-        CHECK(!input_range.OverlapWith(target_vertices));
+        CHECK(!input_range.OverlapWith(target_vertices))
+            << "input: " << input_range.to_string()
+            << ", target: " << target_vertices.to_string();
       }
     }
 
@@ -98,6 +104,8 @@ class VCMessageManager : public MessageManagerBase {
       auto src_vertices = partitioner.get_src_vertices(src_fid);
       if (output_range.IsSubsetOf(src_vertices)) {
         recv_buffers.emplace_back(output_size);
+        MemoryInspector::GetInstance().allocate(output_size *
+                                                sizeof(MESSAGE_T));
         sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
                                 src_fid, 0, comm_, requests);
         continue;
@@ -107,6 +115,8 @@ class VCMessageManager : public MessageManagerBase {
       auto dst_vertices = partitioner.get_dst_vertices(src_fid);
       if (output_range.IsSubsetOf(dst_vertices)) {
         recv_buffers.emplace_back(output_size);
+        MemoryInspector::GetInstance().allocate(output_size *
+                                                sizeof(MESSAGE_T));
         sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
                                 src_fid, 0, comm_, requests);
         continue;
@@ -116,7 +126,9 @@ class VCMessageManager : public MessageManagerBase {
     }
 
     MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+    t0_gather_comm_ += GetCurrentTime();
 
+    t1_gather_calc_ -= GetCurrentTime();
     MESSAGE_T* output_data = &output[*output_range.begin()];
     std::vector<std::thread> threads;
     int thread_num = std::thread::hardware_concurrency();
@@ -164,6 +176,12 @@ class VCMessageManager : public MessageManagerBase {
     for (auto& thrd : threads) {
       thrd.join();
     }
+
+    for (auto& recv_buffer : recv_buffers) {
+      MemoryInspector::GetInstance().deallocate(recv_buffer.size() *
+                                                sizeof(MESSAGE_T));
+    }
+    t1_gather_calc_ += GetCurrentTime();
   }
 
   template <typename GRAPH_T, typename MESSAGE_T>
@@ -237,6 +255,9 @@ class VCMessageManager : public MessageManagerBase {
 
   bool force_terminate_;
   TerminateInfo terminate_info_;
+
+  double t0_gather_comm_ = 0;
+  double t1_gather_calc_ = 0;
 };
 
 }  // namespace grape
diff --git a/grape/utils/hp_allocator.h b/grape/utils/hp_allocator.h
index 8dc4e140..5d3ad0c7 100644
--- a/grape/utils/hp_allocator.h
+++ b/grape/utils/hp_allocator.h
@@ -19,6 +19,8 @@ limitations under the License.
 #include <stdlib.h>
 #include <sys/mman.h>
 
+#include <limits>
+
 #include <glog/logging.h>
 
 #ifdef USE_HUGEPAGES
diff --git a/grape/utils/memory_inspector.h b/grape/utils/memory_inspector.h
new file mode 100644
index 00000000..ad7190b9
--- /dev/null
+++ b/grape/utils/memory_inspector.h
@@ -0,0 +1,57 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_UTILS_MEMORY_INSPECTOR_H_
+#define GRAPE_UTILS_MEMORY_INSPECTOR_H_
+
+#include "glog/logging.h"
+
+namespace grape {
+
+struct MemoryInspector {
+  static MemoryInspector& GetInstance() {
+    static MemoryInspector instance;
+    return instance;
+  }
+
+  void allocate(size_t size) {
+    current_memory_usage += size;
+    peak_memory_usage = std::max(current_memory_usage, peak_memory_usage);
+  }
+
+  void deallocate(size_t size) { current_memory_usage -= size; }
+
+  ~MemoryInspector() {
+    double peak_memory_usage_gb = peak_memory_usage / 1024.0 / 1024.0 / 1024.0;
+    LOG(INFO) << "Peak memory usage: " << peak_memory_usage_gb << " GB";
+  }
+
+  double GetCurrentMemoryUsage() const {
+    return current_memory_usage / 1024.0 / 1024.0 / 1024.0;
+  }
+  double GetPeakMemoryUsage() const {
+    return peak_memory_usage / 1024.0 / 1024.0 / 1024.0;
+  }
+
+ private:
+  MemoryInspector() = default;
+
+  size_t current_memory_usage = 0;
+  size_t peak_memory_usage = 0;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_UTILS_MEMORY_INSPECTOR_H_
\ No newline at end of file
diff --git a/grape/utils/vertex_array.h b/grape/utils/vertex_array.h
index 5bc7fe54..c8d5bf54 100644
--- a/grape/utils/vertex_array.h
+++ b/grape/utils/vertex_array.h
@@ -238,6 +238,10 @@ class VertexRange {
     return out_archive;
   }
 
+  std::string to_string() const {
+    return "[" + std::to_string(begin_) + ", " + std::to_string(end_) + ")";
+  }
+
  private:
   T begin_, end_;
 };
@@ -372,6 +376,12 @@ class DualVertexRange {
     return out_archive;
   }
 
+  std::string to_string() const {
+    return "[" + std::to_string(head_begin_) + ", " +
+           std::to_string(head_end_) + "), [" + std::to_string(tail_begin_) +
+           " " + std::to_string(tail_end_) + ")";
+  }
+
  private:
   VID_T head_begin_;
   VID_T head_end_;
@@ -531,7 +541,7 @@ class VertexArray<DualVertexRange<VID_T>, T> {
     tail_.SetValue(value);
   }
 
-  const DualVertexRange<VID_T>& GetVertexRange() const {
+  DualVertexRange<VID_T> GetVertexRange() const {
     return DualVertexRange<VID_T>(head_.GetVertexRange().begin_value(),
                                   head_.GetVertexRange().end_value(),
                                   tail_.GetVertexRange().begin_value(),
diff --git a/grape/vertex_map/partitioner.h b/grape/vertex_map/partitioner.h
index bffdd0f7..ba1e3b74 100644
--- a/grape/vertex_map/partitioner.h
+++ b/grape/vertex_map/partitioner.h
@@ -283,7 +283,8 @@ struct VCPartitioner<int64_t> {
     fnum_sr_ = std::sqrt(fnum_);
     CHECK_EQ(fnum_sr_ * fnum_sr_, fnum_);
 
-    vchunk_ = (vnum_ + fnum_sr_ - 1) / fnum_sr_;
+    master_vchunk_ = (vnum_ + fnum_ - 1) / fnum_;
+    vchunk_ = master_vchunk_ * fnum_sr_;
   }
 
   fid_t get_edge_partition(const int64_t& src, const int64_t& dst) const {
@@ -303,18 +304,17 @@ struct VCPartitioner<int64_t> {
   vertices_t get_src_vertices(fid_t fid) const {
     fid_t src_coord = get_src_coord(fid);
     return vertices_t(src_coord * vchunk_,
-                      std::min(src_coord + vchunk_, vnum_));
+                      std::min((src_coord + 1) * vchunk_, vnum_));
   }
   vertices_t get_dst_vertices(fid_t fid) const {
     fid_t dst_coord = get_dst_coord(fid);
     return vertices_t(dst_coord * vchunk_,
-                      std::min(dst_coord + vchunk_, vnum_));
+                      std::min((dst_coord + 1) * vchunk_, vnum_));
   }
 
   vertices_t get_master_vertices(fid_t fid) const {
-    int64_t master_vchunk = (vnum_ + fnum_ - 1) / fnum_;
-    return vertices_t(fid * master_vchunk,
-                      std::min((fid + 1) * master_vchunk, vnum_));
+    return vertices_t(fid * master_vchunk_,
+                      std::min((fid + 1) * master_vchunk_, vnum_));
   }
 
   int64_t get_total_vertices_num() const { return vnum_; }
@@ -322,7 +322,7 @@ struct VCPartitioner<int64_t> {
   template <typename IOADAPTOR_T>
   void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
     InArchive arc;
-    arc << fnum_ << fnum_sr_ << vnum_ << vchunk_;
+    arc << fnum_ << fnum_sr_ << vnum_ << master_vchunk_ << vchunk_;
     CHECK(writer->WriteArchive(arc));
   }
 
@@ -330,13 +330,14 @@ struct VCPartitioner<int64_t> {
   void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
     OutArchive arc;
     CHECK(reader->ReadArchive(arc));
-    arc >> fnum_ >> fnum_sr_ >> vnum_ >> vchunk_;
+    arc >> fnum_ >> fnum_sr_ >> vnum_ >> master_vchunk_ >> vchunk_;
   }
 
  private:
   fid_t fnum_;
   fid_t fnum_sr_;
   int64_t vnum_;
+  int64_t master_vchunk_;
   int64_t vchunk_;
 };
 

From 743df58afa2a0906a8b5c6f9e9dbd93ee6f99032 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Fri, 13 Sep 2024 15:01:01 +0800
Subject: [PATCH 03/15] double scan to reduce memory usage

---
 examples/analytical_apps/flags.cc            |   1 +
 examples/analytical_apps/flags.h             |   1 +
 examples/analytical_apps/run_app_vc.h        |   1 +
 grape/communication/shuffle.h                |  11 +
 grape/fragment/basic_fragment_loader_base.h  |   2 +
 grape/fragment/basic_vc_ds_fragment_loader.h | 312 +++++++++++++++++++
 grape/fragment/vc_fragment_loader.h          | 141 ++++++++-
 grape/utils/memory_inspector.h               |  11 +-
 grape/vertex_map/partitioner.h               |   2 +
 9 files changed, 469 insertions(+), 13 deletions(-)
 create mode 100644 grape/fragment/basic_vc_ds_fragment_loader.h

diff --git a/examples/analytical_apps/flags.cc b/examples/analytical_apps/flags.cc
index e6d9a93c..7d788e87 100644
--- a/examples/analytical_apps/flags.cc
+++ b/examples/analytical_apps/flags.cc
@@ -62,6 +62,7 @@ DEFINE_int32(app_concurrency, -1, "concurrency of application");
 DEFINE_int32(load_concurrency, 1, "concurrency of loading graph");
 
 DEFINE_bool(vc, false, "whether to use vertex-cut storage.");
+DEFINE_bool(single_scan_load, true, "whether to load graph in single scan.");
 
 DEFINE_string(lb, "cta",
               "Load balancing policy, these options can be used: "
diff --git a/examples/analytical_apps/flags.h b/examples/analytical_apps/flags.h
index 6f60d0b7..4fdd5413 100644
--- a/examples/analytical_apps/flags.h
+++ b/examples/analytical_apps/flags.h
@@ -54,6 +54,7 @@ DECLARE_int32(app_concurrency);
 DECLARE_int32(load_concurrency);
 
 DECLARE_bool(vc);
+DECLARE_bool(single_scan_load);
 
 DECLARE_string(lb);
 #endif  // EXAMPLES_ANALYTICAL_APPS_FLAGS_H_
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index bd062863..efd573d9 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -33,6 +33,7 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
   } else if (FLAGS_serialize) {
     graph_spec.set_serialize(true, FLAGS_serialization_prefix);
   }
+  graph_spec.single_scan = FLAGS_single_scan_load;
 
   using FRAG_T = ImmutableVertexcutFragment<int64_t, uint32_t, grape::EmptyType,
                                             grape::EmptyType>;
diff --git a/grape/communication/shuffle.h b/grape/communication/shuffle.h
index 5559795c..264edf3f 100644
--- a/grape/communication/shuffle.h
+++ b/grape/communication/shuffle.h
@@ -105,6 +105,12 @@ struct ShuffleBufferTuple : public ShuffleBufferTuple<Rest...> {
                      typename ShuffleBuffer<Rest>::type&&... bx)
       : first(std::move(b0)), ShuffleBufferTuple<Rest...>(std::move(bx)...) {}
 
+  ShuffleBufferTuple& operator=(ShuffleBufferTuple&& rhs) {
+    *static_cast<ShuffleBufferTuple<Rest...>*>(this) = std::move(rhs);
+    first = std::move(rhs.first);
+    return *this;
+  }
+
   static constexpr size_t tuple_size =
       ShuffleBufferTuple<Rest...>::tuple_size + 1;
 
@@ -179,6 +185,11 @@ struct ShuffleBufferTuple<First> {
   explicit ShuffleBufferTuple(typename ShuffleBuffer<First>::type&& b0)
       : first(std::move(b0)) {}
 
+  ShuffleBufferTuple& operator=(ShuffleBufferTuple&& rhs) {
+    first = std::move(rhs.first);
+    return *this;
+  }
+
   static constexpr size_t tuple_size = 1;
 
   void Emplace(const First& v0) { first.emplace_back(v0); }
diff --git a/grape/fragment/basic_fragment_loader_base.h b/grape/fragment/basic_fragment_loader_base.h
index bf689f7e..473106b8 100644
--- a/grape/fragment/basic_fragment_loader_base.h
+++ b/grape/fragment/basic_fragment_loader_base.h
@@ -42,6 +42,7 @@ struct LoadGraphSpec {
   IdxerType idxer_type;
 
   int load_concurrency;
+  bool single_scan;
 
   void set_directed(bool val = true) { directed = val; }
   void set_rebalance(bool flag, int weight) {
@@ -103,6 +104,7 @@ inline LoadGraphSpec DefaultLoadGraphSpec() {
   spec.partitioner_type = PartitionerType::kHashPartitioner;
   spec.idxer_type = IdxerType::kHashMapIdxer;
   spec.load_concurrency = 1;
+  spec.single_scan = true;
   return spec;
 }
 
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
new file mode 100644
index 00000000..b0f41bfc
--- /dev/null
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -0,0 +1,312 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
+#define GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
+
+#include "grape/utils/memory_inspector.h"
+
+namespace grape {
+
+template <typename OID_T>
+class VCEdgeBucketer {
+  struct bucket_info {
+    OID_T src_start;
+    OID_T src_chunk;
+    OID_T dst_start;
+    OID_T dst_chunk;
+  };
+
+ public:
+  VCEdgeBucketer(const VCPartitioner<OID_T>& partitioner, int bucket_num)
+      : partitioner_(partitioner), bucket_num_(bucket_num) {
+    fid_t fnum = partitioner_.fnum();
+    for (fid_t i = 0; i < fnum; ++i) {
+      auto src_vertices = partitioner_.get_src_vertices(i);
+      OID_T src_start = src_vertices.begin_value();
+      OID_T src_chunk = (src_vertices.size() + bucket_num - 1) / bucket_num;
+
+      auto dst_vertices = partitioner_.get_dst_vertices(i);
+      OID_T dst_start = dst_vertices.begin_value();
+      OID_T dst_chunk = (dst_vertices.size() + bucket_num - 1) / bucket_num;
+
+      bucket_infos_.emplace_back(
+          bucket_info{src_start, src_chunk, dst_start, dst_chunk});
+    }
+  }
+
+  int get_bucket_id(const OID_T& src, const OID_T& dst) const {
+    fid_t fid = partitioner_.get_edge_partition(src, dst);
+    auto& info = bucket_infos_[fid];
+    int src_bucket_id = (src - info.src_start) / info.src_chunk;
+    int dst_bucket_id = (dst - info.dst_start) / info.dst_chunk;
+    return src_bucket_id * bucket_num_ + dst_bucket_id;
+  }
+
+  std::pair<fid_t, int> get_partition_bucket_id(const OID_T& src,
+                                                const OID_T& dst) const {
+    fid_t fid = partitioner_.get_edge_partition(src, dst);
+    auto& info = bucket_infos_[fid];
+    int src_bucket_id = (src - info.src_start) / info.src_chunk;
+    int dst_bucket_id = (dst - info.dst_start) / info.dst_chunk;
+    return std::make_pair(fid, src_bucket_id * bucket_num_ + dst_bucket_id);
+  }
+
+ private:
+  const VCPartitioner<OID_T>& partitioner_;
+  int bucket_num_;
+  std::vector<bucket_info> bucket_infos_;
+};
+
+template <typename FRAG_T>
+class BasicVCDSFragmentLoader {
+  using fragment_t = FRAG_T;
+  using oid_t = typename fragment_t::oid_t;
+  using vid_t = typename fragment_t::vid_t;
+  using vdata_t = typename fragment_t::vdata_t;
+  using edata_t = typename fragment_t::edata_t;
+  using edge_t = typename fragment_t::edge_t;
+  using vertices_t = typename fragment_t::vertices_t;
+
+  static constexpr size_t shuffle_out_size = 4096000;
+
+ public:
+  explicit BasicVCDSFragmentLoader(const CommSpec& comm_spec, int64_t vnum,
+                                   int load_concurrency, int bucket_num)
+      : comm_spec_(comm_spec),
+        partitioner_(comm_spec.fnum(), vnum),
+        bucket_num_(bucket_num),
+        bucketer_(partitioner_, bucket_num),
+        vnum_(vnum),
+        load_concurrency_(load_concurrency) {
+    comm_spec_.Dup();
+    partition_bucket_edge_num_.resize(comm_spec_.fnum());
+    for (auto& vec : partition_bucket_edge_num_) {
+      vec.resize(bucket_num * bucket_num, 0);
+    }
+  }
+
+  ~BasicVCDSFragmentLoader() {
+    if (recv_thread_running_) {
+      for (auto& e : edges_to_frag_) {
+        e.Flush();
+      }
+      edge_recv_thread_.join();
+      got_edges_.DecProducerNum();
+      for (auto& thrd : edge_move_threads_) {
+        thrd.join();
+      }
+    }
+  }
+
+  void RecordEdge(const oid_t& src, const oid_t& dst) {
+    auto pair = bucketer_.get_partition_bucket_id(src, dst);
+    ++partition_bucket_edge_num_[pair.first][pair.second];
+  }
+
+  void AllocateBuffers() {
+    for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+      if (comm_spec_.FragToWorker(fid) == comm_spec_.worker_id()) {
+        MPI_Reduce(MPI_IN_PLACE, partition_bucket_edge_num_[fid].data(),
+                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+      } else {
+        MPI_Reduce(partition_bucket_edge_num_[fid].data(), nullptr,
+                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+      }
+    }
+    size_t edge_num = 0;
+    bucket_cursor_ =
+        std::vector<std::atomic<size_t>>(bucket_num_ * bucket_num_);
+    for (int k = 0; k < bucket_num_ * bucket_num_; ++k) {
+      bucket_cursor_[k].store(edge_num);
+      edge_num += partition_bucket_edge_num_[comm_spec_.fid()][k];
+    }
+    edges_.resize(edge_num);
+    MemoryInspector::GetInstance().allocate((sizeof(edge_t)) * edge_num);
+
+    got_edges_.SetProducerNum(2);
+
+    edges_to_frag_.resize(comm_spec_.fnum());
+    for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+      int worker_id = comm_spec_.FragToWorker(fid);
+      edges_to_frag_[fid].Init(comm_spec_.comm(), edge_tag, shuffle_out_size);
+      edges_to_frag_[fid].SetDestination(worker_id, fid);
+      if (worker_id == comm_spec_.worker_id()) {
+        edges_to_frag_[fid].DisableComm();
+      }
+    }
+    for (int i = 0; i < load_concurrency_; ++i) {
+      edge_move_threads_.emplace_back([this] {
+        ShuffleBufferTuple<oid_t, oid_t, edata_t> cur;
+        std::vector<std::vector<edge_t>> edges_cache(bucket_num_ * bucket_num_);
+        MemoryInspector::GetInstance().allocate(
+            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+        while (got_edges_.Get(cur)) {
+          size_t cur_size = cur.size();
+          foreach_rval(cur, [&edges_cache, this](oid_t src, oid_t dst,
+                                                 edata_t data) {
+            int bucket_id = bucketer_.get_bucket_id(src, dst);
+            edges_cache[bucket_id].emplace_back(src, dst, data);
+            if (edges_cache[bucket_id].size() >= 4096) {
+              size_t cursor = bucket_cursor_[bucket_id].fetch_add(
+                  edges_cache[bucket_id].size());
+              std::copy(edges_cache[bucket_id].begin(),
+                        edges_cache[bucket_id].end(), edges_.begin() + cursor);
+              edges_cache[bucket_id].clear();
+            }
+          });
+          MemoryInspector::GetInstance().deallocate(
+              (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur_size);
+        }
+        for (int i = 0; i < bucket_num_ * bucket_num_; ++i) {
+          if (!edges_cache[i].empty()) {
+            size_t cursor = bucket_cursor_[i].fetch_add(edges_cache[i].size());
+            std::copy(edges_cache[i].begin(), edges_cache[i].end(),
+                      edges_.begin() + cursor);
+          }
+        }
+        MemoryInspector::GetInstance().deallocate(
+            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+      });
+    }
+    edge_recv_thread_ =
+        std::thread(&BasicVCDSFragmentLoader::edgeRecvRoutine, this);
+    recv_thread_running_ = true;
+    MemoryInspector::GetInstance().allocate(
+        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
+        shuffle_out_size);
+  }
+
+  void AddEdge(const oid_t& src, const oid_t& dst, const edata_t& data) {
+    fid_t fid = partitioner_.get_edge_partition(src, dst);
+    edges_to_frag_[fid].Emplace(src, dst, data);
+    if (fid == comm_spec_.fid() &&
+        edges_to_frag_[fid].buffers().size() >= shuffle_out_size) {
+      MemoryInspector::GetInstance().allocate(
+          (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
+          edges_to_frag_[fid].buffers().size());
+      got_edges_.Put(std::move(edges_to_frag_[fid].buffers()));
+      edges_to_frag_[fid].Clear();
+    }
+  }
+
+  void ConstructFragment(std::shared_ptr<fragment_t>& fragment) {
+    for (auto& e : edges_to_frag_) {
+      e.Flush();
+    }
+    got_edges_.Put(std::move(edges_to_frag_[comm_spec_.fid()].buffers()));
+    edge_recv_thread_.join();
+    recv_thread_running_ = false;
+    edges_to_frag_.clear();
+    MemoryInspector::GetInstance().deallocate(
+        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
+        shuffle_out_size);
+    got_edges_.DecProducerNum();
+    for (auto& thrd : edge_move_threads_) {
+      thrd.join();
+    }
+
+    MPI_Barrier(comm_spec_.comm());
+
+    std::vector<size_t> bucket_edge_offset;
+    size_t en = 0;
+    for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
+      bucket_edge_offset.emplace_back(en);
+      en += partition_bucket_edge_num_[comm_spec_.fid()][i];
+    }
+    bucket_edge_offset.emplace_back(en);
+
+    {
+      std::atomic<int> d2_bucket_idx(0);
+      int d2_bucket_num = bucket_num_ * bucket_num_;
+      std::vector<std::thread> sort_threads;
+      for (int i = 0; i < load_concurrency_; ++i) {
+        sort_threads.emplace_back(
+            [this, &d2_bucket_idx, d2_bucket_num,
+             &bucket_edge_offset](int tid) {
+              while (true) {
+                int idx = d2_bucket_idx.fetch_add(1);
+                if (idx >= d2_bucket_num) {
+                  break;
+                }
+                std::sort(edges_.begin() + bucket_edge_offset[idx],
+                          edges_.begin() + bucket_edge_offset[idx + 1],
+                          [](const edge_t& a, const edge_t& b) {
+                            return a.src < b.src ||
+                                   (a.src == b.src && a.dst < b.dst);
+                          });
+              }
+            },
+            i);
+      }
+      for (auto& thrd : sort_threads) {
+        thrd.join();
+      }
+    }
+
+    fragment.reset(new fragment_t());
+    fragment->Init(comm_spec_, vnum_, std::move(edges_), bucket_num_,
+                   std::move(bucket_edge_offset));
+  }
+
+ private:
+  void edgeRecvRoutine() {
+    ShuffleIn<oid_t, oid_t, edata_t> data_in;
+    data_in.Init(comm_spec_.fnum(), comm_spec_.comm(), edge_tag);
+    fid_t dst_fid;
+    int src_worker_id;
+    while (!data_in.Finished()) {
+      src_worker_id = data_in.Recv(dst_fid);
+      if (src_worker_id == -1) {
+        break;
+      }
+      CHECK_EQ(dst_fid, comm_spec_.fid());
+      MemoryInspector::GetInstance().allocate(
+          (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
+          data_in.buffers().size());
+      got_edges_.Put(std::move(data_in.buffers()));
+      data_in.Clear();
+    }
+    got_edges_.DecProducerNum();
+  }
+
+  std::vector<std::vector<size_t>> partition_bucket_edge_num_;
+
+  CommSpec comm_spec_;
+  VCPartitioner<oid_t> partitioner_;
+  int bucket_num_;
+  VCEdgeBucketer<oid_t> bucketer_;
+
+  int64_t vnum_;
+  int load_concurrency_;
+
+  std::vector<std::atomic<size_t>> bucket_cursor_;
+  std::vector<edge_t> edges_;
+
+  std::vector<ShuffleOut<oid_t, oid_t, edata_t>> edges_to_frag_;
+  std::thread edge_recv_thread_;
+  bool recv_thread_running_;
+
+  BlockingQueue<ShuffleBufferTuple<oid_t, oid_t, edata_t>> got_edges_;
+  std::vector<std::thread> edge_move_threads_;
+
+  static constexpr int edge_tag = 6;
+};
+
+}  // namespace grape
+
+#endif  // GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
\ No newline at end of file
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
index a03ae2a1..b91e2e02 100644
--- a/grape/fragment/vc_fragment_loader.h
+++ b/grape/fragment/vc_fragment_loader.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef GRAPE_FRAGMENT_VC_FRAGMENT_LOADER_H_
 #define GRAPE_FRAGMENT_VC_FRAGMENT_LOADER_H_
 
+#include "grape/fragment/basic_vc_ds_fragment_loader.h"
 #include "grape/fragment/basic_vc_fragment_loader.h"
 
 namespace grape {
@@ -38,7 +39,7 @@ class VCFragmentLoader {
 
  public:
   explicit VCFragmentLoader(const CommSpec& comm_spec)
-      : comm_spec_(comm_spec), basic_fragment_loader_(nullptr) {}
+      : comm_spec_(comm_spec) {}
   ~VCFragmentLoader() = default;
 
   std::shared_ptr<fragment_t> LoadFragment(int64_t vnum,
@@ -52,10 +53,29 @@ class VCFragmentLoader {
       return fragment;
     }
 
-    basic_fragment_loader_ = std::unique_ptr<BasicVCFragmentLoader<fragment_t>>(
-        new BasicVCFragmentLoader<fragment_t>(
-            comm_spec_, vnum, spec.load_concurrency,
-            std::thread::hardware_concurrency()));
+    if (spec.single_scan) {
+      fragment = loadFragmentSingleScan(vnum, efile, spec);
+    } else {
+      fragment = loadFragmentDoubleScan(vnum, efile, spec);
+    }
+
+    if (spec.serialize) {
+      fragment->template Serialize<IOADAPTOR_T>(spec.serialization_prefix);
+    }
+
+    return fragment;
+  }
+
+ private:
+  std::shared_ptr<fragment_t> loadFragmentSingleScan(
+      int64_t vnum, const std::string& efile, const LoadGraphSpec& spec) {
+    std::shared_ptr<fragment_t> fragment(nullptr);
+
+    auto basic_fragment_loader =
+        std::unique_ptr<BasicVCFragmentLoader<fragment_t>>(
+            new BasicVCFragmentLoader<fragment_t>(
+                comm_spec_, vnum, spec.load_concurrency,
+                std::thread::hardware_concurrency()));
 
     auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
     io_adaptor->SetPartialRead(comm_spec_.worker_id(), comm_spec_.worker_num());
@@ -83,7 +103,7 @@ class VCFragmentLoader {
         continue;
       }
 
-      basic_fragment_loader_->AddEdge(src, dst, e_data);
+      basic_fragment_loader->AddEdge(src, dst, e_data);
     }
     io_adaptor->Close();
 
@@ -94,7 +114,7 @@ class VCFragmentLoader {
     }
 
     double t2 = -grape::GetCurrentTime();
-    basic_fragment_loader_->ConstructFragment(fragment);
+    basic_fragment_loader->ConstructFragment(fragment);
     MPI_Barrier(comm_spec_.comm());
     t2 += grape::GetCurrentTime();
 
@@ -105,18 +125,115 @@ class VCFragmentLoader {
             << MemoryInspector::GetInstance().GetCurrentMemoryUsage() << " GB"
             << ", peak: " << MemoryInspector::GetInstance().GetPeakMemoryUsage()
             << " GB";
+    return fragment;
+  }
 
-    if (spec.serialize) {
-      fragment->template Serialize<IOADAPTOR_T>(spec.serialization_prefix);
+  std::shared_ptr<fragment_t> loadFragmentDoubleScan(
+      int64_t vnum, const std::string& efile, const LoadGraphSpec& spec) {
+    std::shared_ptr<fragment_t> fragment(nullptr);
+
+    auto basic_fragment_loader =
+        std::unique_ptr<BasicVCDSFragmentLoader<fragment_t>>(
+            new BasicVCDSFragmentLoader<fragment_t>(
+                comm_spec_, vnum, spec.load_concurrency,
+                std::thread::hardware_concurrency()));
+    {
+      auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
+      io_adaptor->SetPartialRead(comm_spec_.worker_id(),
+                                 comm_spec_.worker_num());
+      io_adaptor->Open();
+
+      std::string line;
+      edata_t e_data;
+      oid_t src, dst;
+
+      double t0 = -grape::GetCurrentTime();
+      size_t lineNo = 0;
+      while (io_adaptor->ReadLine(line)) {
+        ++lineNo;
+        if (lineNo % 1000000 == 0) {
+          VLOG(10) << "[worker-" << comm_spec_.worker_id()
+                   << "][efile - scan 1 / 2] " << lineNo;
+        }
+        if (line.empty() || line[0] == '#')
+          continue;
+
+        try {
+          line_parser_.LineParserForEFile(line, src, dst, e_data);
+        } catch (std::exception& e) {
+          VLOG(1) << e.what();
+          continue;
+        }
+
+        basic_fragment_loader->RecordEdge(src, dst);
+      }
+      io_adaptor->Close();
+
+      MPI_Barrier(comm_spec_.comm());
+      t0 += grape::GetCurrentTime();
+      if (comm_spec_.worker_id() == 0) {
+        VLOG(1) << "finished first scan of edges inputs, time: " << t0 << " s";
+      }
     }
 
+    basic_fragment_loader->AllocateBuffers();
+    MPI_Barrier(comm_spec_.comm());
+
+    {
+      auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
+      io_adaptor->SetPartialRead(comm_spec_.worker_id(),
+                                 comm_spec_.worker_num());
+      io_adaptor->Open();
+
+      std::string line;
+      edata_t e_data;
+      oid_t src, dst;
+
+      double t0 = -grape::GetCurrentTime();
+      size_t lineNo = 0;
+      while (io_adaptor->ReadLine(line)) {
+        ++lineNo;
+        if (lineNo % 1000000 == 0) {
+          VLOG(10) << "[worker-" << comm_spec_.worker_id()
+                   << "][efile - scan 2 / 2] " << lineNo;
+        }
+        if (line.empty() || line[0] == '#')
+          continue;
+
+        try {
+          line_parser_.LineParserForEFile(line, src, dst, e_data);
+        } catch (std::exception& e) {
+          VLOG(1) << e.what();
+          continue;
+        }
+
+        basic_fragment_loader->AddEdge(src, dst, e_data);
+      }
+      io_adaptor->Close();
+
+      MPI_Barrier(comm_spec_.comm());
+      t0 += grape::GetCurrentTime();
+      if (comm_spec_.worker_id() == 0) {
+        VLOG(1) << "finished second scan of edges inputs, time: " << t0 << " s";
+      }
+    }
+
+    double t2 = -grape::GetCurrentTime();
+    basic_fragment_loader->ConstructFragment(fragment);
+    MPI_Barrier(comm_spec_.comm());
+    t2 += grape::GetCurrentTime();
+
+    if (comm_spec_.worker_id() == 0) {
+      VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
+    }
+    VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
+            << MemoryInspector::GetInstance().GetCurrentMemoryUsage() << " GB"
+            << ", peak: " << MemoryInspector::GetInstance().GetPeakMemoryUsage()
+            << " GB";
     return fragment;
   }
 
- private:
   CommSpec comm_spec_;
-
-  std::unique_ptr<BasicVCFragmentLoader<fragment_t>> basic_fragment_loader_;
   line_parser_t line_parser_;
 };
 
diff --git a/grape/utils/memory_inspector.h b/grape/utils/memory_inspector.h
index ad7190b9..3447878f 100644
--- a/grape/utils/memory_inspector.h
+++ b/grape/utils/memory_inspector.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define GRAPE_UTILS_MEMORY_INSPECTOR_H_
 
 #include "glog/logging.h"
+#include "grape/utils/concurrent_queue.h"
 
 namespace grape {
 
@@ -27,11 +28,17 @@ struct MemoryInspector {
   }
 
   void allocate(size_t size) {
+    lock_.lock();
     current_memory_usage += size;
     peak_memory_usage = std::max(current_memory_usage, peak_memory_usage);
+    lock_.unlock();
   }
 
-  void deallocate(size_t size) { current_memory_usage -= size; }
+  void deallocate(size_t size) {
+    lock_.lock();
+    current_memory_usage -= size;
+    lock_.unlock();
+  }
 
   ~MemoryInspector() {
     double peak_memory_usage_gb = peak_memory_usage / 1024.0 / 1024.0 / 1024.0;
@@ -50,6 +57,8 @@ struct MemoryInspector {
 
   size_t current_memory_usage = 0;
   size_t peak_memory_usage = 0;
+
+  SpinLock lock_;
 };
 
 }  // namespace grape
diff --git a/grape/vertex_map/partitioner.h b/grape/vertex_map/partitioner.h
index ba1e3b74..0210e755 100644
--- a/grape/vertex_map/partitioner.h
+++ b/grape/vertex_map/partitioner.h
@@ -333,6 +333,8 @@ struct VCPartitioner<int64_t> {
     arc >> fnum_ >> fnum_sr_ >> vnum_ >> master_vchunk_ >> vchunk_;
   }
 
+  fid_t fnum() const { return fnum_; }
+
  private:
   fid_t fnum_;
   fid_t fnum_sr_;

From f1a9f469b349b8b6ad01b54a10f14f7b615eb112 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Fri, 13 Sep 2024 17:46:39 +0800
Subject: [PATCH 04/15] xxx

---
 .../analytical_apps/pagerank/pagerank_vc.h    |  24 +-
 .../pagerank/pagerank_vc_context.h            |  38 ++-
 examples/analytical_apps/run_app.cc           |  16 +-
 .../append_only_edgecut_fragment.h            |  12 +-
 ...c_app_base.h => gather_scatter_app_base.h} |  30 ++-
 grape/app/vertex_data_context.h               |  32 ++-
 grape/cuda/fragment/host_fragment.h           |   2 +-
 grape/fragment/edgecut_fragment_base.h        | 217 +++++++++++++++-
 grape/fragment/fragment_base.h                | 243 +-----------------
 grape/fragment/immutable_edgecut_fragment.h   |   7 +-
 grape/fragment/immutable_vertexcut_fragment.h |  42 +--
 grape/fragment/mutable_edgecut_fragment.h     |  37 +--
 grape/grape.h                                 |   5 +-
 ...ger.h => gather_scatter_message_manager.h} |  12 +-
 grape/types.h                                 |  24 +-
 grape/worker/vc_worker.h                      | 117 ---------
 grape/worker/worker.h                         |   4 +
 17 files changed, 394 insertions(+), 468 deletions(-)
 rename grape/app/{vc_app_base.h => gather_scatter_app_base.h} (63%)
 rename grape/parallel/{vc_message_manager.h => gather_scatter_message_manager.h} (96%)
 delete mode 100644 grape/worker/vc_worker.h

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index a7bd7245..a60a824f 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -30,13 +30,15 @@ struct NumericSum {
 };
 
 template <typename FRAG_T>
-class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
-                   public ParallelEngine,
-                   public Communicator {
+class PageRankVC
+    : public GatherScatterAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
+      public ParallelEngine,
+      public Communicator {
   using vertex_t = Vertex<typename FRAG_T::oid_t>;
 
  public:
-  INSTALL_VC_WORKER(PageRankVC<FRAG_T>, PageRankVCContext<FRAG_T>, FRAG_T)
+  INSTALL_GATHER_SCATTER_WORKER(PageRankVC<FRAG_T>, PageRankVCContext<FRAG_T>,
+                                FRAG_T)
 
   void PEval(const fragment_t& frag, context_t& ctx,
              message_manager_t& messages) {
@@ -50,9 +52,9 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
     {
       ctx.t0 -= GetCurrentTime();
       typename fragment_t::template both_vertex_array_t<int> degree(
-          frag.GetVertices(), 0);
+          frag.Vertices(), 0);
       // allocate degree array for both src and dst vertices
-      MemoryInspector::GetInstance().allocate(frag.GetVertices().size() *
+      MemoryInspector::GetInstance().allocate(frag.Vertices().size() *
                                               sizeof(int));
       int bucket_num = frag.GetBucketNum();
       int concurrency = thread_num();
@@ -84,7 +86,7 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
       messages.GatherMasterVertices<fragment_t, int, NumericSum<int>>(
           frag, degree, ctx.master_degree);
       // deallocate degree array for both src and dst vertices
-      MemoryInspector::GetInstance().deallocate(frag.GetVertices().size() *
+      MemoryInspector::GetInstance().deallocate(frag.Vertices().size() *
                                                 sizeof(int));
     }
 
@@ -92,7 +94,7 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
     int64_t dangling_vnum_local = 0;
     ctx.t2 -= GetCurrentTime();
     std::vector<int64_t> dangling_vnum_local_vec(thread_num(), 0);
-    ForEach(frag.GetMasterVertices(), [&](int tid, vertex_t v) {
+    ForEach(frag.MasterVertices(), [&](int tid, vertex_t v) {
       if (ctx.master_degree[v] == 0) {
         ++dangling_vnum_local_vec[tid];
         ctx.master_result[v] = p;
@@ -123,7 +125,7 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
     ctx.dangling_sum = base * ctx.total_dangling_vnum;
 
     ctx.t11 -= GetCurrentTime();
-    ForEach(frag.GetVertices(),
+    ForEach(frag.Vertices(),
             [&ctx](int tid, vertex_t v) { ctx.next_result[v] = 0; });
     ctx.t11 += GetCurrentTime();
 
@@ -166,7 +168,7 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
           frag, ctx.next_result, ctx.master_result);
 
       ctx.t2 -= GetCurrentTime();
-      ForEach(frag.GetMasterVertices(), [&ctx, base](int tid, vertex_t v) {
+      ForEach(frag.MasterVertices(), [&ctx, base](int tid, vertex_t v) {
         if (ctx.master_degree[v] > 0) {
           ctx.master_result[v] =
               (base + ctx.delta * ctx.master_result[v]) / ctx.master_degree[v];
@@ -218,7 +220,7 @@ class PageRankVC : public VCAppBase<FRAG_T, PageRankVCContext<FRAG_T>>,
           frag, ctx.next_result, ctx.master_result);
 
       ctx.t2 -= GetCurrentTime();
-      ForEach(frag.GetMasterVertices(), [&ctx, base](int tid, vertex_t v) {
+      ForEach(frag.MasterVertices(), [&ctx, base](int tid, vertex_t v) {
         ctx.master_result[v] = ctx.master_result[v] * ctx.delta + base;
       });
       ctx.t2 += GetCurrentTime();
diff --git a/examples/analytical_apps/pagerank/pagerank_vc_context.h b/examples/analytical_apps/pagerank/pagerank_vc_context.h
index 11bf766c..0faf7a5b 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc_context.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc_context.h
@@ -23,29 +23,29 @@ limitations under the License.
 namespace grape {
 
 template <typename FRAG_T>
-class PageRankVCContext {
+class PageRankVCContext : public VertexDataContext<FRAG_T, double> {
   using oid_t = typename FRAG_T::oid_t;
 
  public:
-  const FRAG_T& fragment() { return fragment_; }
-
-  explicit PageRankVCContext(const FRAG_T& fragment) : fragment_(fragment) {
-    curr_result.Init(fragment.GetVertices());
-    next_result.Init(fragment.GetVertices());
-    master_result.Init(fragment.GetMasterVertices());
-    master_degree.Init(fragment.GetMasterVertices());
-
-    MemoryInspector::GetInstance().allocate(fragment.GetVertices().size() *
+  explicit PageRankVCContext(const FRAG_T& fragment)
+      : VertexDataContext<FRAG_T, double>(fragment),
+        master_result(this->data()) {
+    curr_result.Init(fragment.Vertices());
+    next_result.Init(fragment.Vertices());
+    master_degree.Init(fragment.MasterVertices());
+
+    MemoryInspector::GetInstance().allocate(fragment.Vertices().size() *
                                             sizeof(double));
-    MemoryInspector::GetInstance().allocate(fragment.GetVertices().size() *
+    MemoryInspector::GetInstance().allocate(fragment.Vertices().size() *
                                             sizeof(double));
-    MemoryInspector::GetInstance().allocate(
-        fragment.GetMasterVertices().size() * sizeof(double));
-    MemoryInspector::GetInstance().allocate(
-        fragment.GetMasterVertices().size() * sizeof(int));
+    MemoryInspector::GetInstance().allocate(fragment.MasterVertices().size() *
+                                            sizeof(double));
+    MemoryInspector::GetInstance().allocate(fragment.MasterVertices().size() *
+                                            sizeof(int));
   }
 
-  void Init(VCMessageManager& messages, double delta, int max_round) {
+  void Init(GatherScatterMessageManager& messages, double delta,
+            int max_round) {
     this->delta = delta;
     this->max_round = max_round;
     step = 0;
@@ -53,7 +53,7 @@ class PageRankVCContext {
 
   void Output(std::ostream& os) {
     auto& frag = this->fragment();
-    auto master_vertices = frag.GetMasterVertices();
+    auto master_vertices = frag.MasterVertices();
     for (auto v : master_vertices) {
       os << v.GetValue() << " " << std::scientific << std::setprecision(15)
          << master_result[v] << std::endl;
@@ -68,7 +68,7 @@ class PageRankVCContext {
 
   typename FRAG_T::template both_vertex_array_t<double> curr_result;
   typename FRAG_T::template both_vertex_array_t<double> next_result;
-  typename FRAG_T::template vertex_array_t<double> master_result;
+  typename FRAG_T::template vertex_array_t<double>& master_result;
   typename FRAG_T::template vertex_array_t<int> master_degree;
 
   int64_t total_dangling_vnum = 0;
@@ -79,8 +79,6 @@ class PageRankVCContext {
 
   double dangling_sum = 0.0;
 
-  const FRAG_T& fragment_;
-
   double t0 = 0;   // init degree
   double t2 = 0;   // calc master result
   double t4 = 0;   // propogate
diff --git a/examples/analytical_apps/run_app.cc b/examples/analytical_apps/run_app.cc
index f1c445b2..7af79bd8 100644
--- a/examples/analytical_apps/run_app.cc
+++ b/examples/analytical_apps/run_app.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// #include "run_app.h"
+#include "run_app.h"
 
 #include <gflags/gflags.h>
 #include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 
-// #include "run_app_opt.h"
+#include "run_app_opt.h"
 #include "run_app_vc.h"
 
 int main(int argc, char* argv[]) {
@@ -41,15 +41,15 @@ int main(int argc, char* argv[]) {
 
   std::string name = FLAGS_application;
   if (FLAGS_opt) {
-    // grape::RunOpt();
+    grape::RunOpt();
   } else if (FLAGS_vc) {
     grape::RunVC();
   } else {
-    // if (name.find("sssp") != std::string::npos) {
-    //   grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
-    // } else {
-    //   grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
-    // }
+    if (name.find("sssp") != std::string::npos) {
+      grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
+    } else {
+      grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
+    }
   }
 
   grape::Finalize();
diff --git a/examples/gnn_sampler/append_only_edgecut_fragment.h b/examples/gnn_sampler/append_only_edgecut_fragment.h
index e3543114..87cffdc1 100644
--- a/examples/gnn_sampler/append_only_edgecut_fragment.h
+++ b/examples/gnn_sampler/append_only_edgecut_fragment.h
@@ -323,8 +323,7 @@ class AppendOnlyEdgecutFragment
   /** Constructor.
    * @param vm_ptr the vertex map.
    */
-  AppendOnlyEdgecutFragment()
-      : FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, traits_t>() {}
+  AppendOnlyEdgecutFragment() : FragmentBase<OID_T, VDATA_T, EDATA_T>() {}
 
   virtual ~AppendOnlyEdgecutFragment() {}
 
@@ -336,7 +335,7 @@ class AppendOnlyEdgecutFragment
   void Init(const CommSpec& comm_spec, bool directed,
             std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
             std::vector<internal_vertex_t>& vertices,
-            std::vector<edge_t>& edges) override {
+            std::vector<edge_t>& edges) {
     init(comm_spec.fid(), directed, std::move(vm_ptr));
 
     ovnum_ = 0;
@@ -482,6 +481,13 @@ class AppendOnlyEdgecutFragment
     initOuterVerticesOfFragment();
   }
 
+  void ParallelInit(const CommSpec& comm_spec, bool directed,
+                    std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
+                    std::vector<internal_vertex_t>& vertices,
+                    std::vector<edge_t>& edges, int concurrency) {
+    Init(comm_spec, directed, std::move(vm_ptr), vertices, edges);
+  }
+
   void InitIndices() {
     fragment_indices_ =
         FragmentIndicesBase<oid_t, vid_t, vdata_t, edata_t>::Create();
diff --git a/grape/app/vc_app_base.h b/grape/app/gather_scatter_app_base.h
similarity index 63%
rename from grape/app/vc_app_base.h
rename to grape/app/gather_scatter_app_base.h
index a52b92a1..d341a419 100644
--- a/grape/app/vc_app_base.h
+++ b/grape/app/gather_scatter_app_base.h
@@ -13,26 +13,32 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#ifndef GRAPE_APP_VC_APP_BASE_H_
-#define GRAPE_APP_VC_APP_BASE_H_
+#ifndef GRAPE_APP_GATHER_SCATTER_APP_BASE_H_
+#define GRAPE_APP_GATHER_SCATTER_APP_BASE_H_
 
 #include <memory>
 
 #include "grape/types.h"
-#include "grape/worker/vc_worker.h"
+#include "grape/worker/worker.h"
 
 namespace grape {
 
-class VCMessageManager;
+class GatherScatterMessageManager;
 
 template <typename FRAG_T, typename CONTEXT_T,
-          typename MESSAGE_MANAGER_T = VCMessageManager>
-class VCAppBase {
+          typename MESSAGE_MANAGER_T = GatherScatterMessageManager>
+class GatherScatterAppBase {
  public:
+  static constexpr bool need_split_edges = false;
+  static constexpr bool need_split_edges_by_fragment = false;
+  static constexpr MessageStrategy message_strategy =
+      MessageStrategy::kGatherScatter;
+  static constexpr LoadStrategy load_strategy = LoadStrategy::kOnlyOut;
+
   using message_manager_t = MESSAGE_MANAGER_T;
 
-  VCAppBase() = default;
-  virtual ~VCAppBase() = default;
+  GatherScatterAppBase() = default;
+  virtual ~GatherScatterAppBase() = default;
 
   virtual void PEval(const FRAG_T& graph, CONTEXT_T& context,
                      message_manager_t& messages) = 0;
@@ -41,12 +47,12 @@ class VCAppBase {
                        message_manager_t& messages) = 0;
 };
 
-#define INSTALL_VC_WORKER(APP_T, CONTEXT_T, FRAG_T)               \
+#define INSTALL_GATHER_SCATTER_WORKER(APP_T, CONTEXT_T, FRAG_T)   \
  public:                                                          \
   using fragment_t = FRAG_T;                                      \
   using context_t = CONTEXT_T;                                    \
-  using message_manager_t = grape::VCMessageManager;              \
-  using worker_t = grape::VCWorker<APP_T, message_manager_t>;     \
+  using message_manager_t = grape::GatherScatterMessageManager;   \
+  using worker_t = grape::GatherScatterWorker<APP_T>;             \
   virtual ~APP_T() {}                                             \
   static std::shared_ptr<worker_t> CreateWorker(                  \
       std::shared_ptr<APP_T> app, std::shared_ptr<FRAG_T> frag) { \
@@ -55,4 +61,4 @@ class VCAppBase {
 
 }  // namespace grape
 
-#endif  // GRAPE_APP_VC_APP_BASE_H_
+#endif  // GRAPE_APP_GATHER_SCATTER_APP_BASE_H_
diff --git a/grape/app/vertex_data_context.h b/grape/app/vertex_data_context.h
index f23c6d13..d526e59e 100644
--- a/grape/app/vertex_data_context.h
+++ b/grape/app/vertex_data_context.h
@@ -21,8 +21,14 @@ limitations under the License.
 
 namespace grape {
 
+template <typename FRAG_T, typename DATA_T, typename Enable = void>
+class VertexDataContext;
+
 template <typename FRAG_T, typename DATA_T>
-class VertexDataContext : public ContextBase {
+class VertexDataContext<
+    FRAG_T, DATA_T,
+    typename std::enable_if<is_edge_cut_fragment<FRAG_T>::value>::type>
+    : public ContextBase {
   using fragment_t = FRAG_T;
   using vertex_t = typename fragment_t::vertex_t;
   using vertex_array_t = typename fragment_t::template vertex_array_t<DATA_T>;
@@ -49,6 +55,30 @@ class VertexDataContext : public ContextBase {
   vertex_array_t data_;
 };
 
+template <typename FRAG_T, typename DATA_T>
+class VertexDataContext<
+    FRAG_T, DATA_T,
+    typename std::enable_if<is_vertex_cut_fragment<FRAG_T>::value>::type>
+    : public ContextBase {
+  using fragment_t = FRAG_T;
+  using vertex_array_t = typename fragment_t::template vertex_array_t<DATA_T>;
+
+ public:
+  using data_t = DATA_T;
+
+  explicit VertexDataContext(const fragment_t& fragment) : fragment_(fragment) {
+    data_.Init(fragment.MasterVertices());
+  }
+
+  const fragment_t& fragment() { return fragment_; }
+
+  inline vertex_array_t& data() { return data_; }
+
+ private:
+  const fragment_t& fragment_;
+  vertex_array_t data_;
+};
+
 }  // namespace grape
 
 #endif  // GRAPE_APP_VERTEX_DATA_CONTEXT_H_
diff --git a/grape/cuda/fragment/host_fragment.h b/grape/cuda/fragment/host_fragment.h
index dbb4ec8b..0b55f911 100644
--- a/grape/cuda/fragment/host_fragment.h
+++ b/grape/cuda/fragment/host_fragment.h
@@ -95,7 +95,7 @@ class HostFragment : public ImmutableEdgecutFragment<OID_T, VID_T, VDATA_T,
 
   static constexpr grape::LoadStrategy load_strategy = _load_strategy;
 
-  HostFragment() : FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, traits_t>() {}
+  HostFragment() : FragmentBase<OID_T, VDATA_T, EDATA_T>() {}
 
   void Init(const CommSpec& comm_spec, bool directed,
             std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
diff --git a/grape/fragment/edgecut_fragment_base.h b/grape/fragment/edgecut_fragment_base.h
index 55521f17..7e668795 100644
--- a/grape/fragment/edgecut_fragment_base.h
+++ b/grape/fragment/edgecut_fragment_base.h
@@ -42,14 +42,184 @@ namespace grape {
 template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T,
           typename TRAITS_T>
 class EdgecutFragmentBase
-    : virtual public FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, TRAITS_T> {
+    : virtual public FragmentBase<OID_T, VDATA_T, EDATA_T> {
  public:
-  using base_t = FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, TRAITS_T>;
+  using base_t = FragmentBase<OID_T, VDATA_T, EDATA_T>;
   using vid_t = VID_T;
   using vertex_t = Vertex<VID_T>;
+  using fragment_adj_list_t = typename TRAITS_T::fragment_adj_list_t;
+  using fragment_const_adj_list_t =
+      typename TRAITS_T::fragment_const_adj_list_t;
 
   EdgecutFragmentBase() {}
 
+  static constexpr FragmentType fragment_type = FragmentType::kEdgeCut;
+
+  VertexMap<OID_T, VID_T>& GetVertexMap() { return *vm_ptr_; }
+  const VertexMap<OID_T, VID_T>& GetVertexMap() const { return *vm_ptr_; }
+
+  using vertices_t = typename TRAITS_T::vertices_t;
+  /**
+   * @brief Get all vertices referenced to this fragment.
+   *
+   * @return A vertex set can be iterate on.
+   */
+  const vertices_t& Vertices() const { return vertices_; }
+
+  size_t GetVerticesNum() const override { return vertices_.size(); }
+
+  size_t GetTotalVerticesNum() const override {
+    return vm_ptr_->GetTotalVertexSize();
+  }
+
+  /**
+   * @brief Get a vertex with original ID vid.
+   *
+   * @param vid Original ID.
+   * @param v Got vertex.
+   *
+   * @return If find the vertex in this fragment, return true. Otherwise, return
+   * false.
+   */
+  bool GetVertex(const OID_T& oid, Vertex<VID_T>& v) const {
+    VID_T gid;
+    if (vm_ptr_->GetGid(oid, gid)) {
+      return Gid2Vertex(gid, v);
+    }
+    return false;
+  }
+
+  /**
+   * @brief Get the original ID of a vertex.
+   *
+   * @param v Input vertex.
+   *
+   * @return Its original ID.
+   */
+  OID_T GetId(const Vertex<VID_T>& v) const {
+    OID_T oid{};
+    vm_ptr_->GetOid(Vertex2Gid(v), oid);
+    return oid;
+  }
+
+  using internal_id_t = typename InternalOID<OID_T>::type;
+
+  internal_id_t GetInternalId(const Vertex<VID_T>& v) const {
+    internal_id_t oid{};
+    vm_ptr_->GetInternalOid(Vertex2Gid(v), oid);
+    return oid;
+  }
+
+  OID_T Gid2Oid(VID_T gid) const {
+    OID_T oid;
+    vm_ptr_->GetOid(gid, oid);
+    return oid;
+  }
+
+  /**
+   * @brief Get the ID of fragment the input vertex belongs to.
+   *
+   * @param u Input vertex.
+   *
+   * @return Its fragment ID.
+   */
+  fid_t GetFragId(const Vertex<VID_T>& u) const {
+    VID_T gid = Vertex2Gid(u);
+    return id_parser_.get_fragment_id(gid);
+  }
+
+  /**
+   * @brief Get the data of a vertex.
+   *
+   * @param v Input vertex.
+   *
+   * @return Data on it.
+   */
+  virtual const VDATA_T& GetData(const Vertex<VID_T>& v) const = 0;
+
+  /**
+   * @brief Set the data of a vertex.
+   *
+   * @param v Input vertex.
+   * @param val Data to write.
+   * @attention This will only be applied locally, won't sync on other mirrors
+   * globally.
+   */
+  virtual void SetData(const Vertex<VID_T>& v, const VDATA_T& val) = 0;
+
+  /**
+   * @brief Check if vertex v has a child, that is, existing an edge v->u.
+   *
+   * @param v Input vertex.
+   *
+   * @return True if vertex v has a child, false otherwise.
+   */
+  virtual bool HasChild(const Vertex<VID_T>& v) const = 0;
+
+  /**
+   * @brief Check if vertex v has a parent, that is, existing an edge u->v.
+   *
+   * @param v Input vertex.
+   *
+   * @return True if vertex v has a parent, false otherwise.
+   */
+  virtual bool HasParent(const Vertex<VID_T>& v) const = 0;
+
+  /**
+   * @brief Returns the in-degree of vertex v in this fragment.
+   *
+   * @param v Input vertex.
+   *
+   * @return In-degree of vertex v in this fragment.
+   */
+  virtual int GetLocalInDegree(const Vertex<VID_T>& v) const = 0;
+
+  /**
+   * @brief Returns the out-degree of vertex v in this fragment.<Paste>
+   *
+   * @param v Input vertex.
+   *
+   * @return Out-degree of vertex v in this fragment.
+   */
+  virtual int GetLocalOutDegree(const Vertex<VID_T>& v) const = 0;
+
+  /**
+   * @brief Returns the incoming adjacent vertices of v.
+   *
+   * @param v Input vertex.
+   *
+   * @return The incoming adjacent vertices of v.
+   */
+  virtual AdjList<VID_T, EDATA_T> GetIncomingAdjList(
+      const Vertex<VID_T>& v) = 0;
+
+  virtual ConstAdjList<VID_T, EDATA_T> GetIncomingAdjList(
+      const Vertex<VID_T>& v) const = 0;
+
+  virtual fragment_adj_list_t GetIncomingAdjList(const Vertex<VID_T>& v,
+                                                 fid_t fid) = 0;
+
+  virtual fragment_const_adj_list_t GetIncomingAdjList(const Vertex<VID_T>& v,
+                                                       fid_t fid) const = 0;
+  /**
+   * @brief Returns the outgoing adjacent vertices of v.
+   *
+   * @param v Input vertex.
+   *
+   * @return The outgoing adjacent vertices of v.
+   */
+  virtual AdjList<VID_T, EDATA_T> GetOutgoingAdjList(
+      const Vertex<VID_T>& v) = 0;
+
+  virtual ConstAdjList<VID_T, EDATA_T> GetOutgoingAdjList(
+      const Vertex<VID_T>& v) const = 0;
+
+  virtual fragment_adj_list_t GetOutgoingAdjList(const Vertex<VID_T>& v,
+                                                 fid_t fid) = 0;
+
+  virtual fragment_const_adj_list_t GetOutgoingAdjList(const Vertex<VID_T>& v,
+                                                       fid_t fid) const = 0;
+
   /**
    * @brief Returns the number of inner vertices in this fragment.
    *
@@ -341,16 +511,41 @@ class EdgecutFragmentBase
   using base_t::fid;
   using base_t::fnum;
 
-  bool Gid2Vertex(const vid_t& gid, vertex_t& v) const override {
+  /**
+   * @brief Convert from global id to a vertex handle.
+   *
+   * @param gid Input global id.
+   * @param v Output vertex handle.
+   *
+   * @return True if exists a vertex with global id as gid in this fragment,
+   * false otherwise.
+   */
+  bool Gid2Vertex(const vid_t& gid, vertex_t& v) const {
     return IsInnerVertexGid(gid) ? InnerVertexGid2Vertex(gid, v)
                                  : OuterVertexGid2Vertex(gid, v);
   }
 
-  vid_t Vertex2Gid(const vertex_t& v) const override {
+  /**
+   * @brief Convert from vertex handle to its global id.
+   *
+   * @param v Input vertex handle.
+   *
+   * @return Global id of the vertex.
+   */
+  vid_t Vertex2Gid(const vertex_t& v) const {
     return IsInnerVertex(v) ? GetInnerVertexGid(v) : GetOuterVertexGid(v);
   }
 
  protected:
+  void init(fid_t fid, bool directed,
+            std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr) {
+    fid_t fnum = vm_ptr->GetFragmentNum();
+    base_t::init(fid, fnum, directed);
+    vm_ptr_ = std::move(vm_ptr);
+    ivnum_ = vm_ptr_->GetInnerVertexSize(fid);
+    id_parser_.init(fnum);
+  }
+
   inline bool IsInnerVertexGid(VID_T gid) const {
     return id_parser_.get_fragment_id(gid) == fid();
   }
@@ -410,7 +605,7 @@ class EdgecutFragmentBase
   void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
     base_t::serialize(writer);
     InArchive arc;
-    arc << inner_vertices_ << outer_vertices_;
+    arc << ivnum_ << inner_vertices_ << outer_vertices_ << vertices_;
     CHECK(writer->WriteArchive(arc));
   }
 
@@ -419,15 +614,21 @@ class EdgecutFragmentBase
     base_t::deserialize(reader);
     OutArchive arc;
     CHECK(reader->ReadArchive(arc));
-    arc >> inner_vertices_ >> outer_vertices_;
+    arc >> ivnum_ >> inner_vertices_ >> outer_vertices_ >> vertices_;
+    id_parser_.init(fnum());
   }
 
+  VID_T ivnum_;
+
   inner_vertices_t inner_vertices_;
   outer_vertices_t outer_vertices_;
+  vertices_t vertices_;
+
   std::vector<sub_vertices_t> outer_vertices_of_frag_;
   std::vector<mirror_vertices_t> mirrors_of_frag_;
-  using base_t::id_parser_;
-  using base_t::vm_ptr_;
+
+  std::unique_ptr<VertexMap<OID_T, VID_T>> vm_ptr_;
+  IdParser<VID_T> id_parser_;
 };
 
 }  // namespace grape
diff --git a/grape/fragment/fragment_base.h b/grape/fragment/fragment_base.h
index 11d88e4e..7ef41b41 100644
--- a/grape/fragment/fragment_base.h
+++ b/grape/fragment/fragment_base.h
@@ -43,66 +43,23 @@ struct PrepareConf {
  * derived classes would be invoked directly, not via virtual functions.
  *
  * @tparam OID_T
- * @tparam VID_T
  * @tparam VDATA_T
  * @tparam EDATA_T
- * @tparam TRAITS_T
  */
-template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T,
-          typename TRAITS_T>
+template <typename OID_T, typename VDATA_T, typename EDATA_T>
 class FragmentBase {
  public:
-  using fragment_adj_list_t = typename TRAITS_T::fragment_adj_list_t;
-  using fragment_const_adj_list_t =
-      typename TRAITS_T::fragment_const_adj_list_t;
-
-  FragmentBase() : vm_ptr_(nullptr) {}
+  FragmentBase() {}
   virtual ~FragmentBase() {}
 
-  VertexMap<OID_T, VID_T>& GetVertexMap() { return *vm_ptr_; }
-  const VertexMap<OID_T, VID_T>& GetVertexMap() const { return *vm_ptr_; }
-
  protected:
-  void init(fid_t fid, bool directed,
-            std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr) {
+  void init(fid_t fid, fid_t fnum, bool directed) {
     fid_ = fid;
+    fnum_ = fnum;
     directed_ = directed;
-    fnum_ = vm_ptr->GetFragmentNum();
-    vm_ptr_ = std::move(vm_ptr);
-    id_parser_.init(fnum_);
-    ivnum_ = vm_ptr_->GetInnerVertexSize(fid);
   }
 
  public:
-  /**
-   * @brief Construct a fragment with a set of vertices and edges.
-   *
-   * @param fid Fragment ID
-   * @param vertices A set of vertices.
-   * @param edges A set of edges.
-   */
-  virtual void Init(const CommSpec& comm_spec, bool directed,
-                    std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
-                    std::vector<internal::Vertex<VID_T, VDATA_T>>& vertices,
-                    std::vector<Edge<VID_T, EDATA_T>>& edges) = 0;
-
-  virtual void ParallelInit(
-      const CommSpec& comm_spec, bool directed,
-      std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
-      std::vector<internal::Vertex<VID_T, VDATA_T>>& vertices,
-      std::vector<Edge<VID_T, EDATA_T>>& edges, int concurrency) {
-    return Init(comm_spec, directed, std::move(vm_ptr), vertices, edges);
-  }
-
-  /**
-   * @brief For some kind of applications, specific data structures will be
-   * generated.
-   *
-   * @param strategy
-   * @param need_split_edge
-   */
-  virtual void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) = 0;
-
   /**
    * @brief Returns true if the fragment is directed, false otherwise.
    *
@@ -136,196 +93,29 @@ class FragmentBase {
    *
    * @return The number of vertices in this fragment.
    */
-  VID_T GetVerticesNum() const { return vertices_.size(); }
+  virtual size_t GetVerticesNum() const = 0;
 
   /**
    * @brief Returns the number of vertices in the entire graph.
    *
    * @return The number of vertices in the entire graph.
    */
-  size_t GetTotalVerticesNum() const { return vm_ptr_->GetTotalVertexSize(); }
+  virtual size_t GetTotalVerticesNum() const = 0;
 
-  using vertices_t = typename TRAITS_T::vertices_t;
   /**
-   * @brief Get all vertices referenced to this fragment.
-   *
-   * @return A vertex set can be iterate on.
-   */
-  const vertices_t& Vertices() const { return vertices_; }
-
-  /**
-   * @brief Get a vertex with original ID vid.
-   *
-   * @param vid Original ID.
-   * @param v Got vertex.
-   *
-   * @return If find the vertex in this fragment, return true. Otherwise, return
-   * false.
-   */
-  bool GetVertex(const OID_T& oid, Vertex<VID_T>& v) const {
-    VID_T gid;
-    if (vm_ptr_->GetGid(oid, gid)) {
-      return Gid2Vertex(gid, v);
-    }
-    return false;
-  }
-
-  /**
-   * @brief Get the original ID of a vertex.
-   *
-   * @param v Input vertex.
-   *
-   * @return Its original ID.
-   */
-  OID_T GetId(const Vertex<VID_T>& v) const {
-    OID_T oid{};
-    vm_ptr_->GetOid(Vertex2Gid(v), oid);
-    return oid;
-  }
-
-  using internal_id_t = typename InternalOID<OID_T>::type;
-
-  internal_id_t GetInternalId(const Vertex<VID_T>& v) const {
-    internal_id_t oid{};
-    vm_ptr_->GetInternalOid(Vertex2Gid(v), oid);
-    return oid;
-  }
-
-  OID_T Gid2Oid(VID_T gid) const {
-    OID_T oid;
-    vm_ptr_->GetOid(gid, oid);
-    return oid;
-  }
-
-  /**
-   * @brief Get the ID of fragment the input vertex belongs to.
-   *
-   * @param u Input vertex.
-   *
-   * @return Its fragment ID.
-   */
-  fid_t GetFragId(const Vertex<VID_T>& u) const {
-    VID_T gid = Vertex2Gid(u);
-    return id_parser_.get_fragment_id(gid);
-  }
-
-  /**
-   * @brief Get the data of a vertex.
-   *
-   * @param v Input vertex.
-   *
-   * @return Data on it.
-   */
-  virtual const VDATA_T& GetData(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Set the data of a vertex.
-   *
-   * @param v Input vertex.
-   * @param val Data to write.
-   * @attention This will only be applied locally, won't sync on other mirrors
-   * globally.
-   */
-  virtual void SetData(const Vertex<VID_T>& v, const VDATA_T& val) = 0;
-
-  /**
-   * @brief Check if vertex v has a child, that is, existing an edge v->u.
-   *
-   * @param v Input vertex.
-   *
-   * @return True if vertex v has a child, false otherwise.
-   */
-  virtual bool HasChild(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Check if vertex v has a parent, that is, existing an edge u->v.
-   *
-   * @param v Input vertex.
-   *
-   * @return True if vertex v has a parent, false otherwise.
-   */
-  virtual bool HasParent(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Returns the in-degree of vertex v in this fragment.
-   *
-   * @param v Input vertex.
-   *
-   * @return In-degree of vertex v in this fragment.
-   */
-  virtual int GetLocalInDegree(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Returns the out-degree of vertex v in this fragment.<Paste>
-   *
-   * @param v Input vertex.
-   *
-   * @return Out-degree of vertex v in this fragment.
-   */
-  virtual int GetLocalOutDegree(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Convert from global id to a vertex handle.
-   *
-   * @param gid Input global id.
-   * @param v Output vertex handle.
-   *
-   * @return True if exists a vertex with global id as gid in this fragment,
-   * false otherwise.
-   */
-  virtual bool Gid2Vertex(const VID_T& gid, Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Convert from vertex handle to its global id.
-   *
-   * @param v Input vertex handle.
-   *
-   * @return Global id of the vertex.
-   */
-  virtual VID_T Vertex2Gid(const Vertex<VID_T>& v) const = 0;
-
-  /**
-   * @brief Returns the incoming adjacent vertices of v.
-   *
-   * @param v Input vertex.
-   *
-   * @return The incoming adjacent vertices of v.
-   */
-  virtual AdjList<VID_T, EDATA_T> GetIncomingAdjList(
-      const Vertex<VID_T>& v) = 0;
-
-  virtual ConstAdjList<VID_T, EDATA_T> GetIncomingAdjList(
-      const Vertex<VID_T>& v) const = 0;
-
-  virtual fragment_adj_list_t GetIncomingAdjList(const Vertex<VID_T>& v,
-                                                 fid_t fid) = 0;
-
-  virtual fragment_const_adj_list_t GetIncomingAdjList(const Vertex<VID_T>& v,
-                                                       fid_t fid) const = 0;
-  /**
-   * @brief Returns the outgoing adjacent vertices of v.
-   *
-   * @param v Input vertex.
+   * @brief For some kind of applications, specific data structures will be
+   * generated.
    *
-   * @return The outgoing adjacent vertices of v.
+   * @param strategy
+   * @param need_split_edge
    */
-  virtual AdjList<VID_T, EDATA_T> GetOutgoingAdjList(
-      const Vertex<VID_T>& v) = 0;
-
-  virtual ConstAdjList<VID_T, EDATA_T> GetOutgoingAdjList(
-      const Vertex<VID_T>& v) const = 0;
-
-  virtual fragment_adj_list_t GetOutgoingAdjList(const Vertex<VID_T>& v,
-                                                 fid_t fid) = 0;
-
-  virtual fragment_const_adj_list_t GetOutgoingAdjList(const Vertex<VID_T>& v,
-                                                       fid_t fid) const = 0;
+  virtual void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) = 0;
 
  protected:
   template <typename IOADAPTOR_T>
   void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
     InArchive arc;
-    arc << fid_ << fnum_ << directed_ << ivnum_ << vertices_;
+    arc << fid_ << fnum_ << directed_;
     CHECK(writer->WriteArchive(arc));
   }
 
@@ -333,18 +123,11 @@ class FragmentBase {
   void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
     OutArchive arc;
     CHECK(reader->ReadArchive(arc));
-    arc >> fid_ >> fnum_ >> directed_ >> ivnum_ >> vertices_;
-    id_parser_.init(fnum_);
+    arc >> fid_ >> fnum_ >> directed_;
   }
 
   fid_t fid_, fnum_;
   bool directed_;
-  VID_T ivnum_;
-
-  vertices_t vertices_;
-  std::unique_ptr<VertexMap<OID_T, VID_T>> vm_ptr_;
-
-  IdParser<VID_T> id_parser_;
 };
 
 }  // namespace grape
diff --git a/grape/fragment/immutable_edgecut_fragment.h b/grape/fragment/immutable_edgecut_fragment.h
index 1ce5a158..cbcab57b 100644
--- a/grape/fragment/immutable_edgecut_fragment.h
+++ b/grape/fragment/immutable_edgecut_fragment.h
@@ -149,8 +149,7 @@ class ImmutableEdgecutFragment
   template <typename T>
   using vertex_array_t = VertexArray<vertices_t, T>;
 
-  ImmutableEdgecutFragment()
-      : FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, traits_t>() {}
+  ImmutableEdgecutFragment() : FragmentBase<OID_T, VDATA_T, EDATA_T>() {}
 
   virtual ~ImmutableEdgecutFragment() = default;
 
@@ -216,7 +215,7 @@ class ImmutableEdgecutFragment
   void Init(const CommSpec& comm_spec, bool directed,
             std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
             std::vector<internal_vertex_t>& vertices,
-            std::vector<edge_t>& edges) override {
+            std::vector<edge_t>& edges) {
     init(comm_spec.fid(), directed, std::move(vm_ptr));
 
     static constexpr VID_T invalid_vid = std::numeric_limits<VID_T>::max();
@@ -353,7 +352,7 @@ class ImmutableEdgecutFragment
   void ParallelInit(const CommSpec& comm_spec, bool directed,
                     std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
                     std::vector<internal_vertex_t>& vertices,
-                    std::vector<edge_t>& edges, int concurrency) override {
+                    std::vector<edge_t>& edges, int concurrency) {
     init(comm_spec.fid(), directed, std::move(vm_ptr));
 
     static constexpr VID_T invalid_vid = std::numeric_limits<VID_T>::max();
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index c27cb45a..886ebede 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -48,7 +48,8 @@ struct IteratorPair {
 };
 
 template <typename VID_T>
-class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
+class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType>
+    : public FragmentBase<int64_t, EmptyType, EmptyType> {
  public:
   using oid_t = int64_t;
   using vid_t = VID_T;
@@ -57,6 +58,10 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   using vertices_t = VertexRange<oid_t>;
   using both_vertices_t = DualVertexRange<oid_t>;
   using edge_t = Edge<oid_t, edata_t>;
+  using base_t = FragmentBase<oid_t, vdata_t, edata_t>;
+
+  static constexpr FragmentType fragment_type = FragmentType::kVertexCut;
+  static constexpr LoadStrategy load_strategy = LoadStrategy::kOnlyOut;
 
   template <typename T>
   using vertex_array_t = VertexArray<vertices_t, T>;
@@ -67,11 +72,13 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   ImmutableVertexcutFragment() = default;
   ~ImmutableVertexcutFragment() {}
 
+  using base_t::fid_;
+  using base_t::fnum_;
+
   void Init(const CommSpec& comm_spec, int64_t vnum,
             std::vector<edge_t>&& edges, int bucket_num,
             std::vector<size_t>&& bucket_edge_offsets) {
-    fid_ = comm_spec.fid();
-    fnum_ = comm_spec.fnum();
+    base_t::init(comm_spec.fid(), comm_spec.fnum(), false);
 
     partitioner_.init(comm_spec.fnum(), vnum);
     src_vertices_ = partitioner_.get_src_vertices(fid_);
@@ -103,11 +110,13 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
     bucket_edge_offsets_ = std::move(bucket_edge_offsets);
   }
 
-  const vertices_t& GetSourceVertices() const { return src_vertices_; }
-  const vertices_t& GetDestinationVertices() const { return dst_vertices_; }
-  const both_vertices_t& GetVertices() const { return vertices_; }
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {}
 
-  const vertices_t& GetMasterVertices() const { return master_vertices_; }
+  const vertices_t& SourceVertices() const { return src_vertices_; }
+  const vertices_t& DestinationVertices() const { return dst_vertices_; }
+  const both_vertices_t& Vertices() const { return vertices_; }
+
+  const vertices_t& MasterVertices() const { return master_vertices_; }
 
 #ifdef USE_EDGE_ARRAY
   const Array<edge_t, Allocator<edge_t>>& GetEdges() const { return edges_; }
@@ -115,9 +124,6 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   const std::vector<edge_t>& GetEdges() const { return edges_; }
 #endif
 
-  fid_t fid() const { return fid_; }
-  fid_t fnum() const { return fnum_; }
-
   template <typename IOADAPTOR_T>
   void Serialize(const std::string& prefix) {
     char fbuf[1024];
@@ -128,8 +134,10 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
         std::unique_ptr<IOADAPTOR_T>(new IOADAPTOR_T(std::string(fbuf)));
     io_adaptor->Open("wb");
 
+    base_t::serialize(io_adaptor);
+
     InArchive arc;
-    arc << fid_ << fnum_ << edges_.size();
+    arc << edges_.size();
     arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_;
     arc << bucket_num_ << bucket_edge_offsets_;
     CHECK(io_adaptor->WriteArchive(arc));
@@ -162,10 +170,12 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
         std::unique_ptr<IOADAPTOR_T>(new IOADAPTOR_T(std::string(fbuf)));
     io_adaptor->Open();
 
+    base_t::deserialize(io_adaptor);
+
     OutArchive arc;
     size_t edge_num;
     CHECK(io_adaptor->ReadArchive(arc));
-    arc >> fid_ >> fnum_ >> edge_num;
+    arc >> edge_num;
     CHECK_EQ(fid_, comm_spec.fid());
     CHECK_EQ(fnum_, comm_spec.fnum());
     arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_;
@@ -189,10 +199,14 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
     MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edges_.size());
   }
 
-  int64_t GetTotalVerticesNum() const {
+  size_t GetTotalVerticesNum() const override {
     return partitioner_.get_total_vertices_num();
   }
 
+  size_t GetEdgeNum() const override { return edges_.size(); }
+
+  size_t GetVerticesNum() const override { return vertices_.size(); }
+
   const VCPartitioner<int64_t>& GetPartitioner() const { return partitioner_; }
 
   IteratorPair<edge_t> GetEdgesOfBucket(int src_bucket_id,
@@ -208,8 +222,6 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType> {
   int GetBucketNum() const { return bucket_num_; }
 
  private:
-  fid_t fid_;
-  fid_t fnum_;
   VCPartitioner<int64_t> partitioner_;
   vertices_t src_vertices_;
   vertices_t dst_vertices_;
diff --git a/grape/fragment/mutable_edgecut_fragment.h b/grape/fragment/mutable_edgecut_fragment.h
index b666ec3a..02702ed6 100644
--- a/grape/fragment/mutable_edgecut_fragment.h
+++ b/grape/fragment/mutable_edgecut_fragment.h
@@ -87,8 +87,7 @@ class MutableEdgecutFragment
   template <typename T>
   using vertex_array_t = VertexArray<vertices_t, T>;
 
-  MutableEdgecutFragment()
-      : FragmentBase<OID_T, VID_T, VDATA_T, EDATA_T, traits_t>() {}
+  MutableEdgecutFragment() : FragmentBase<OID_T, VDATA_T, EDATA_T>() {}
   virtual ~MutableEdgecutFragment() = default;
 
   using base_t::buildCSR;
@@ -152,7 +151,7 @@ class MutableEdgecutFragment
   void Init(const CommSpec& comm_spec, bool directed,
             std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
             std::vector<internal_vertex_t>& vertices,
-            std::vector<edge_t>& edges) override {
+            std::vector<edge_t>& edges) {
     init(comm_spec.fid(), directed, std::move(vm_ptr));
 
     ovnum_ = 0;
@@ -231,7 +230,15 @@ class MutableEdgecutFragment
     }
   }
 
+  void ParallelInit(const CommSpec& comm_spec, bool directed,
+                    std::unique_ptr<VertexMap<OID_T, VID_T>>&& vm_ptr,
+                    std::vector<internal_vertex_t>& vertices,
+                    std::vector<edge_t>& edges, int concurrency) {
+    Init(comm_spec, directed, std::move(vm_ptr), vertices, edges);
+  }
+
   using base_t::Gid2Lid;
+  using base_t::Gid2Vertex;
   using base_t::ie_;
   using base_t::oe_;
   using base_t::vm_ptr_;
@@ -513,30 +520,6 @@ class MutableEdgecutFragment
     return ovgid_[outerVertexLidToIndex(v.GetValue())];
   }
 
-  inline bool Gid2Vertex(const VID_T& gid, vertex_t& v) const override {
-    fid_t fid = id_parser_.get_fragment_id(gid);
-    if (fid == fid_) {
-      v.SetValue(id_parser_.get_local_id(gid));
-      return true;
-    } else {
-      auto iter = ovg2i_.find(gid);
-      if (iter != ovg2i_.end()) {
-        v.SetValue(iter->second);
-        return true;
-      } else {
-        return false;
-      }
-    }
-  }
-
-  inline VID_T Vertex2Gid(const vertex_t& v) const override {
-    if (IsInnerVertex(v)) {
-      return id_parser_.generate_global_id(fid_, v.GetValue());
-    } else {
-      return ovgid_[outerVertexLidToIndex(v.GetValue())];
-    }
-  }
-
  public:
   using base_t::GetIncomingAdjList;
   using base_t::GetOutgoingAdjList;
diff --git a/grape/grape.h b/grape/grape.h
index 1a04165d..76a728d2 100644
--- a/grape/grape.h
+++ b/grape/grape.h
@@ -19,19 +19,18 @@ limitations under the License.
 #include "grape/app/auto_app_base.h"
 #include "grape/app/batch_shuffle_app_base.h"
 #include "grape/app/context_base.h"
+#include "grape/app/gather_scatter_app_base.h"
 #include "grape/app/parallel_app_base.h"
-#include "grape/app/vc_app_base.h"
 #include "grape/app/vertex_data_context.h"
 #include "grape/app/void_context.h"
 #include "grape/parallel/auto_parallel_message_manager.h"
 #include "grape/parallel/batch_shuffle_message_manager.h"
 #include "grape/parallel/default_message_manager.h"
+#include "grape/parallel/gather_scatter_message_manager.h"
 #include "grape/parallel/parallel_message_manager.h"
-#include "grape/parallel/vc_message_manager.h"
 #include "grape/utils/atomic_ops.h"
 #include "grape/utils/vertex_array.h"
 #include "grape/utils/vertex_set.h"
-#include "grape/worker/vc_worker.h"
 #include "grape/worker/worker.h"
 
 #ifdef __CUDACC__
diff --git a/grape/parallel/vc_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
similarity index 96%
rename from grape/parallel/vc_message_manager.h
rename to grape/parallel/gather_scatter_message_manager.h
index 0f13a6b7..7c8c816f 100644
--- a/grape/parallel/vc_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -13,15 +13,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#ifndef GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
-#define GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
+#ifndef GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
+#define GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
 
 namespace grape {
 
-class VCMessageManager : public MessageManagerBase {
+class GatherScatterMessageManager : public MessageManagerBase {
  public:
-  VCMessageManager() : comm_(NULL_COMM) {}
-  ~VCMessageManager() override {
+  GatherScatterMessageManager() : comm_(NULL_COMM) {}
+  ~GatherScatterMessageManager() override {
     if (ValidComm(comm_)) {
       MPI_Comm_free(&comm_);
     }
@@ -262,4 +262,4 @@ class VCMessageManager : public MessageManagerBase {
 
 }  // namespace grape
 
-#endif  // GRAPE_PARALLEL_VC_MESSAGE_MANAGER_H_
+#endif  // GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
diff --git a/grape/types.h b/grape/types.h
index 34138683..a4f610f7 100644
--- a/grape/types.h
+++ b/grape/types.h
@@ -57,6 +57,22 @@ to_enum(T value) noexcept {
   return static_cast<E>(value);
 }
 
+enum class FragmentType {
+  kEdgeCut = 0,
+  kVertexCut = 1,
+};
+
+template <typename FRAG_T>
+struct is_vertex_cut_fragment {
+  static constexpr bool value =
+      FRAG_T::fragment_type == FragmentType::kVertexCut;
+};
+
+template <typename FRAG_T>
+struct is_edge_cut_fragment {
+  static constexpr bool value = FRAG_T::fragment_type == FragmentType::kEdgeCut;
+};
+
 /**
  * @brief LoadStrategy specifies the which edges should be loadded when building
  * the graph from a location.
@@ -84,6 +100,7 @@ enum class MessageStrategy {
   kAlongIncomingEdgeToOuterVertex = 1,  /// from c to a;
   kAlongEdgeToOuterVertex = 2,          /// from a to b, a to c;
   kSyncOnOuterVertex = 3,               /// from b' to b and c' to c;
+  kGatherScatter = 4,  /// gather from b' to b and scatter from c' to c;
 };
 
 template <typename APP_T, typename GRAPH_T>
@@ -95,7 +112,8 @@ constexpr inline bool check_load_strategy_compatible() {
            (GRAPH_T::load_strategy == LoadStrategy::kOnlyIn))) ||
          ((APP_T::load_strategy == LoadStrategy::kOnlyOut) &&
           ((GRAPH_T::load_strategy == LoadStrategy::kBothOutIn) ||
-           (GRAPH_T::load_strategy == LoadStrategy::kOnlyOut)));
+           (GRAPH_T::load_strategy == LoadStrategy::kOnlyOut))) ||
+         (GRAPH_T::fragment_type == FragmentType::kVertexCut);
 }
 
 template <typename APP_T, typename GRAPH_T>
@@ -111,7 +129,9 @@ constexpr inline bool check_message_strategy_valid() {
            MessageStrategy::kAlongOutgoingEdgeToOuterVertex) &&
           ((GRAPH_T::load_strategy == LoadStrategy::kOnlyOut) ||
            (GRAPH_T::load_strategy == LoadStrategy::kBothOutIn))) ||
-         (APP_T::message_strategy == MessageStrategy::kSyncOnOuterVertex);
+         (APP_T::message_strategy == MessageStrategy::kSyncOnOuterVertex) ||
+         ((APP_T::message_strategy == MessageStrategy::kGatherScatter) &&
+          (GRAPH_T::fragment_type == FragmentType::kVertexCut));
 }
 
 template <typename APP_T, typename GRAPH_T>
diff --git a/grape/worker/vc_worker.h b/grape/worker/vc_worker.h
deleted file mode 100644
index 6840341e..00000000
--- a/grape/worker/vc_worker.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/** Copyright 2020 Alibaba Group Holding Limited.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#ifndef GRAPE_VC_WORKER_WORKER_H_
-#define GRAPE_VC_WORKER_WORKER_H_
-
-#include "grape/app/vc_app_base.h"
-#include "grape/communication/communicator.h"
-#include "grape/parallel/parallel_engine.h"
-#include "grape/worker/comm_spec.h"
-
-namespace grape {
-
-template <typename APP_T, typename MESSAGE_MANAGER_T>
-class VCWorker {
- public:
-  using fragment_t = typename APP_T::fragment_t;
-  using context_t = typename APP_T::context_t;
-
-  using message_manager_t = MESSAGE_MANAGER_T;
-
-  VCWorker(std::shared_ptr<APP_T> app, std::shared_ptr<fragment_t> graph)
-      : app_(app), context_(nullptr), fragment_(graph) {}
-  ~VCWorker() = default;
-
-  void Init(const CommSpec& comm_spec,
-            const ParallelEngineSpec& pe_spec = DefaultParallelEngineSpec()) {
-    auto& graph = *fragment_;
-
-    comm_spec_ = comm_spec;
-    MPI_Barrier(comm_spec_.comm());
-    if (context_ == nullptr) {
-      context_ = std::make_shared<context_t>(graph);
-    }
-
-    messages_.Init(comm_spec_.comm());
-
-    InitParallelEngine(app_, pe_spec);
-    InitCommunicator(app_, comm_spec_.comm());
-  }
-
-  void Finalize() {}
-
-  template <class... Args>
-  void Query(Args&&... args) {
-    double t = GetCurrentTime();
-
-    context_->Init(messages_, std::forward<Args>(args)...);
-    auto& graph = context_->fragment();
-
-    messages_.Start();
-
-    messages_.StartARound();
-
-    app_->PEval(graph, *context_, messages_);
-
-    messages_.FinishARound();
-
-    if (comm_spec_.worker_id() == kCoordinatorRank) {
-      VLOG(1) << "[Coordinator]: Finished PEval, time: " << GetCurrentTime() - t
-              << " sec";
-    }
-
-    int step = 1;
-
-    while (!messages_.ToTerminate()) {
-      t = GetCurrentTime();
-      messages_.StartARound();
-
-      app_->IncEval(graph, *context_, messages_);
-
-      messages_.FinishARound();
-
-      if (comm_spec_.worker_id() == kCoordinatorRank) {
-        VLOG(1) << "[Coordinator]: Finished IncEval - " << step
-                << ", time: " << GetCurrentTime() - t << " sec";
-      }
-      ++step;
-    }
-
-    MPI_Barrier(comm_spec_.comm());
-
-    messages_.Finalize();
-  }
-
-  std::shared_ptr<context_t> GetContext() { return context_; }
-
-  const TerminateInfo& GetTerminateInfo() const {
-    return messages_.GetTerminateInfo();
-  }
-
-  void Output(std::ostream& os) { context_->Output(os); }
-
- private:
-  std::shared_ptr<APP_T> app_;
-  std::shared_ptr<context_t> context_;
-  std::shared_ptr<fragment_t> fragment_;
-  message_manager_t messages_;
-
-  CommSpec comm_spec_;
-};
-
-}  // namespace grape
-
-#endif  // GRAPE_VC_WORKER_WORKER_H_
diff --git a/grape/worker/worker.h b/grape/worker/worker.h
index 25d88a47..e872839c 100644
--- a/grape/worker/worker.h
+++ b/grape/worker/worker.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "grape/config.h"
 #include "grape/parallel/auto_parallel_message_manager.h"
 #include "grape/parallel/batch_shuffle_message_manager.h"
+#include "grape/parallel/gather_scatter_message_manager.h"
 #include "grape/parallel/parallel_engine.h"
 #include "grape/parallel/parallel_message_manager.h"
 #include "grape/parallel/parallel_message_manager_opt.h"
@@ -241,6 +242,9 @@ using AutoWorker =
 template <typename APP_T>
 using BatchShuffleWorker = Worker<APP_T, BatchShuffleMessageManager>;
 
+template <typename APP_T>
+using GatherScatterWorker = Worker<APP_T, GatherScatterMessageManager>;
+
 }  // namespace grape
 
 #endif  // GRAPE_WORKER_WORKER_H_

From 2a031f45fb69b6897bb563c815b6679110e4e13a Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Fri, 13 Sep 2024 18:04:14 +0800
Subject: [PATCH 05/15] adapt

---
 .../analytical_apps/pagerank/pagerank_vc.h    |  4 +-
 .../parallel/gather_scatter_message_manager.h | 38 ++++++++++++++++---
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index a60a824f..64ddae12 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -114,6 +114,7 @@ class PageRankVC
     messages.ScatterMasterVertices<fragment_t, double>(frag, ctx.master_result,
                                                        ctx.curr_result);
     ctx.t7 += GetCurrentTime();
+    messages.ForceContinue();
   }
 
   void IncEval(const fragment_t& frag, context_t& ctx,
@@ -182,6 +183,7 @@ class PageRankVC
       messages.ScatterMasterVertices<fragment_t, double>(
           frag, ctx.master_result, ctx.curr_result);
       ctx.t7 += GetCurrentTime();
+      messages.ForceContinue();
     } else {
       ctx.t4 -= GetCurrentTime();
       int bucket_num = frag.GetBucketNum();
@@ -224,8 +226,6 @@ class PageRankVC
         ctx.master_result[v] = ctx.master_result[v] * ctx.delta + base;
       });
       ctx.t2 += GetCurrentTime();
-
-      messages.ForceTerminate();
     }
   }
 };
diff --git a/grape/parallel/gather_scatter_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
index 7c8c816f..21aaa85a 100644
--- a/grape/parallel/gather_scatter_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -16,6 +16,11 @@ limitations under the License.
 #ifndef GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
 #define GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
 
+#include <mpi.h>
+
+#include "grape/communication/sync_comm.h"
+#include "grape/parallel/message_manager_base.h"
+
 namespace grape {
 
 class GatherScatterMessageManager : public MessageManagerBase {
@@ -43,22 +48,44 @@ class GatherScatterMessageManager : public MessageManagerBase {
 
   void Start() override {}
 
-  void StartARound() override {}
-
-  void FinishARound() override {}
+  void StartARound() override {
+    vote_terminate_ = true;
+    force_terminate_ = false;
+  }
 
-  bool ToTerminate() override { return force_terminate_; }
+  void FinishARound() override { ++round_; }
+
+  bool ToTerminate() override {
+    int flag[3];
+    flag[0] = vote_terminate_ ? 1 : 0;
+    flag[1] = force_terminate_ ? 1 : 0;
+    int ret[2];
+    MPI_Allreduce(&flag[0], &ret[0], 2, MPI_INT, MPI_SUM, comm_);
+    if (ret[1] > 0) {
+      terminate_info_.success = false;
+      sync_comm::AllGather(terminate_info_.info, comm_);
+      return true;
+    } else if (ret[0] == static_cast<int>(fnum_)) {
+      return true;
+    } else {
+      return false;
+    }
+  }
 
   void Finalize() override {
     LOG(INFO) << "[frag-" << fid_ << "] gather comm: " << t0_gather_comm_
               << ", gather calc: " << t1_gather_calc_;
   }
 
-  void ForceContinue() override {}
+  void ForceContinue() override {
+    force_terminate_ = false;
+    vote_terminate_ = false;
+  }
 
   void ForceTerminate(const std::string& terminate_info = "") override {
     force_terminate_ = true;
     terminate_info_.info[comm_spec_.fid()] = terminate_info;
+    vote_terminate_ = false;
   }
 
   const TerminateInfo& GetTerminateInfo() const override {
@@ -255,6 +282,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
 
   bool force_terminate_;
   TerminateInfo terminate_info_;
+  bool vote_terminate_;
 
   double t0_gather_comm_ = 0;
   double t1_gather_calc_ = 0;

From b28c69108fef12ca4d8973117aac0a5cf3c043a8 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Fri, 13 Sep 2024 18:41:05 +0800
Subject: [PATCH 06/15] lint

---
 examples/analytical_apps/utils.h             |   2 +-
 grape/fragment/basic_vc_ds_fragment_loader.h |   2 +-
 grape/fragment/partitioner.h                 | 271 -------------------
 grape/utils/memory_inspector.h               |   2 +-
 4 files changed, 3 insertions(+), 274 deletions(-)
 delete mode 100644 grape/fragment/partitioner.h

diff --git a/examples/analytical_apps/utils.h b/examples/analytical_apps/utils.h
index 79a7f3f5..7a6a5a89 100644
--- a/examples/analytical_apps/utils.h
+++ b/examples/analytical_apps/utils.h
@@ -83,4 +83,4 @@ void DoQuery(std::shared_ptr<FRAG_T> fragment, std::shared_ptr<APP_T> app,
 
 }  // namespace grape
 
-#endif  // EXAMPLES_ANALYTICAL_APPS_UTILS_H_
\ No newline at end of file
+#endif  // EXAMPLES_ANALYTICAL_APPS_UTILS_H_
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index b0f41bfc..dd79932a 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -309,4 +309,4 @@ class BasicVCDSFragmentLoader {
 
 }  // namespace grape
 
-#endif  // GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
\ No newline at end of file
+#endif  // GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
diff --git a/grape/fragment/partitioner.h b/grape/fragment/partitioner.h
deleted file mode 100644
index 29b7807c..00000000
--- a/grape/fragment/partitioner.h
+++ /dev/null
@@ -1,271 +0,0 @@
-/** Copyright 2020 Alibaba Group Holding Limited.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#ifndef GRAPE_FRAGMENT_PARTITIONER_H_
-#define GRAPE_FRAGMENT_PARTITIONER_H_
-
-#include <vector>
-
-#include "flat_hash_map/flat_hash_map.hpp"
-#include "grape/config.h"
-
-namespace grape {
-
-/**
- * @brief HashPartitoner is a partitioner with the strategy of hashing on
- * original vertex_ids.
- *
- * @tparam OID_T
- */
-template <typename OID_T>
-class HashPartitioner {
- public:
-  HashPartitioner() : fnum_(1) {}
-  explicit HashPartitioner(size_t frag_num) : fnum_(frag_num) {}
-  HashPartitioner(size_t frag_num, std::vector<OID_T>&) : fnum_(frag_num) {}
-
-  inline fid_t GetPartitionId(const OID_T& oid) const {
-    return static_cast<fid_t>(static_cast<uint64_t>(oid) % fnum_);
-  }
-
-  void SetPartitionId(const OID_T& oid, fid_t fid) {
-    LOG(FATAL) << "not support";
-  }
-
-  HashPartitioner& operator=(const HashPartitioner& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    return *this;
-  }
-
-  HashPartitioner& operator=(HashPartitioner&& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    return *this;
-  }
-
-  template <typename IOADAPTOR_T>
-  void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
-    CHECK(writer->Write(&fnum_, sizeof(fid_t)));
-  }
-
-  template <typename IOADAPTOR_T>
-  void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
-    CHECK(reader->Read(&fnum_, sizeof(fid_t)));
-  }
-
- private:
-  fid_t fnum_;
-};
-
-template <>
-class HashPartitioner<std::string> {
- public:
-  using oid_t = std::string;
-  using internal_oid_t = nonstd::string_view;
-
-  HashPartitioner() : fnum_(1) {}
-  explicit HashPartitioner(size_t frag_num) : fnum_(frag_num) {}
-  HashPartitioner(size_t frag_num, std::vector<std::string>&)
-      : fnum_(frag_num) {}
-
-  inline fid_t GetPartitionId(const internal_oid_t& oid) const {
-    return static_cast<fid_t>(
-        static_cast<uint64_t>(std::hash<internal_oid_t>()(oid) % fnum_));
-  }
-
-  inline fid_t GetPartitionId(const oid_t& oid) const {
-    internal_oid_t internal_oid(oid);
-    return GetPartitionId(internal_oid);
-  }
-
-  void SetPartitionId(const oid_t& oid, fid_t fid) {
-    LOG(FATAL) << "not support";
-  }
-
-  void SetPartitionId(const internal_oid_t& oid, fid_t fid) {
-    LOG(FATAL) << "not support";
-  }
-
-  HashPartitioner& operator=(const HashPartitioner& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    return *this;
-  }
-
-  HashPartitioner(const HashPartitioner& other) { fnum_ = other.fnum_; }
-
-  HashPartitioner& operator=(HashPartitioner&& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    return *this;
-  }
-
-  template <typename IOADAPTOR_T>
-  void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
-    CHECK(writer->Write(&fnum_, sizeof(fid_t)));
-  }
-
-  template <typename IOADAPTOR_T>
-  void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
-    CHECK(reader->Read(&fnum_, sizeof(fid_t)));
-  }
-
- private:
-  fid_t fnum_;
-};
-
-/**
- * @brief SegmentedPartitioner is a partitioner with a strategy of chunking
- * original vertex_ids.
- *
- * @tparam OID_T
- */
-template <typename OID_T>
-class SegmentedPartitioner {
- public:
-  SegmentedPartitioner() : fnum_(1) {}
-  explicit SegmentedPartitioner(size_t frag_num) : fnum_(frag_num) {}
-  SegmentedPartitioner(size_t frag_num, std::vector<OID_T>& oid_list) {
-    fnum_ = frag_num;
-    size_t vnum = oid_list.size();
-    size_t frag_vnum = (vnum + fnum_ - 1) / fnum_;
-    o2f_.reserve(vnum);
-    for (size_t i = 0; i < vnum; ++i) {
-      fid_t fid = static_cast<fid_t>(i / frag_vnum);
-      o2f_.emplace(oid_list[i], fid);
-    }
-  }
-
-  inline fid_t GetPartitionId(const OID_T& oid) const { return o2f_.at(oid); }
-
-  void SetPartitionId(const OID_T& oid, fid_t fid) { o2f_[oid] = fid; }
-
-  SegmentedPartitioner& operator=(const SegmentedPartitioner& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    o2f_ = other.o2f_;
-    return *this;
-  }
-
-  SegmentedPartitioner& operator=(SegmentedPartitioner&& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    o2f_ = std::move(other.o2f_);
-    return *this;
-  }
-
-  template <typename IOADAPTOR_T>
-  void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
-    InArchive arc;
-    arc << fnum_ << o2f_;
-    CHECK(writer->WriteArchive(arc));
-  }
-
-  template <typename IOADAPTOR_T>
-  void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
-    OutArchive arc;
-    CHECK(reader->ReadArchive(arc));
-    arc >> fnum_ >> o2f_;
-  }
-
- private:
-  fid_t fnum_;
-  ska::flat_hash_map<OID_T, fid_t> o2f_;
-};
-
-template <>
-class SegmentedPartitioner<std::string> {
-  using oid_t = std::string;
-  using internal_oid_t = nonstd::string_view;
-
- public:
-  SegmentedPartitioner() : fnum_(1) {}
-  explicit SegmentedPartitioner(size_t frag_num) : fnum_(frag_num) {}
-  SegmentedPartitioner(size_t frag_num, std::vector<oid_t>& oid_list) {
-    fnum_ = frag_num;
-    size_t vnum = oid_list.size();
-    size_t frag_vnum = (vnum + fnum_ - 1) / fnum_;
-    o2f_.reserve(vnum);
-    for (size_t i = 0; i < vnum; ++i) {
-      fid_t fid = static_cast<fid_t>(i / frag_vnum);
-      o2f_.emplace(oid_list[i], fid);
-    }
-  }
-
-  inline fid_t GetPartitionId(const oid_t& oid) const { return o2f_.at(oid); }
-
-  inline fid_t GetPartitionId(const internal_oid_t& oid) const {
-    return o2f_.at(std::string(oid));
-  }
-
-  void SetPartitionId(const oid_t& oid, fid_t fid) { o2f_[oid] = fid; }
-
-  void SetPartitionId(const internal_oid_t& oid, fid_t fid) {
-    o2f_[std::string(oid)] = fid;
-  }
-
-  SegmentedPartitioner& operator=(const SegmentedPartitioner& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    o2f_ = other.o2f_;
-    return *this;
-  }
-
-  SegmentedPartitioner& operator=(SegmentedPartitioner&& other) {
-    if (this == &other) {
-      return *this;
-    }
-    fnum_ = other.fnum_;
-    o2f_ = std::move(other.o2f_);
-    return *this;
-  }
-
-  template <typename IOADAPTOR_T>
-  void serialize(std::unique_ptr<IOADAPTOR_T>& writer) {
-    InArchive arc;
-    arc << fnum_ << o2f_;
-    CHECK(writer->WriteArchive(arc));
-  }
-
-  template <typename IOADAPTOR_T>
-  void deserialize(std::unique_ptr<IOADAPTOR_T>& reader) {
-    OutArchive arc;
-    CHECK(reader->ReadArchive(arc));
-    arc >> fnum_ >> o2f_;
-  }
-
- private:
-  fid_t fnum_;
-  ska::flat_hash_map<oid_t, fid_t> o2f_;
-};
-
-}  // namespace grape
-
-#endif  // GRAPE_FRAGMENT_PARTITIONER_H_
diff --git a/grape/utils/memory_inspector.h b/grape/utils/memory_inspector.h
index 3447878f..b0398e11 100644
--- a/grape/utils/memory_inspector.h
+++ b/grape/utils/memory_inspector.h
@@ -63,4 +63,4 @@ struct MemoryInspector {
 
 }  // namespace grape
 
-#endif  // GRAPE_UTILS_MEMORY_INSPECTOR_H_
\ No newline at end of file
+#endif  // GRAPE_UTILS_MEMORY_INSPECTOR_H_

From 073b4cfae8f9ac5d6096c95ea4c1eea0f058ab7c Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Sat, 14 Sep 2024 12:16:10 +0800
Subject: [PATCH 07/15] trim

---
 examples/analytical_apps/run_app.cc           | 10 ++---
 examples/analytical_apps/run_app_vc.h         |  4 +-
 grape/fragment/basic_vc_ds_fragment_loader.h  | 33 ++++++++++-----
 grape/fragment/basic_vc_fragment_loader.h     |  1 -
 grape/fragment/immutable_vertexcut_fragment.h | 42 +++++++++----------
 grape/fragment/vc_fragment_loader.h           |  1 -
 6 files changed, 50 insertions(+), 41 deletions(-)

diff --git a/examples/analytical_apps/run_app.cc b/examples/analytical_apps/run_app.cc
index 7af79bd8..6dfdd063 100644
--- a/examples/analytical_apps/run_app.cc
+++ b/examples/analytical_apps/run_app.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include "run_app.h"
+// #include "run_app.h"
 
 #include <gflags/gflags.h>
 #include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 
-#include "run_app_opt.h"
+// #include "run_app_opt.h"
 #include "run_app_vc.h"
 
 int main(int argc, char* argv[]) {
@@ -41,14 +41,14 @@ int main(int argc, char* argv[]) {
 
   std::string name = FLAGS_application;
   if (FLAGS_opt) {
-    grape::RunOpt();
+    // grape::RunOpt();
   } else if (FLAGS_vc) {
     grape::RunVC();
   } else {
     if (name.find("sssp") != std::string::npos) {
-      grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
+      // grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
     } else {
-      grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
+      // grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
     }
   }
 
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index efd573d9..1162d8e3 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -35,8 +35,8 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
   }
   graph_spec.single_scan = FLAGS_single_scan_load;
 
-  using FRAG_T = ImmutableVertexcutFragment<int64_t, uint32_t, grape::EmptyType,
-                                            grape::EmptyType>;
+  using FRAG_T =
+      ImmutableVertexcutFragment<int64_t, grape::EmptyType, grape::EmptyType>;
   std::shared_ptr<FRAG_T> fragment = LoadVertexcutGraph<FRAG_T>(
       FLAGS_efile, FLAGS_vertex_num, comm_spec, graph_spec);
   VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after loading graph: "
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index dd79932a..4bec75ab 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -74,7 +74,6 @@ template <typename FRAG_T>
 class BasicVCDSFragmentLoader {
   using fragment_t = FRAG_T;
   using oid_t = typename fragment_t::oid_t;
-  using vid_t = typename fragment_t::vid_t;
   using vdata_t = typename fragment_t::vdata_t;
   using edata_t = typename fragment_t::edata_t;
   using edge_t = typename fragment_t::edge_t;
@@ -117,17 +116,31 @@ class BasicVCDSFragmentLoader {
   }
 
   void AllocateBuffers() {
-    for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
-      if (comm_spec_.FragToWorker(fid) == comm_spec_.worker_id()) {
-        MPI_Reduce(MPI_IN_PLACE, partition_bucket_edge_num_[fid].data(),
-                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
-                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
-      } else {
-        MPI_Reduce(partition_bucket_edge_num_[fid].data(), nullptr,
-                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
-                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+    {
+      std::vector<std::vector<size_t>> partition_bucket_edge_num_out(
+          comm_spec_.fnum());
+      sync_comm::AllToAll(partition_bucket_edge_num_,
+                          partition_bucket_edge_num_out, comm_spec_.comm());
+      std::vector<size_t> partition_edge_num(bucket_num_ * bucket_num_, 0);
+      for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+        for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
+          partition_edge_num[i] += partition_bucket_edge_num_out[fid][i];
+        }
       }
+      partition_bucket_edge_num_[comm_spec_.fid()] =
+          std::move(partition_edge_num);
     }
+    // for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+    //   if (comm_spec_.FragToWorker(fid) == comm_spec_.worker_id()) {
+    //     MPI_Reduce(MPI_IN_PLACE, partition_bucket_edge_num_[fid].data(),
+    //                partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+    //                MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+    //   } else {
+    //     MPI_Reduce(partition_bucket_edge_num_[fid].data(), nullptr,
+    //                partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+    //                MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+    //   }
+    // }
     size_t edge_num = 0;
     bucket_cursor_ =
         std::vector<std::atomic<size_t>>(bucket_num_ * bucket_num_);
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
index 68d67e93..1b05db9e 100644
--- a/grape/fragment/basic_vc_fragment_loader.h
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -24,7 +24,6 @@ template <typename FRAG_T>
 class BasicVCFragmentLoader {
   using fragment_t = FRAG_T;
   using oid_t = typename fragment_t::oid_t;
-  using vid_t = typename fragment_t::vid_t;
   using vdata_t = typename fragment_t::vdata_t;
   using edata_t = typename fragment_t::edata_t;
   using edge_t = typename fragment_t::edge_t;
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index 886ebede..bb14aaa5 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -20,6 +20,7 @@ limitations under the License.
 
 #include <glog/logging.h>
 
+#include "grape/fragment/fragment_base.h"
 #include "grape/graph/edge.h"
 #include "grape/types.h"
 #include "grape/utils/memory_inspector.h"
@@ -31,28 +32,14 @@ namespace grape {
 
 // #define USE_EDGE_ARRAY
 
-template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T>
+template <typename OID_T, typename VDATA_T, typename EDATA_T>
 class ImmutableVertexcutFragment {};
 
-template <typename T>
-struct IteratorPair {
-  IteratorPair() : begin_(nullptr), end_(nullptr) {}
-  IteratorPair(const T* begin, const T* end) : begin_(begin), end_(end) {}
-
-  const T* begin() const { return begin_; }
-
-  const T* end() const { return end_; }
-
-  const T* begin_;
-  const T* end_;
-};
-
-template <typename VID_T>
-class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType>
+template <>
+class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     : public FragmentBase<int64_t, EmptyType, EmptyType> {
  public:
   using oid_t = int64_t;
-  using vid_t = VID_T;
   using edata_t = EmptyType;
   using vdata_t = EmptyType;
   using vertices_t = VertexRange<oid_t>;
@@ -60,6 +47,18 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType>
   using edge_t = Edge<oid_t, edata_t>;
   using base_t = FragmentBase<oid_t, vdata_t, edata_t>;
 
+  struct edge_bucket_t {
+    edge_bucket_t() : begin_(nullptr), end_(nullptr) {}
+    edge_bucket_t(const edge_t* begin, const edge_t* end)
+        : begin_(begin), end_(end) {}
+
+    const edge_t* begin() const { return begin_; }
+    const edge_t* end() const { return end_; }
+
+    const edge_t* begin_;
+    const edge_t* end_;
+  };
+
   static constexpr FragmentType fragment_type = FragmentType::kVertexCut;
   static constexpr LoadStrategy load_strategy = LoadStrategy::kOnlyOut;
 
@@ -209,14 +208,13 @@ class ImmutableVertexcutFragment<int64_t, VID_T, EmptyType, EmptyType>
 
   const VCPartitioner<int64_t>& GetPartitioner() const { return partitioner_; }
 
-  IteratorPair<edge_t> GetEdgesOfBucket(int src_bucket_id,
-                                        int dst_bucket_id) const {
+  edge_bucket_t GetEdgesOfBucket(int src_bucket_id, int dst_bucket_id) const {
     int idx = src_bucket_id * bucket_num_ + dst_bucket_id;
     if (static_cast<size_t>(idx) >= bucket_edge_offsets_.size()) {
-      return IteratorPair<edge_t>();
+      return edge_bucket_t();
     }
-    return IteratorPair<edge_t>(edges_.data() + bucket_edge_offsets_[idx],
-                                edges_.data() + bucket_edge_offsets_[idx + 1]);
+    return edge_bucket_t(edges_.data() + bucket_edge_offsets_[idx],
+                         edges_.data() + bucket_edge_offsets_[idx + 1]);
   }
 
   int GetBucketNum() const { return bucket_num_; }
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
index b91e2e02..c37ed090 100644
--- a/grape/fragment/vc_fragment_loader.h
+++ b/grape/fragment/vc_fragment_loader.h
@@ -28,7 +28,6 @@ template <typename FRAG_T, typename IOADAPTOR_T = LocalIOAdaptor,
 class VCFragmentLoader {
   using fragment_t = FRAG_T;
   using oid_t = typename fragment_t::oid_t;
-  using vid_t = typename fragment_t::vid_t;
   using vdata_t = typename fragment_t::vdata_t;
   using edata_t = typename fragment_t::edata_t;
 

From f126520561983939c9424716e5cba6a54b738548 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Sat, 14 Sep 2024 16:02:06 +0800
Subject: [PATCH 08/15] reorg

---
 .../analytical_apps/pagerank/pagerank_vc.h    | 153 ++++++++----------
 .../pagerank/pagerank_vc_context.h            |  40 ++---
 examples/analytical_apps/run_app_vc.h         |  13 +-
 .../append_only_edgecut_fragment.h            |   3 +-
 grape/cuda/fragment/host_fragment.h           |   5 +-
 grape/cuda/worker/gpu_batch_shuffle_worker.h  |   3 +-
 grape/cuda/worker/gpu_worker.h                |   3 +-
 grape/fragment/basic_vc_ds_fragment_loader.h  | 100 +++++++-----
 grape/fragment/basic_vc_fragment_loader.h     |  37 +++--
 grape/fragment/csr_edgecut_fragment_base.h    |   3 +-
 grape/fragment/fragment_base.h                |   4 +-
 grape/fragment/immutable_edgecut_fragment.h   |   5 +-
 grape/fragment/immutable_vertexcut_fragment.h | 104 ++++++++++--
 grape/fragment/mutable_edgecut_fragment.h     |   5 +-
 grape/fragment/vc_fragment_loader.h           |  22 ++-
 .../parallel/gather_scatter_message_manager.h |  44 ++++-
 grape/utils/memory_inspector.h                |  66 --------
 grape/utils/memory_tracker.cc                 |  59 +++++++
 grape/utils/memory_tracker.h                  |  49 ++++++
 grape/worker/worker.h                         |   6 +-
 20 files changed, 443 insertions(+), 281 deletions(-)
 delete mode 100644 grape/utils/memory_inspector.h
 create mode 100644 grape/utils/memory_tracker.cc
 create mode 100644 grape/utils/memory_tracker.h

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index 64ddae12..871a2ce4 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -50,12 +50,16 @@ class PageRankVC
     ctx.graph_vnum = frag.GetTotalVerticesNum();
 
     {
+#ifdef PROFILING
       ctx.t0 -= GetCurrentTime();
+#endif
       typename fragment_t::template both_vertex_array_t<int> degree(
           frag.Vertices(), 0);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
       // allocate degree array for both src and dst vertices
-      MemoryInspector::GetInstance().allocate(frag.Vertices().size() *
-                                              sizeof(int));
+      MemoryTracker::GetInstance().allocate(frag.Vertices().size() *
+                                            sizeof(int));
+#endif
       int bucket_num = frag.GetBucketNum();
       int concurrency = thread_num();
       if (bucket_num < (concurrency / 2)) {
@@ -81,18 +85,24 @@ class PageRankVC
                   }
                 });
       }
+#ifdef PROFILING
       ctx.t0 += GetCurrentTime();
+#endif
 
       messages.GatherMasterVertices<fragment_t, int, NumericSum<int>>(
           frag, degree, ctx.master_degree);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
       // deallocate degree array for both src and dst vertices
-      MemoryInspector::GetInstance().deallocate(frag.Vertices().size() *
-                                                sizeof(int));
+      MemoryTracker::GetInstance().deallocate(frag.Vertices().size() *
+                                              sizeof(int));
+#endif
     }
 
     double p = 1.0 / ctx.graph_vnum;
     int64_t dangling_vnum_local = 0;
-    ctx.t2 -= GetCurrentTime();
+#ifdef PROFILING
+    ctx.t1 -= GetCurrentTime();
+#endif
     std::vector<int64_t> dangling_vnum_local_vec(thread_num(), 0);
     ForEach(frag.MasterVertices(), [&](int tid, vertex_t v) {
       if (ctx.master_degree[v] == 0) {
@@ -105,15 +115,15 @@ class PageRankVC
     for (auto x : dangling_vnum_local_vec) {
       dangling_vnum_local += x;
     }
-    ctx.t2 += GetCurrentTime();
+#ifdef PROFILING
+    ctx.t1 += GetCurrentTime();
+#endif
 
     Sum(dangling_vnum_local, ctx.total_dangling_vnum);
     ctx.dangling_sum = p * ctx.total_dangling_vnum;
 
-    ctx.t7 -= GetCurrentTime();
     messages.ScatterMasterVertices<fragment_t, double>(frag, ctx.master_result,
                                                        ctx.curr_result);
-    ctx.t7 += GetCurrentTime();
     messages.ForceContinue();
   }
 
@@ -125,50 +135,53 @@ class PageRankVC
                   ctx.delta * ctx.dangling_sum / ctx.graph_vnum;
     ctx.dangling_sum = base * ctx.total_dangling_vnum;
 
-    ctx.t11 -= GetCurrentTime();
     ForEach(frag.Vertices(),
             [&ctx](int tid, vertex_t v) { ctx.next_result[v] = 0; });
-    ctx.t11 += GetCurrentTime();
 
-    if (ctx.step != ctx.max_round) {
-      ctx.t4 -= GetCurrentTime();
+    int bucket_num = frag.GetBucketNum();
+    int concurrency = thread_num();
 
-      int bucket_num = frag.GetBucketNum();
-      int concurrency = thread_num();
-      if (bucket_num < (concurrency / 2)) {
-        ForEach(
-            frag.GetEdges().begin(), frag.GetEdges().end(),
-            [&ctx](int tid, const typename fragment_t::edge_t& e) {
-              atomic_add(ctx.next_result[vertex_t(e.dst)],
-                         ctx.curr_result[vertex_t(e.src)]);
-              atomic_add(ctx.next_result[vertex_t(e.src)],
-                         ctx.curr_result[vertex_t(e.dst)]);
-            },
-            4096);
-      } else {
-        ForEach(0, bucket_num,
-                [&ctx, &frag, bucket_num](int tid, int bucket_id) {
-                  for (int i = 0; i < bucket_num; ++i) {
-                    for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
-                      ctx.next_result[vertex_t(e.dst)] +=
-                          ctx.curr_result[vertex_t(e.src)];
-                    }
-                  }
-                  for (int i = 0; i < bucket_num; ++i) {
-                    for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
-                      ctx.next_result[vertex_t(e.src)] +=
-                          ctx.curr_result[vertex_t(e.dst)];
-                    }
-                  }
-                });
-      }
+#ifdef PROFILING
+    ctx.t2 -= GetCurrentTime();
+#endif
+    if (bucket_num < (concurrency / 2)) {
+      ForEach(
+          frag.GetEdges().begin(), frag.GetEdges().end(),
+          [&ctx](int tid, const typename fragment_t::edge_t& e) {
+            atomic_add(ctx.next_result[vertex_t(e.dst)],
+                       ctx.curr_result[vertex_t(e.src)]);
+            atomic_add(ctx.next_result[vertex_t(e.src)],
+                       ctx.curr_result[vertex_t(e.dst)]);
+          },
+          4096);
+    } else {
+      ForEach(0, bucket_num, [&ctx, &frag, bucket_num](int tid, int bucket_id) {
+        for (int i = 0; i < bucket_num; ++i) {
+          for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
+            ctx.next_result[vertex_t(e.dst)] +=
+                ctx.curr_result[vertex_t(e.src)];
+          }
+        }
+        for (int i = 0; i < bucket_num; ++i) {
+          for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
+            ctx.next_result[vertex_t(e.src)] +=
+                ctx.curr_result[vertex_t(e.dst)];
+          }
+        }
+      });
+    }
 
-      ctx.t4 += GetCurrentTime();
+#ifdef PROFILING
+    ctx.t2 += GetCurrentTime();
+#endif
 
-      messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
-          frag, ctx.next_result, ctx.master_result);
+    messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
+        frag, ctx.next_result, ctx.master_result);
 
-      ctx.t2 -= GetCurrentTime();
+    if (ctx.step != ctx.max_round) {
+#ifdef PROFILING
+      ctx.t1 -= GetCurrentTime();
+#endif
       ForEach(frag.MasterVertices(), [&ctx, base](int tid, vertex_t v) {
         if (ctx.master_degree[v] > 0) {
           ctx.master_result[v] =
@@ -177,55 +190,23 @@ class PageRankVC
           ctx.master_result[v] = base;
         }
       });
-      ctx.t2 += GetCurrentTime();
+#ifdef PROFILING
+      ctx.t1 += GetCurrentTime();
+#endif
 
-      ctx.t7 -= GetCurrentTime();
       messages.ScatterMasterVertices<fragment_t, double>(
           frag, ctx.master_result, ctx.curr_result);
-      ctx.t7 += GetCurrentTime();
       messages.ForceContinue();
     } else {
-      ctx.t4 -= GetCurrentTime();
-      int bucket_num = frag.GetBucketNum();
-      int concurrency = thread_num();
-      if (bucket_num < (concurrency / 2)) {
-        ForEach(
-            frag.GetEdges().begin(), frag.GetEdges().end(),
-            [&ctx](int tid, const typename fragment_t::edge_t& e) {
-              atomic_add(ctx.next_result[vertex_t(e.dst)],
-                         ctx.curr_result[vertex_t(e.src)]);
-              atomic_add(ctx.next_result[vertex_t(e.src)],
-                         ctx.curr_result[vertex_t(e.dst)]);
-            },
-            4096);
-      } else {
-        ForEach(0, bucket_num,
-                [&ctx, &frag, bucket_num](int tid, int bucket_id) {
-                  for (int i = 0; i < bucket_num; ++i) {
-                    for (auto& e : frag.GetEdgesOfBucket(i, bucket_id)) {
-                      ctx.next_result[vertex_t(e.dst)] +=
-                          ctx.curr_result[vertex_t(e.src)];
-                    }
-                  }
-                  for (int i = 0; i < bucket_num; ++i) {
-                    for (auto& e : frag.GetEdgesOfBucket(bucket_id, i)) {
-                      ctx.next_result[vertex_t(e.src)] +=
-                          ctx.curr_result[vertex_t(e.dst)];
-                    }
-                  }
-                });
-      }
-
-      ctx.t4 += GetCurrentTime();
-
-      messages.GatherMasterVertices<fragment_t, double, NumericSum<double>>(
-          frag, ctx.next_result, ctx.master_result);
-
-      ctx.t2 -= GetCurrentTime();
+#ifdef PROFILING
+      ctx.t1 -= GetCurrentTime();
+#endif
       ForEach(frag.MasterVertices(), [&ctx, base](int tid, vertex_t v) {
         ctx.master_result[v] = ctx.master_result[v] * ctx.delta + base;
       });
-      ctx.t2 += GetCurrentTime();
+#ifdef PROFILING
+      ctx.t1 += GetCurrentTime();
+#endif
     }
   }
 };
diff --git a/examples/analytical_apps/pagerank/pagerank_vc_context.h b/examples/analytical_apps/pagerank/pagerank_vc_context.h
index 0faf7a5b..1dd7f001 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc_context.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc_context.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
 #define EXAMPLES_ANALYTICAL_APPS_PAGERANK_PAGERANK_VC_CONTEXT_H_
 
-#include "grape/utils/memory_inspector.h"
+#include "grape/utils/memory_tracker.h"
 
 #include <iomanip>
 
@@ -34,14 +34,16 @@ class PageRankVCContext : public VertexDataContext<FRAG_T, double> {
     next_result.Init(fragment.Vertices());
     master_degree.Init(fragment.MasterVertices());
 
-    MemoryInspector::GetInstance().allocate(fragment.Vertices().size() *
-                                            sizeof(double));
-    MemoryInspector::GetInstance().allocate(fragment.Vertices().size() *
-                                            sizeof(double));
-    MemoryInspector::GetInstance().allocate(fragment.MasterVertices().size() *
-                                            sizeof(double));
-    MemoryInspector::GetInstance().allocate(fragment.MasterVertices().size() *
-                                            sizeof(int));
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    MemoryTracker::GetInstance().allocate(fragment.Vertices().size() *
+                                          sizeof(double));
+    MemoryTracker::GetInstance().allocate(fragment.Vertices().size() *
+                                          sizeof(double));
+    MemoryTracker::GetInstance().allocate(fragment.MasterVertices().size() *
+                                          sizeof(double));
+    MemoryTracker::GetInstance().allocate(fragment.MasterVertices().size() *
+                                          sizeof(int));
+#endif
   }
 
   void Init(GatherScatterMessageManager& messages, double delta,
@@ -59,11 +61,11 @@ class PageRankVCContext : public VertexDataContext<FRAG_T, double> {
          << master_result[v] << std::endl;
     }
 
-    LOG(INFO) << "[frag-" << frag.fid() << "]: init degree: " << t0 << " s, "
-              << "calc master result: " << t2 << " s, "
-              << "propogate: " << t4 << " s, "
-              << "scatter: " << t7 << " s, "
-              << "reset next result: " << t11 << " s";
+#ifdef PROFILING
+    VLOG(2) << "[frag-" << frag.fid() << "]: init degree: " << t0 << " s, "
+            << "calc master result: " << t1 << " s, "
+            << "propogate: " << t2 << " s";
+#endif
   }
 
   typename FRAG_T::template both_vertex_array_t<double> curr_result;
@@ -79,11 +81,11 @@ class PageRankVCContext : public VertexDataContext<FRAG_T, double> {
 
   double dangling_sum = 0.0;
 
-  double t0 = 0;   // init degree
-  double t2 = 0;   // calc master result
-  double t4 = 0;   // propogate
-  double t7 = 0;   // scatter
-  double t11 = 0;  // reset next result
+#ifdef PROFILING
+  double t0 = 0;  // init degree
+  double t1 = 0;  // calc master result
+  double t2 = 0;  // propogate
+#endif
 };
 
 }  // namespace grape
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index 1162d8e3..c7d1b7e1 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "grape/fragment/immutable_vertexcut_fragment.h"
 #include "grape/fragment/loader.h"
+#include "grape/utils/memory_tracker.h"
 #include "pagerank/pagerank_vc.h"
 #include "utils.h"
 
@@ -39,18 +40,18 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
       ImmutableVertexcutFragment<int64_t, grape::EmptyType, grape::EmptyType>;
   std::shared_ptr<FRAG_T> fragment = LoadVertexcutGraph<FRAG_T>(
       FLAGS_efile, FLAGS_vertex_num, comm_spec, graph_spec);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
   VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after loading graph: "
-          << MemoryInspector::GetInstance().GetCurrentMemoryUsage()
-          << " GB, peak: "
-          << MemoryInspector::GetInstance().GetPeakMemoryUsage() << " GB";
+          << MemoryTracker::GetInstance().GetMemoryUsageInfo();
+#endif
   using AppType = APP_T<FRAG_T>;
   auto app = std::make_shared<AppType>();
   DoQuery<FRAG_T, AppType, Args...>(fragment, app, comm_spec, spec, out_prefix,
                                     args...);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
   VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after query: "
-          << MemoryInspector::GetInstance().GetCurrentMemoryUsage()
-          << " GB, peak: "
-          << MemoryInspector::GetInstance().GetPeakMemoryUsage() << " GB";
+          << MemoryTracker::GetInstance().GetMemoryUsageInfo();
+#endif
 }
 
 void RunVC() {
diff --git a/examples/gnn_sampler/append_only_edgecut_fragment.h b/examples/gnn_sampler/append_only_edgecut_fragment.h
index 87cffdc1..24477ba9 100644
--- a/examples/gnn_sampler/append_only_edgecut_fragment.h
+++ b/examples/gnn_sampler/append_only_edgecut_fragment.h
@@ -738,7 +738,8 @@ class AppendOnlyEdgecutFragment
     initOuterVerticesOfFragment();
   }
 
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {}
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& pe_spec) override {}
 
   fid_t GetFragIdByGid(const vid_t& gid) const {
     return id_parser_.get_fragment_id(gid);
diff --git a/grape/cuda/fragment/host_fragment.h b/grape/cuda/fragment/host_fragment.h
index 0b55f911..4ef321bf 100644
--- a/grape/cuda/fragment/host_fragment.h
+++ b/grape/cuda/fragment/host_fragment.h
@@ -119,8 +119,9 @@ class HostFragment : public ImmutableEdgecutFragment<OID_T, VID_T, VDATA_T,
     __allocate_device_fragment__(comm_spec.local_id());
   }
 
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) {
-    base_t::PrepareToRunApp(comm_spec, conf);
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& engine_spec) {
+    base_t::PrepareToRunApp(comm_spec, conf, pe_spec);
 
     Stream stream;
     if (conf.message_strategy ==
diff --git a/grape/cuda/worker/gpu_batch_shuffle_worker.h b/grape/cuda/worker/gpu_batch_shuffle_worker.h
index 2e825bb8..3152912f 100644
--- a/grape/cuda/worker/gpu_batch_shuffle_worker.h
+++ b/grape/cuda/worker/gpu_batch_shuffle_worker.h
@@ -50,7 +50,8 @@ class GPUBatchShuffleWorker {
     prepare_conf.need_split_edges = APP_T::need_split_edges;
     prepare_conf.need_mirror_info = true;
     prepare_conf.need_build_device_vm = APP_T::need_build_device_vm;
-    graph.PrepareToRunApp(comm_spec, prepare_conf);
+    graph.PrepareToRunApp(comm_spec, prepare_conf,
+                          grape::DefaultParallelEngineSpec());
 
     comm_spec_ = comm_spec;
 
diff --git a/grape/cuda/worker/gpu_worker.h b/grape/cuda/worker/gpu_worker.h
index d8f7fca9..82ef3bc8 100644
--- a/grape/cuda/worker/gpu_worker.h
+++ b/grape/cuda/worker/gpu_worker.h
@@ -49,7 +49,8 @@ class GPUWorker {
     prepare_conf.need_split_edges = APP_T::need_split_edges;
     prepare_conf.need_mirror_info = false;
     prepare_conf.need_build_device_vm = APP_T::need_build_device_vm;
-    graph.PrepareToRunApp(comm_spec, prepare_conf);
+    graph.PrepareToRunApp(comm_spec, prepare_conf,
+                          grape::DefaultParallelEngineSpec());
 
     comm_spec_ = comm_spec;
 
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index 4bec75ab..c5d0b221 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
 #define GRAPE_FRAGMENT_BASIC_VC_DS_FRAGMENT_LOADER_H_
 
-#include "grape/utils/memory_inspector.h"
+#include "grape/utils/memory_tracker.h"
 
 namespace grape {
 
@@ -83,17 +83,22 @@ class BasicVCDSFragmentLoader {
 
  public:
   explicit BasicVCDSFragmentLoader(const CommSpec& comm_spec, int64_t vnum,
-                                   int load_concurrency, int bucket_num)
+                                   int load_concurrency)
       : comm_spec_(comm_spec),
         partitioner_(comm_spec.fnum(), vnum),
-        bucket_num_(bucket_num),
-        bucketer_(partitioner_, bucket_num),
+        bucketer_(nullptr),
         vnum_(vnum),
         load_concurrency_(load_concurrency) {
     comm_spec_.Dup();
+
+    MPI_Allreduce(&load_concurrency_, &bucket_num_, 1, MPI_INT, MPI_MIN,
+                  comm_spec_.comm());
+    bucketer_ = std::unique_ptr<VCEdgeBucketer<oid_t>>(
+        new VCEdgeBucketer<oid_t>(partitioner_, bucket_num_));
+
     partition_bucket_edge_num_.resize(comm_spec_.fnum());
     for (auto& vec : partition_bucket_edge_num_) {
-      vec.resize(bucket_num * bucket_num, 0);
+      vec.resize(bucket_num_ * bucket_num_, 0);
     }
   }
 
@@ -111,36 +116,22 @@ class BasicVCDSFragmentLoader {
   }
 
   void RecordEdge(const oid_t& src, const oid_t& dst) {
-    auto pair = bucketer_.get_partition_bucket_id(src, dst);
+    auto pair = bucketer_->get_partition_bucket_id(src, dst);
     ++partition_bucket_edge_num_[pair.first][pair.second];
   }
 
   void AllocateBuffers() {
-    {
-      std::vector<std::vector<size_t>> partition_bucket_edge_num_out(
-          comm_spec_.fnum());
-      sync_comm::AllToAll(partition_bucket_edge_num_,
-                          partition_bucket_edge_num_out, comm_spec_.comm());
-      std::vector<size_t> partition_edge_num(bucket_num_ * bucket_num_, 0);
-      for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
-        for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
-          partition_edge_num[i] += partition_bucket_edge_num_out[fid][i];
-        }
+    for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
+      if (comm_spec_.FragToWorker(fid) == comm_spec_.worker_id()) {
+        MPI_Reduce(MPI_IN_PLACE, partition_bucket_edge_num_[fid].data(),
+                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
+      } else {
+        MPI_Reduce(partition_bucket_edge_num_[fid].data(), nullptr,
+                   partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
+                   MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
       }
-      partition_bucket_edge_num_[comm_spec_.fid()] =
-          std::move(partition_edge_num);
     }
-    // for (fid_t fid = 0; fid < comm_spec_.fnum(); ++fid) {
-    //   if (comm_spec_.FragToWorker(fid) == comm_spec_.worker_id()) {
-    //     MPI_Reduce(MPI_IN_PLACE, partition_bucket_edge_num_[fid].data(),
-    //                partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
-    //                MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
-    //   } else {
-    //     MPI_Reduce(partition_bucket_edge_num_[fid].data(), nullptr,
-    //                partition_bucket_edge_num_[fid].size(), MPI_LONG_LONG_INT,
-    //                MPI_SUM, comm_spec_.FragToWorker(fid), comm_spec_.comm());
-    //   }
-    // }
     size_t edge_num = 0;
     bucket_cursor_ =
         std::vector<std::atomic<size_t>>(bucket_num_ * bucket_num_);
@@ -149,7 +140,10 @@ class BasicVCDSFragmentLoader {
       edge_num += partition_bucket_edge_num_[comm_spec_.fid()][k];
     }
     edges_.resize(edge_num);
-    MemoryInspector::GetInstance().allocate((sizeof(edge_t)) * edge_num);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // allocate memory for edges
+    MemoryTracker::GetInstance().allocate((sizeof(edge_t)) * edge_num);
+#endif
 
     got_edges_.SetProducerNum(2);
 
@@ -162,17 +156,26 @@ class BasicVCDSFragmentLoader {
         edges_to_frag_[fid].DisableComm();
       }
     }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // allocate memory for edge shuffle-out buffer
+    MemoryTracker::GetInstance().allocate(
+        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
+        shuffle_out_size);
+#endif
     for (int i = 0; i < load_concurrency_; ++i) {
       edge_move_threads_.emplace_back([this] {
         ShuffleBufferTuple<oid_t, oid_t, edata_t> cur;
         std::vector<std::vector<edge_t>> edges_cache(bucket_num_ * bucket_num_);
-        MemoryInspector::GetInstance().allocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // allocate memory for thread local edge cache
+        MemoryTracker::GetInstance().allocate(
             (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+#endif
         while (got_edges_.Get(cur)) {
           size_t cur_size = cur.size();
           foreach_rval(cur, [&edges_cache, this](oid_t src, oid_t dst,
                                                  edata_t data) {
-            int bucket_id = bucketer_.get_bucket_id(src, dst);
+            int bucket_id = bucketer_->get_bucket_id(src, dst);
             edges_cache[bucket_id].emplace_back(src, dst, data);
             if (edges_cache[bucket_id].size() >= 4096) {
               size_t cursor = bucket_cursor_[bucket_id].fetch_add(
@@ -182,8 +185,11 @@ class BasicVCDSFragmentLoader {
               edges_cache[bucket_id].clear();
             }
           });
-          MemoryInspector::GetInstance().deallocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+          // deallocate edge shuffle-in buffer
+          MemoryTracker::GetInstance().deallocate(
               (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur_size);
+#endif
         }
         for (int i = 0; i < bucket_num_ * bucket_num_; ++i) {
           if (!edges_cache[i].empty()) {
@@ -192,16 +198,16 @@ class BasicVCDSFragmentLoader {
                       edges_.begin() + cursor);
           }
         }
-        MemoryInspector::GetInstance().deallocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // deallocate memory for thread local edge cache
+        MemoryTracker::GetInstance().deallocate(
             (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+#endif
       });
     }
     edge_recv_thread_ =
         std::thread(&BasicVCDSFragmentLoader::edgeRecvRoutine, this);
     recv_thread_running_ = true;
-    MemoryInspector::GetInstance().allocate(
-        (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
-        shuffle_out_size);
   }
 
   void AddEdge(const oid_t& src, const oid_t& dst, const edata_t& data) {
@@ -209,9 +215,12 @@ class BasicVCDSFragmentLoader {
     edges_to_frag_[fid].Emplace(src, dst, data);
     if (fid == comm_spec_.fid() &&
         edges_to_frag_[fid].buffers().size() >= shuffle_out_size) {
-      MemoryInspector::GetInstance().allocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+      // allocate memory for edge shuffle-out(to self) buffer
+      MemoryTracker::GetInstance().allocate(
           (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
           edges_to_frag_[fid].buffers().size());
+#endif
       got_edges_.Put(std::move(edges_to_frag_[fid].buffers()));
       edges_to_frag_[fid].Clear();
     }
@@ -225,9 +234,12 @@ class BasicVCDSFragmentLoader {
     edge_recv_thread_.join();
     recv_thread_running_ = false;
     edges_to_frag_.clear();
-    MemoryInspector::GetInstance().deallocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // deallocate memory for edge shuffle-out buffer
+    MemoryTracker::GetInstance().deallocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
         shuffle_out_size);
+#endif
     got_edges_.DecProducerNum();
     for (auto& thrd : edge_move_threads_) {
       thrd.join();
@@ -272,8 +284,7 @@ class BasicVCDSFragmentLoader {
     }
 
     fragment.reset(new fragment_t());
-    fragment->Init(comm_spec_, vnum_, std::move(edges_), bucket_num_,
-                   std::move(bucket_edge_offset));
+    fragment->Init(comm_spec_, vnum_, std::move(edges_));
   }
 
  private:
@@ -288,9 +299,12 @@ class BasicVCDSFragmentLoader {
         break;
       }
       CHECK_EQ(dst_fid, comm_spec_.fid());
-      MemoryInspector::GetInstance().allocate(
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+      // allocate memory for edge shuffle-in buffer
+      MemoryTracker::GetInstance().allocate(
           (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
           data_in.buffers().size());
+#endif
       got_edges_.Put(std::move(data_in.buffers()));
       data_in.Clear();
     }
@@ -302,7 +316,7 @@ class BasicVCDSFragmentLoader {
   CommSpec comm_spec_;
   VCPartitioner<oid_t> partitioner_;
   int bucket_num_;
-  VCEdgeBucketer<oid_t> bucketer_;
+  std::unique_ptr<VCEdgeBucketer<oid_t>> bucketer_;
 
   int64_t vnum_;
   int load_concurrency_;
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
index 1b05db9e..0bcaec3a 100644
--- a/grape/fragment/basic_vc_fragment_loader.h
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
 #define GRAPE_FRAGMENT_BASIC_VC_FRAGMENT_LOADER_H_
 
-#include "grape/utils/memory_inspector.h"
+#include "grape/utils/memory_tracker.h"
 
 namespace grape {
 
@@ -33,10 +33,10 @@ class BasicVCFragmentLoader {
 
  public:
   explicit BasicVCFragmentLoader(const CommSpec& comm_spec, int64_t vnum,
-                                 int load_concurrency, int bucket_num)
+                                 int load_concurrency)
       : comm_spec_(comm_spec),
         partitioner_(comm_spec.fnum(), vnum),
-        bucket_num_(bucket_num),
+        bucket_num_(load_concurrency),
         vnum_(vnum),
         load_concurrency_(load_concurrency) {
     comm_spec_.Dup();
@@ -50,10 +50,12 @@ class BasicVCFragmentLoader {
       }
     }
 
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     // allocate shuffle out buffers
-    MemoryInspector::GetInstance().allocate(
+    MemoryTracker::GetInstance().allocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
         shuffle_out_size);
+#endif
     edge_recv_thread_ =
         std::thread(&BasicVCFragmentLoader::edgeRecvRoutine, this);
     recv_thread_running_ = true;
@@ -97,14 +99,18 @@ class BasicVCFragmentLoader {
       got_edges_memory_usage +=
           buf.size() * (sizeof(oid_t) * 2 + sizeof(edata_t));
     }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     // allocate edges recv buffer
-    MemoryInspector::GetInstance().allocate(got_edges_memory_usage);
+    MemoryTracker::GetInstance().allocate(got_edges_memory_usage);
+#endif
 
     edges_to_frag_[comm_spec_.fid()].Clear();
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     // deallocate shuffle out buffers
-    MemoryInspector::GetInstance().deallocate(
+    MemoryTracker::GetInstance().deallocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
         shuffle_out_size);
+#endif
 
     std::vector<size_t> bucket_edge_num(bucket_num_ * bucket_num_, 0);
     {
@@ -157,15 +163,19 @@ class BasicVCFragmentLoader {
 
     std::vector<edge_t> edges;
     edges.resize(edge_num);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     // allocate edges buffer
-    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edge_num);
+    MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edge_num);
+#endif
 
     {
       static constexpr size_t thread_local_cache_size = 4096;
+#ifdef TRACKING_MEMORY_ALLOCATIONS
       // allocate thread local cache
-      MemoryInspector::GetInstance().allocate(
+      MemoryTracker::GetInstance().allocate(
           sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
           load_concurrency_);
+#endif
       std::vector<std::thread> insert_threads;
       for (int i = 0; i < load_concurrency_; ++i) {
         insert_threads.emplace_back(
@@ -210,13 +220,17 @@ class BasicVCFragmentLoader {
       for (auto& thrd : insert_threads) {
         thrd.join();
       }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
       // deallocate thread local cache
-      MemoryInspector::GetInstance().deallocate(
+      MemoryTracker::GetInstance().deallocate(
           sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
           load_concurrency_);
+#endif
       got_edges_.clear();
+#ifdef TRACKING_MEMORY_ALLOCATIONS
       // deallocate edges recv buffer
-      MemoryInspector::GetInstance().deallocate(got_edges_memory_usage);
+      MemoryTracker::GetInstance().deallocate(got_edges_memory_usage);
+#endif
     }
 
     {
@@ -248,8 +262,7 @@ class BasicVCFragmentLoader {
     }
 
     fragment.reset(new fragment_t());
-    fragment->Init(comm_spec_, vnum_, std::move(edges), bucket_num_,
-                   std::move(bucket_edge_offset));
+    fragment->Init(comm_spec_, vnum_, std::move(edges));
   }
 
  private:
diff --git a/grape/fragment/csr_edgecut_fragment_base.h b/grape/fragment/csr_edgecut_fragment_base.h
index 1b51e91e..38934299 100644
--- a/grape/fragment/csr_edgecut_fragment_base.h
+++ b/grape/fragment/csr_edgecut_fragment_base.h
@@ -387,7 +387,8 @@ class CSREdgecutFragmentBase
 
  public:
   using base_t::initMirrorInfo;
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& pe_spec) override {
     if (conf.message_strategy == MessageStrategy::kAlongEdgeToOuterVertex ||
         conf.message_strategy ==
             MessageStrategy::kAlongIncomingEdgeToOuterVertex ||
diff --git a/grape/fragment/fragment_base.h b/grape/fragment/fragment_base.h
index 7ef41b41..6e513485 100644
--- a/grape/fragment/fragment_base.h
+++ b/grape/fragment/fragment_base.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "grape/graph/adj_list.h"
 #include "grape/graph/edge.h"
 #include "grape/graph/vertex.h"
+#include "grape/parallel/parallel_engine_spec.h"
 #include "grape/vertex_map/vertex_map.h"
 #include "grape/worker/comm_spec.h"
 
@@ -109,7 +110,8 @@ class FragmentBase {
    * @param strategy
    * @param need_split_edge
    */
-  virtual void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) = 0;
+  virtual void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                               const ParallelEngineSpec& pe_spec) = 0;
 
  protected:
   template <typename IOADAPTOR_T>
diff --git a/grape/fragment/immutable_edgecut_fragment.h b/grape/fragment/immutable_edgecut_fragment.h
index cbcab57b..a17cc392 100644
--- a/grape/fragment/immutable_edgecut_fragment.h
+++ b/grape/fragment/immutable_edgecut_fragment.h
@@ -583,8 +583,9 @@ class ImmutableEdgecutFragment
     io_adaptor->Close();
   }
 
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {
-    base_t::PrepareToRunApp(comm_spec, conf);
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& pe_spec) override {
+    base_t::PrepareToRunApp(comm_spec, conf, pe_spec);
     if (conf.need_split_edges_by_fragment && !splited_edges_by_fragment_) {
       splitEdgesByFragment();
       splited_edges_by_fragment_ = true;
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index bb14aaa5..ffc5ad36 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -23,7 +23,7 @@ limitations under the License.
 #include "grape/fragment/fragment_base.h"
 #include "grape/graph/edge.h"
 #include "grape/types.h"
-#include "grape/utils/memory_inspector.h"
+#include "grape/utils/memory_tracker.h"
 #include "grape/utils/vertex_array.h"
 #include "grape/vertex_map/partitioner.h"
 #include "grape/worker/comm_spec.h"
@@ -75,8 +75,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
   using base_t::fnum_;
 
   void Init(const CommSpec& comm_spec, int64_t vnum,
-            std::vector<edge_t>&& edges, int bucket_num,
-            std::vector<size_t>&& bucket_edge_offsets) {
+            std::vector<edge_t>&& edges) {
     base_t::init(comm_spec.fid(), comm_spec.fnum(), false);
 
     partitioner_.init(comm_spec.fnum(), vnum);
@@ -84,10 +83,10 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     dst_vertices_ = partitioner_.get_dst_vertices(fid_);
     master_vertices_ = partitioner_.get_master_vertices(fid_);
 
-    int64_t src_begin = src_vertices_.begin_value();
-    int64_t src_end = src_vertices_.end_value();
-    int64_t dst_begin = dst_vertices_.begin_value();
-    int64_t dst_end = dst_vertices_.end_value();
+    oid_t src_begin = src_vertices_.begin_value();
+    oid_t src_end = src_vertices_.end_value();
+    oid_t dst_begin = dst_vertices_.begin_value();
+    oid_t dst_end = dst_vertices_.end_value();
 
     if (src_end < dst_begin) {
       vertices_ = both_vertices_t(src_begin, src_end, dst_begin, dst_end);
@@ -100,16 +99,27 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
 
 #ifdef USE_EDGE_ARRAY
     edges_.resize(edges.size());
-    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edges_.size());
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // reallocate memory for edges
+    MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
+    MemoryTracker::GetInstance().deallocate(sizeof(edge_t) * edges_.size());
+#endif
     std::copy(edges.begin(), edges.end(), edges_.begin());
 #else
     edges_ = std::move(edges);
 #endif
-    bucket_num_ = bucket_num;
-    bucket_edge_offsets_ = std::move(bucket_edge_offsets);
+
+    bucket_num_ = 1;
+    bucket_edge_offsets_.push_back(0);
+    bucket_edge_offsets_.push_back(edges_.size());
   }
 
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {}
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& pe_spec) override {
+    if (bucket_num_ < static_cast<int>(pe_spec.thread_num)) {
+      buildBucket(pe_spec.thread_num);
+    }
+  }
 
   const vertices_t& SourceVertices() const { return src_vertices_; }
   const vertices_t& DestinationVertices() const { return dst_vertices_; }
@@ -138,7 +148,6 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     InArchive arc;
     arc << edges_.size();
     arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_;
-    arc << bucket_num_ << bucket_edge_offsets_;
     CHECK(io_adaptor->WriteArchive(arc));
     arc.Clear();
 
@@ -178,13 +187,15 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     CHECK_EQ(fid_, comm_spec.fid());
     CHECK_EQ(fnum_, comm_spec.fnum());
     arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_;
-    arc >> bucket_num_ >> bucket_edge_offsets_;
 
     partitioner_.deserialize(io_adaptor);
 
     if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
       LOG(INFO) << "is pod";
       edges_.resize(edge_num);
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+      MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
+#endif
       if (edge_num > 0) {
         CHECK(io_adaptor->Read(edges_.data(), edge_num * sizeof(edge_t)));
       }
@@ -195,7 +206,9 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
       arc >> edges_;
     }
 
-    MemoryInspector::GetInstance().allocate(sizeof(edge_t) * edges_.size());
+    bucket_num_ = 1;
+    bucket_edge_offsets_.push_back(0);
+    bucket_edge_offsets_.push_back(edges_.size());
   }
 
   size_t GetTotalVerticesNum() const override {
@@ -206,7 +219,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
 
   size_t GetVerticesNum() const override { return vertices_.size(); }
 
-  const VCPartitioner<int64_t>& GetPartitioner() const { return partitioner_; }
+  const VCPartitioner<oid_t>& GetPartitioner() const { return partitioner_; }
 
   edge_bucket_t GetEdgesOfBucket(int src_bucket_id, int dst_bucket_id) const {
     int idx = src_bucket_id * bucket_num_ + dst_bucket_id;
@@ -220,7 +233,66 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
   int GetBucketNum() const { return bucket_num_; }
 
  private:
-  VCPartitioner<int64_t> partitioner_;
+  void buildBucket(int thread_num) {
+    if (bucket_num_ == thread_num) {
+      return;
+    }
+    std::vector<std::vector<size_t>> thread_bucket_edge_nums(thread_num);
+    std::vector<std::thread> threads;
+    std::atomic<size_t> edge_idx(0);
+    size_t edge_num = edges_.size();
+
+    oid_t src_begin = src_vertices_.begin_value();
+    size_t src_size = src_vertices_.size();
+    size_t src_chunk = (src_size + thread_num - 1) / thread_num;
+    oid_t dst_begin = dst_vertices_.begin_value();
+    size_t dst_size = dst_vertices_.size();
+    size_t dst_chunk = (dst_size + thread_num - 1) / thread_num;
+
+    for (int i = 0; i < thread_num; ++i) {
+      threads.emplace_back(
+          [&, this](int tid) {
+            thread_bucket_edge_nums[tid].clear();
+            thread_bucket_edge_nums[tid].resize(thread_num * thread_num, 0);
+            while (true) {
+              size_t idx = std::min(edge_idx.fetch_add(4096), edge_num);
+              size_t end = std::min(idx + 4096, edge_num);
+              if (idx == end) {
+                break;
+              }
+              while (idx < end) {
+                auto& edge = edges_[idx];
+                int src_bucket_id = (edge.src - src_begin) / src_chunk;
+                int dst_bucket_id = (edge.dst - dst_begin) / dst_chunk;
+                thread_bucket_edge_nums[tid][src_bucket_id * thread_num +
+                                             dst_bucket_id]++;
+                ++idx;
+              }
+            }
+          },
+          i);
+    }
+    for (auto& t : threads) {
+      t.join();
+    }
+    std::vector<size_t> bucket_edge_num(thread_num * thread_num, 0);
+    for (int i = 0; i < thread_num; ++i) {
+      for (int j = 0; j < thread_num * thread_num; ++j) {
+        bucket_edge_num[j] += thread_bucket_edge_nums[i][j];
+      }
+    }
+
+    bucket_num_ = thread_num;
+    bucket_edge_offsets_.clear();
+    edge_num = 0;
+    for (size_t i = 0; i < bucket_edge_num.size(); ++i) {
+      bucket_edge_offsets_.push_back(edge_num);
+      edge_num += bucket_edge_num[i];
+    }
+    bucket_edge_offsets_.push_back(edge_num);
+  }
+
+  VCPartitioner<oid_t> partitioner_;
   vertices_t src_vertices_;
   vertices_t dst_vertices_;
   both_vertices_t vertices_;
diff --git a/grape/fragment/mutable_edgecut_fragment.h b/grape/fragment/mutable_edgecut_fragment.h
index 02702ed6..7182391a 100644
--- a/grape/fragment/mutable_edgecut_fragment.h
+++ b/grape/fragment/mutable_edgecut_fragment.h
@@ -480,8 +480,9 @@ class MutableEdgecutFragment
     io_adaptor->Close();
   }
 
-  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf) override {
-    base_t::PrepareToRunApp(comm_spec, conf);
+  void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
+                       const ParallelEngineSpec& pe_spec) override {
+    base_t::PrepareToRunApp(comm_spec, conf, pe_spec);
     if (conf.need_split_edges_by_fragment) {
       LOG(FATAL) << "MutableEdgecutFragment cannot split edges by fragment";
     } else if (conf.need_split_edges) {
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
index c37ed090..17da0ec8 100644
--- a/grape/fragment/vc_fragment_loader.h
+++ b/grape/fragment/vc_fragment_loader.h
@@ -72,9 +72,8 @@ class VCFragmentLoader {
 
     auto basic_fragment_loader =
         std::unique_ptr<BasicVCFragmentLoader<fragment_t>>(
-            new BasicVCFragmentLoader<fragment_t>(
-                comm_spec_, vnum, spec.load_concurrency,
-                std::thread::hardware_concurrency()));
+            new BasicVCFragmentLoader<fragment_t>(comm_spec_, vnum,
+                                                  spec.load_concurrency));
 
     auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
     io_adaptor->SetPartialRead(comm_spec_.worker_id(), comm_spec_.worker_num());
@@ -120,10 +119,10 @@ class VCFragmentLoader {
     if (comm_spec_.worker_id() == 0) {
       VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
     }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
-            << MemoryInspector::GetInstance().GetCurrentMemoryUsage() << " GB"
-            << ", peak: " << MemoryInspector::GetInstance().GetPeakMemoryUsage()
-            << " GB";
+            << MemoryTracker::GetInstance().GetMemoryUsageInfo();
+#endif
     return fragment;
   }
 
@@ -133,9 +132,8 @@ class VCFragmentLoader {
 
     auto basic_fragment_loader =
         std::unique_ptr<BasicVCDSFragmentLoader<fragment_t>>(
-            new BasicVCDSFragmentLoader<fragment_t>(
-                comm_spec_, vnum, spec.load_concurrency,
-                std::thread::hardware_concurrency()));
+            new BasicVCDSFragmentLoader<fragment_t>(comm_spec_, vnum,
+                                                    spec.load_concurrency));
     {
       auto io_adaptor = std::unique_ptr<io_adaptor_t>(new io_adaptor_t(efile));
       io_adaptor->SetPartialRead(comm_spec_.worker_id(),
@@ -225,10 +223,10 @@ class VCFragmentLoader {
     if (comm_spec_.worker_id() == 0) {
       VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
     }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
     VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
-            << MemoryInspector::GetInstance().GetCurrentMemoryUsage() << " GB"
-            << ", peak: " << MemoryInspector::GetInstance().GetPeakMemoryUsage()
-            << " GB";
+            << MemoryTracker::GetInstance().GetMemoryUsageInfo();
+#endif
     return fragment;
   }
 
diff --git a/grape/parallel/gather_scatter_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
index 21aaa85a..1ed12338 100644
--- a/grape/parallel/gather_scatter_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -73,8 +73,11 @@ class GatherScatterMessageManager : public MessageManagerBase {
   }
 
   void Finalize() override {
-    LOG(INFO) << "[frag-" << fid_ << "] gather comm: " << t0_gather_comm_
-              << ", gather calc: " << t1_gather_calc_;
+#ifdef PROFILING
+    VLOG(2) << "[frag-" << fid_ << "] gather comm: " << t0_gather_comm_
+            << " s, gather calc: " << t1_gather_calc_
+            << " s, scatter: " << t2_scatter_ << " s";
+#endif
   }
 
   void ForceContinue() override {
@@ -103,7 +106,9 @@ class GatherScatterMessageManager : public MessageManagerBase {
       LOG(FATAL) << "not implemented for non-POD type";
     }
 
+#ifdef PROFILING
     t0_gather_comm_ -= GetCurrentTime();
+#endif
     const auto& partitioner = frag.GetPartitioner();
     const auto& input_range = input.GetVertexRange();
     std::vector<MPI_Request> requests;
@@ -131,8 +136,10 @@ class GatherScatterMessageManager : public MessageManagerBase {
       auto src_vertices = partitioner.get_src_vertices(src_fid);
       if (output_range.IsSubsetOf(src_vertices)) {
         recv_buffers.emplace_back(output_size);
-        MemoryInspector::GetInstance().allocate(output_size *
-                                                sizeof(MESSAGE_T));
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // allocate memory for received messages
+        MemoryTracker::GetInstance().allocate(output_size * sizeof(MESSAGE_T));
+#endif
         sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
                                 src_fid, 0, comm_, requests);
         continue;
@@ -142,8 +149,10 @@ class GatherScatterMessageManager : public MessageManagerBase {
       auto dst_vertices = partitioner.get_dst_vertices(src_fid);
       if (output_range.IsSubsetOf(dst_vertices)) {
         recv_buffers.emplace_back(output_size);
-        MemoryInspector::GetInstance().allocate(output_size *
-                                                sizeof(MESSAGE_T));
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // allocate memory for received messages
+        MemoryTracker::GetInstance().allocate(output_size * sizeof(MESSAGE_T));
+#endif
         sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
                                 src_fid, 0, comm_, requests);
         continue;
@@ -153,9 +162,14 @@ class GatherScatterMessageManager : public MessageManagerBase {
     }
 
     MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+#ifdef PROFILING
     t0_gather_comm_ += GetCurrentTime();
+#endif
 
+#ifdef PROFILING
     t1_gather_calc_ -= GetCurrentTime();
+#endif
+
     MESSAGE_T* output_data = &output[*output_range.begin()];
     std::vector<std::thread> threads;
     int thread_num = std::thread::hardware_concurrency();
@@ -204,11 +218,16 @@ class GatherScatterMessageManager : public MessageManagerBase {
       thrd.join();
     }
 
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // deallocate memory for received messages
     for (auto& recv_buffer : recv_buffers) {
-      MemoryInspector::GetInstance().deallocate(recv_buffer.size() *
-                                                sizeof(MESSAGE_T));
+      MemoryTracker::GetInstance().deallocate(recv_buffer.size() *
+                                              sizeof(MESSAGE_T));
     }
+#endif
+#ifdef PROFILING
     t1_gather_calc_ += GetCurrentTime();
+#endif
   }
 
   template <typename GRAPH_T, typename MESSAGE_T>
@@ -216,6 +235,9 @@ class GatherScatterMessageManager : public MessageManagerBase {
       const GRAPH_T& frag,
       const typename GRAPH_T::template vertex_array_t<MESSAGE_T>& input,
       typename GRAPH_T::template both_vertex_array_t<MESSAGE_T>& output) {
+#ifdef PROFILING
+    t2_scatter_ -= GetCurrentTime();
+#endif
     if (!std::is_pod<MESSAGE_T>::value) {
       LOG(FATAL) << "not implemented for non-POD type";
     }
@@ -268,6 +290,9 @@ class GatherScatterMessageManager : public MessageManagerBase {
     }
 
     MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
+#ifdef PROFILING
+    t2_scatter_ += GetCurrentTime();
+#endif
   }
 
  private:
@@ -284,8 +309,11 @@ class GatherScatterMessageManager : public MessageManagerBase {
   TerminateInfo terminate_info_;
   bool vote_terminate_;
 
+#ifdef PROFILING
   double t0_gather_comm_ = 0;
   double t1_gather_calc_ = 0;
+  double t2_scatter_ = 0;
+#endif
 };
 
 }  // namespace grape
diff --git a/grape/utils/memory_inspector.h b/grape/utils/memory_inspector.h
deleted file mode 100644
index b0398e11..00000000
--- a/grape/utils/memory_inspector.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/** Copyright 2020 Alibaba Group Holding Limited.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#ifndef GRAPE_UTILS_MEMORY_INSPECTOR_H_
-#define GRAPE_UTILS_MEMORY_INSPECTOR_H_
-
-#include "glog/logging.h"
-#include "grape/utils/concurrent_queue.h"
-
-namespace grape {
-
-struct MemoryInspector {
-  static MemoryInspector& GetInstance() {
-    static MemoryInspector instance;
-    return instance;
-  }
-
-  void allocate(size_t size) {
-    lock_.lock();
-    current_memory_usage += size;
-    peak_memory_usage = std::max(current_memory_usage, peak_memory_usage);
-    lock_.unlock();
-  }
-
-  void deallocate(size_t size) {
-    lock_.lock();
-    current_memory_usage -= size;
-    lock_.unlock();
-  }
-
-  ~MemoryInspector() {
-    double peak_memory_usage_gb = peak_memory_usage / 1024.0 / 1024.0 / 1024.0;
-    LOG(INFO) << "Peak memory usage: " << peak_memory_usage_gb << " GB";
-  }
-
-  double GetCurrentMemoryUsage() const {
-    return current_memory_usage / 1024.0 / 1024.0 / 1024.0;
-  }
-  double GetPeakMemoryUsage() const {
-    return peak_memory_usage / 1024.0 / 1024.0 / 1024.0;
-  }
-
- private:
-  MemoryInspector() = default;
-
-  size_t current_memory_usage = 0;
-  size_t peak_memory_usage = 0;
-
-  SpinLock lock_;
-};
-
-}  // namespace grape
-
-#endif  // GRAPE_UTILS_MEMORY_INSPECTOR_H_
diff --git a/grape/utils/memory_tracker.cc b/grape/utils/memory_tracker.cc
new file mode 100644
index 00000000..bacb3ed3
--- /dev/null
+++ b/grape/utils/memory_tracker.cc
@@ -0,0 +1,59 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "grape/utils/memory_tracker.h"
+
+namespace grape {
+
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+
+MemoryTracker& MemoryTracker::GetInstance() {
+  static MemoryTracker instance;
+  return instance;
+}
+
+MemoryTracker::MemoryTracker()
+    : current_memory_usage_(0), peak_memory_usage_(0) {}
+MemoryTracker::~MemoryTracker() {}
+
+void MemoryTracker::allocate(size_t size) {
+  std::lock_guard<SpinLock> lock(lock_);
+  current_memory_usage_ += size;
+  peak_memory_usage_ = std::max(current_memory_usage_, peak_memory_usage_);
+}
+
+void MemoryTracker::deallocate(size_t size) {
+  std::lock_guard<SpinLock> lock(lock_);
+  current_memory_usage_ =
+      (current_memory_usage_ > size) ? (current_memory_usage_ - size) : 0;
+}
+
+size_t MemoryTracker::GetCurrentMemoryUsage() const {
+  return current_memory_usage_;
+}
+
+size_t MemoryTracker::GetPeakMemoryUsage() const { return peak_memory_usage_; }
+
+std::string MemoryTracker::GetMemoryUsageInfo() const {
+  std::lock_guard<SpinLock> lock(lock_);
+  return "Current memory usage: " +
+         std::to_string(current_memory_usage_ / 1024.0 / 1024.0 / 1024.0) +
+         " GB, peak memory usage: " +
+         std::to_string(peak_memory_usage_ / 1024.0 / 1024.0 / 1024.0) + " GB.";
+}
+
+#endif
+
+}  // namespace grape
diff --git a/grape/utils/memory_tracker.h b/grape/utils/memory_tracker.h
new file mode 100644
index 00000000..311a1e9d
--- /dev/null
+++ b/grape/utils/memory_tracker.h
@@ -0,0 +1,49 @@
+/** Copyright 2020 Alibaba Group Holding Limited.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#ifndef GRAPE_UTILS_MEMORY_TRACKER_H_
+#define GRAPE_UTILS_MEMORY_TRACKER_H_
+
+#include "grape/utils/concurrent_queue.h"
+
+namespace grape {
+
+#define TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+
+struct MemoryTracker {
+  static MemoryTracker& GetInstance();
+
+  void allocate(size_t size);
+  void deallocate(size_t size);
+
+  size_t GetCurrentMemoryUsage() const;
+  size_t GetPeakMemoryUsage() const;
+  std::string GetMemoryUsageInfo() const;
+
+ private:
+  MemoryTracker();
+  ~MemoryTracker();
+
+  size_t current_memory_usage_;
+  size_t peak_memory_usage_;
+  mutable SpinLock lock_;
+};
+
+#endif
+
+}  // namespace grape
+
+#endif  // GRAPE_UTILS_MEMORY_TRACKER_H_
diff --git a/grape/worker/worker.h b/grape/worker/worker.h
index e872839c..de356f08 100644
--- a/grape/worker/worker.h
+++ b/grape/worker/worker.h
@@ -82,8 +82,9 @@ class Worker {
   void Init(const CommSpec& comm_spec,
             const ParallelEngineSpec& pe_spec = DefaultParallelEngineSpec()) {
     auto& graph = *fragment_;
+    pe_spec_ = pe_spec;
     // prepare for the query
-    graph.PrepareToRunApp(comm_spec, prepare_conf_);
+    graph.PrepareToRunApp(comm_spec, prepare_conf_, pe_spec_);
 
     comm_spec_ = comm_spec;
     MPI_Barrier(comm_spec_.comm());
@@ -212,7 +213,7 @@ class Worker {
       std::is_base_of<MutationContext<fragment_t>, T>::value>::type
   processMutation() {
     context_->apply_mutation(fragment_, comm_spec_);
-    fragment_->PrepareToRunApp(comm_spec_, prepare_conf_);
+    fragment_->PrepareToRunApp(comm_spec_, prepare_conf_, pe_spec_);
   }
 
   template <typename T = context_t>
@@ -227,6 +228,7 @@ class Worker {
 
   CommSpec comm_spec_;
   PrepareConf prepare_conf_;
+  ParallelEngineSpec pe_spec_;
 };
 
 template <typename APP_T>

From 74ea0acc26df2e7dd6ed0f489b710a7a02613fe6 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Sat, 14 Sep 2024 16:03:42 +0800
Subject: [PATCH 09/15] xx

---
 examples/analytical_apps/run_app.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/examples/analytical_apps/run_app.cc b/examples/analytical_apps/run_app.cc
index 6dfdd063..7af79bd8 100644
--- a/examples/analytical_apps/run_app.cc
+++ b/examples/analytical_apps/run_app.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// #include "run_app.h"
+#include "run_app.h"
 
 #include <gflags/gflags.h>
 #include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 
-// #include "run_app_opt.h"
+#include "run_app_opt.h"
 #include "run_app_vc.h"
 
 int main(int argc, char* argv[]) {
@@ -41,14 +41,14 @@ int main(int argc, char* argv[]) {
 
   std::string name = FLAGS_application;
   if (FLAGS_opt) {
-    // grape::RunOpt();
+    grape::RunOpt();
   } else if (FLAGS_vc) {
     grape::RunVC();
   } else {
     if (name.find("sssp") != std::string::npos) {
-      // grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
+      grape::Run<int64_t, uint32_t, grape::EmptyType, double>();
     } else {
-      // grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
+      grape::Run<int64_t, uint32_t, grape::EmptyType, grape::EmptyType>();
     }
   }
 

From 260ae20a08e5edf3bcd810b457c4a8648c78c39c Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Wed, 18 Sep 2024 14:06:32 +0800
Subject: [PATCH 10/15] trim logs

---
 examples/analytical_apps/bc/bc_context.h      |  5 ----
 .../analytical_apps/bfs/bfs_opt_context.h     |  5 ----
 examples/analytical_apps/sssp/sssp_opt.h      | 28 -------------------
 grape/communication/sync_comm.h               | 24 ++++++++--------
 4 files changed, 12 insertions(+), 50 deletions(-)

diff --git a/examples/analytical_apps/bc/bc_context.h b/examples/analytical_apps/bc/bc_context.h
index 2978719b..e9b385fc 100644
--- a/examples/analytical_apps/bc/bc_context.h
+++ b/examples/analytical_apps/bc/bc_context.h
@@ -56,11 +56,6 @@ class BCContext : public VertexDataContext<FRAG_T, float> {
       LOG(INFO) << "[frag-" << frag.fid()
                 << "] BC(0) = " << centrality_value[s];
     }
-#ifdef PROFILING
-    VLOG(2) << "preprocess_time: " << preprocess_time << "s.";
-    VLOG(2) << "exec_time: " << exec_time << "s.";
-    VLOG(2) << "postprocess_time: " << postprocess_time << "s.";
-#endif
   }
 
   oid_t source_id;
diff --git a/examples/analytical_apps/bfs/bfs_opt_context.h b/examples/analytical_apps/bfs/bfs_opt_context.h
index 08fe6967..c69b7ba5 100644
--- a/examples/analytical_apps/bfs/bfs_opt_context.h
+++ b/examples/analytical_apps/bfs/bfs_opt_context.h
@@ -49,11 +49,6 @@ class BFSOptContext : public VertexDataContext<FRAG_T, int64_t> {
     for (auto v : inner_vertices) {
       os << frag.GetId(v) << " " << partial_result[v] << std::endl;
     }
-#ifdef PROFILING
-    VLOG(2) << "preprocess_time: " << preprocess_time << "s.";
-    VLOG(2) << "exec_time: " << exec_time << "s.";
-    VLOG(2) << "postprocess_time: " << postprocess_time << "s.";
-#endif
   }
 
   oid_t source_id;
diff --git a/examples/analytical_apps/sssp/sssp_opt.h b/examples/analytical_apps/sssp/sssp_opt.h
index 0203803a..679c8546 100644
--- a/examples/analytical_apps/sssp/sssp_opt.h
+++ b/examples/analytical_apps/sssp/sssp_opt.h
@@ -56,10 +56,6 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
     vertex_t source;
     bool native_source = frag.GetInnerVertex(ctx.source_id, source);
 
-#ifdef PROFILING
-    ctx.exec_time -= GetCurrentTime();
-#endif
-
     ctx.next_modified.ParallelClear(GetThreadPool());
 
     // Get the channel. Messages assigned to this channel will be sent by the
@@ -83,17 +79,9 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
       }
     }
 
-#ifdef PROFILING
-    ctx.exec_time += GetCurrentTime();
-    ctx.postprocess_time -= GetCurrentTime();
-#endif
-
     messages.ForceContinue();
 
     ctx.next_modified.Swap(ctx.curr_modified);
-#ifdef PROFILING
-    ctx.postprocess_time += GetCurrentTime();
-#endif
   }
 
   /**
@@ -109,10 +97,6 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
 
     auto& channels = messages.Channels();
 
-#ifdef PROFILING
-    ctx.preprocess_time -= GetCurrentTime();
-#endif
-
     ctx.next_modified.ParallelClear(GetThreadPool());
 
     // parallel process and reduce the received messages
@@ -124,11 +108,6 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
           }
         });
 
-#ifdef PROFILING
-    ctx.preprocess_time += GetCurrentTime();
-    ctx.exec_time -= GetCurrentTime();
-#endif
-
     // incremental evaluation.
     ForEach(ctx.curr_modified, inner_vertices,
             [&frag, &ctx](int tid, vertex_t v) {
@@ -146,10 +125,6 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
 
     // put messages into channels corresponding to the destination fragments.
 
-#ifdef PROFILING
-    ctx.exec_time += GetCurrentTime();
-    ctx.postprocess_time -= GetCurrentTime();
-#endif
     auto outer_vertices = frag.OuterVertices();
     ForEach(ctx.next_modified, outer_vertices,
             [&channels, &frag, &ctx](int tid, vertex_t v) {
@@ -164,9 +139,6 @@ class SSSPOpt : public ParallelAppBase<FRAG_T, SSSPOptContext<FRAG_T>,
     }
 
     ctx.next_modified.Swap(ctx.curr_modified);
-#ifdef PROFILING
-    ctx.postprocess_time += GetCurrentTime();
-#endif
   }
 
   void EstimateMessageSize(const fragment_t& frag, size_t& send_size,
diff --git a/grape/communication/sync_comm.h b/grape/communication/sync_comm.h
index 2417fb29..3ca50cc9 100644
--- a/grape/communication/sync_comm.h
+++ b/grape/communication/sync_comm.h
@@ -95,8 +95,8 @@ static inline void send_buffer(const T* ptr, size_t len, int dst_worker_id,
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "sending large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "sending large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Send(ptr, chunk_size_in_bytes, MPI_CHAR, dst_worker_id, tag, comm);
     ptr += chunk_num;
@@ -120,8 +120,8 @@ static inline void isend_buffer(const T* ptr, size_t len, int dst_worker_id,
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "isending large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "isending large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Request req;
     MPI_Isend(ptr, chunk_size_in_bytes, MPI_CHAR, dst_worker_id, tag, comm,
@@ -147,8 +147,8 @@ static inline void recv_buffer(T* ptr, size_t len, int src_worker_id, int tag,
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "recving large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "recving large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Recv(ptr, chunk_size_in_bytes, MPI_CHAR, src_worker_id, tag, comm,
              MPI_STATUS_IGNORE);
@@ -177,8 +177,8 @@ static inline void irecv_buffer(T* ptr, size_t len, int src_worker_id, int tag,
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "irecving large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "irecving large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Irecv(ptr, chunk_size_in_bytes, MPI_CHAR, src_worker_id, tag, comm,
               &reqs[i]);
@@ -202,8 +202,8 @@ static inline void irecv_buffer(T* ptr, size_t len, int src_worker_id, int tag,
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "irecving large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "irecving large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Request req;
     MPI_Irecv(ptr, chunk_size_in_bytes, MPI_CHAR, src_worker_id, tag, comm,
@@ -236,8 +236,8 @@ static inline void bcast_buffer(T* ptr, size_t len, int root, MPI_Comm comm) {
   const size_t chunk_size_in_bytes = chunk_num * sizeof(T);
   int iter = len / chunk_num;
   size_t remaining = (len % chunk_num) * sizeof(T);
-  LOG(INFO) << "bcast large buffer in " << iter + (remaining != 0)
-            << " iterations";
+  VLOG(10) << "bcast large buffer in " << iter + (remaining != 0)
+           << " iterations";
   for (int i = 0; i < iter; ++i) {
     MPI_Bcast(ptr, chunk_size_in_bytes, MPI_CHAR, root, comm);
     ptr += chunk_num;

From 19e0992a68517e9d8fd0d22d0002b48738f6bb38 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 19 Sep 2024 13:58:50 +0800
Subject: [PATCH 11/15] fix bucket

---
 examples/analytical_apps/run_app_vc.h         |   1 +
 grape/fragment/basic_vc_ds_fragment_loader.h  |  40 ++----
 grape/fragment/basic_vc_fragment_loader.h     |  44 ++----
 grape/fragment/immutable_vertexcut_fragment.h | 130 +++++++++++-------
 4 files changed, 99 insertions(+), 116 deletions(-)

diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index c7d1b7e1..d495be62 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -35,6 +35,7 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
     graph_spec.set_serialize(true, FLAGS_serialization_prefix);
   }
   graph_spec.single_scan = FLAGS_single_scan_load;
+  graph_spec.load_concurrency = FLAGS_load_concurrency;
 
   using FRAG_T =
       ImmutableVertexcutFragment<int64_t, grape::EmptyType, grape::EmptyType>;
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index c5d0b221..24bde1cd 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -162,6 +162,7 @@ class BasicVCDSFragmentLoader {
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
         shuffle_out_size);
 #endif
+    static constexpr size_t thread_local_cache_size = 128;
     for (int i = 0; i < load_concurrency_; ++i) {
       edge_move_threads_.emplace_back([this] {
         ShuffleBufferTuple<oid_t, oid_t, edata_t> cur;
@@ -169,7 +170,8 @@ class BasicVCDSFragmentLoader {
 #ifdef TRACKING_MEMORY_ALLOCATIONS
         // allocate memory for thread local edge cache
         MemoryTracker::GetInstance().allocate(
-            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
+            thread_local_cache_size);
 #endif
         while (got_edges_.Get(cur)) {
           size_t cur_size = cur.size();
@@ -177,7 +179,7 @@ class BasicVCDSFragmentLoader {
                                                  edata_t data) {
             int bucket_id = bucketer_->get_bucket_id(src, dst);
             edges_cache[bucket_id].emplace_back(src, dst, data);
-            if (edges_cache[bucket_id].size() >= 4096) {
+            if (edges_cache[bucket_id].size() >= thread_local_cache_size) {
               size_t cursor = bucket_cursor_[bucket_id].fetch_add(
                   edges_cache[bucket_id].size());
               std::copy(edges_cache[bucket_id].begin(),
@@ -201,7 +203,8 @@ class BasicVCDSFragmentLoader {
 #ifdef TRACKING_MEMORY_ALLOCATIONS
         // deallocate memory for thread local edge cache
         MemoryTracker::GetInstance().deallocate(
-            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * 4096);
+            (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
+            thread_local_cache_size);
 #endif
       });
     }
@@ -255,36 +258,9 @@ class BasicVCDSFragmentLoader {
     }
     bucket_edge_offset.emplace_back(en);
 
-    {
-      std::atomic<int> d2_bucket_idx(0);
-      int d2_bucket_num = bucket_num_ * bucket_num_;
-      std::vector<std::thread> sort_threads;
-      for (int i = 0; i < load_concurrency_; ++i) {
-        sort_threads.emplace_back(
-            [this, &d2_bucket_idx, d2_bucket_num,
-             &bucket_edge_offset](int tid) {
-              while (true) {
-                int idx = d2_bucket_idx.fetch_add(1);
-                if (idx >= d2_bucket_num) {
-                  break;
-                }
-                std::sort(edges_.begin() + bucket_edge_offset[idx],
-                          edges_.begin() + bucket_edge_offset[idx + 1],
-                          [](const edge_t& a, const edge_t& b) {
-                            return a.src < b.src ||
-                                   (a.src == b.src && a.dst < b.dst);
-                          });
-              }
-            },
-            i);
-      }
-      for (auto& thrd : sort_threads) {
-        thrd.join();
-      }
-    }
-
     fragment.reset(new fragment_t());
-    fragment->Init(comm_spec_, vnum_, std::move(edges_));
+    fragment->Init(comm_spec_, vnum_, std::move(edges_), bucket_num_,
+                   std::move(bucket_edge_offset));
   }
 
  private:
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
index 0bcaec3a..cfa81059 100644
--- a/grape/fragment/basic_vc_fragment_loader.h
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -169,7 +169,7 @@ class BasicVCFragmentLoader {
 #endif
 
     {
-      static constexpr size_t thread_local_cache_size = 4096;
+      static constexpr size_t thread_local_cache_size = 128;
 #ifdef TRACKING_MEMORY_ALLOCATIONS
       // allocate thread local cache
       MemoryTracker::GetInstance().allocate(
@@ -209,10 +209,13 @@ class BasicVCFragmentLoader {
               }
 
               for (size_t i = 0; i < bucket_num_ * bucket_num_; ++i) {
-                size_t cursor = bucket_edge_cursor[i].fetch_add(
-                    thread_local_edges[i].size());
-                std::copy(thread_local_edges[i].begin(),
-                          thread_local_edges[i].end(), edges.begin() + cursor);
+                if (!thread_local_edges[i].empty()) {
+                  size_t cursor = bucket_edge_cursor[i].fetch_add(
+                      thread_local_edges[i].size());
+                  std::copy(thread_local_edges[i].begin(),
+                            thread_local_edges[i].end(),
+                            edges.begin() + cursor);
+                }
               }
             },
             i);
@@ -233,36 +236,9 @@ class BasicVCFragmentLoader {
 #endif
     }
 
-    {
-      std::atomic<int> d2_bucket_idx(0);
-      int d2_bucket_num = bucket_num_ * bucket_num_;
-      std::vector<std::thread> sort_threads;
-      for (int i = 0; i < load_concurrency_; ++i) {
-        sort_threads.emplace_back(
-            [this, &d2_bucket_idx, d2_bucket_num, &edges,
-             &bucket_edge_offset](int tid) {
-              while (true) {
-                int idx = d2_bucket_idx.fetch_add(1);
-                if (idx >= d2_bucket_num) {
-                  break;
-                }
-                std::sort(edges.begin() + bucket_edge_offset[idx],
-                          edges.begin() + bucket_edge_offset[idx + 1],
-                          [](const edge_t& a, const edge_t& b) {
-                            return a.src < b.src ||
-                                   (a.src == b.src && a.dst < b.dst);
-                          });
-              }
-            },
-            i);
-      }
-      for (auto& thrd : sort_threads) {
-        thrd.join();
-      }
-    }
-
     fragment.reset(new fragment_t());
-    fragment->Init(comm_spec_, vnum_, std::move(edges));
+    fragment->Init(comm_spec_, vnum_, std::move(edges), bucket_num_,
+                   std::move(bucket_edge_offset));
   }
 
  private:
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index ffc5ad36..346c6640 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -75,7 +75,8 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
   using base_t::fnum_;
 
   void Init(const CommSpec& comm_spec, int64_t vnum,
-            std::vector<edge_t>&& edges) {
+            std::vector<edge_t>&& edges, int bucket_num = 1,
+            std::vector<size_t>&& bucket_edge_offsets = {}) {
     base_t::init(comm_spec.fid(), comm_spec.fnum(), false);
 
     partitioner_.init(comm_spec.fnum(), vnum);
@@ -109,9 +110,13 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     edges_ = std::move(edges);
 #endif
 
-    bucket_num_ = 1;
-    bucket_edge_offsets_.push_back(0);
-    bucket_edge_offsets_.push_back(edges_.size());
+    bucket_num_ = bucket_num;
+    if (bucket_num_ == 1) {
+      bucket_edge_offsets_.push_back(0);
+      bucket_edge_offsets_.push_back(edges_.size());
+    } else {
+      bucket_edge_offsets_ = std::move(bucket_edge_offsets);
+    }
   }
 
   void PrepareToRunApp(const CommSpec& comm_spec, PrepareConf conf,
@@ -119,6 +124,22 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     if (bucket_num_ < static_cast<int>(pe_spec.thread_num)) {
       buildBucket(pe_spec.thread_num);
     }
+    oid_t src_begin = src_vertices_.begin_value();
+    size_t src_size = src_vertices_.size();
+    size_t src_chunk = (src_size + pe_spec.thread_num - 1) / pe_spec.thread_num;
+    oid_t dst_begin = dst_vertices_.begin_value();
+    size_t dst_size = dst_vertices_.size();
+    size_t dst_chunk = (dst_size + pe_spec.thread_num - 1) / pe_spec.thread_num;
+    for (int src_i = 0; src_i < bucket_num_; ++src_i) {
+      for (int dst_i = 0; dst_i < bucket_num_; ++dst_i) {
+        for (auto& e : GetEdgesOfBucket(src_i, dst_i)) {
+          int src_bucket_id = (e.src - src_begin) / src_chunk;
+          int dst_bucket_id = (e.dst - dst_begin) / dst_chunk;
+          CHECK_EQ(src_bucket_id, src_i);
+          CHECK_EQ(dst_bucket_id, dst_i);
+        }
+      }
+    }
   }
 
   const vertices_t& SourceVertices() const { return src_vertices_; }
@@ -147,20 +168,18 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
 
     InArchive arc;
     arc << edges_.size();
-    arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_;
+    arc << src_vertices_ << dst_vertices_ << vertices_ << master_vertices_
+        << bucket_num_ << bucket_edge_offsets_;
     CHECK(io_adaptor->WriteArchive(arc));
     arc.Clear();
 
     partitioner_.serialize(io_adaptor);
 
     if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
-      LOG(INFO) << "is pod";
       if (!edges_.empty()) {
         io_adaptor->Write(edges_.data(), edges_.size() * sizeof(edge_t));
       }
     } else {
-      LOG(FATAL) << "is not pod";
-
       arc << edges_;
       if (!arc.Empty()) {
         CHECK(io_adaptor->WriteArchive(arc));
@@ -186,12 +205,12 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     arc >> edge_num;
     CHECK_EQ(fid_, comm_spec.fid());
     CHECK_EQ(fnum_, comm_spec.fnum());
-    arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_;
+    arc >> src_vertices_ >> dst_vertices_ >> vertices_ >> master_vertices_ >>
+        bucket_num_ >> bucket_edge_offsets_;
 
     partitioner_.deserialize(io_adaptor);
 
     if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
-      LOG(INFO) << "is pod";
       edges_.resize(edge_num);
 #ifdef TRACKING_MEMORY_ALLOCATIONS
       MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
@@ -200,15 +219,10 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
         CHECK(io_adaptor->Read(edges_.data(), edge_num * sizeof(edge_t)));
       }
     } else {
-      LOG(FATAL) << "is not pod";
       arc.Clear();
       CHECK(io_adaptor->ReadArchive(arc));
       arc >> edges_;
     }
-
-    bucket_num_ = 1;
-    bucket_edge_offsets_.push_back(0);
-    bucket_edge_offsets_.push_back(edges_.size());
   }
 
   size_t GetTotalVerticesNum() const override {
@@ -237,11 +251,6 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     if (bucket_num_ == thread_num) {
       return;
     }
-    std::vector<std::vector<size_t>> thread_bucket_edge_nums(thread_num);
-    std::vector<std::thread> threads;
-    std::atomic<size_t> edge_idx(0);
-    size_t edge_num = edges_.size();
-
     oid_t src_begin = src_vertices_.begin_value();
     size_t src_size = src_vertices_.size();
     size_t src_chunk = (src_size + thread_num - 1) / thread_num;
@@ -249,47 +258,68 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
     size_t dst_size = dst_vertices_.size();
     size_t dst_chunk = (dst_size + thread_num - 1) / thread_num;
 
-    for (int i = 0; i < thread_num; ++i) {
-      threads.emplace_back(
-          [&, this](int tid) {
-            thread_bucket_edge_nums[tid].clear();
-            thread_bucket_edge_nums[tid].resize(thread_num * thread_num, 0);
-            while (true) {
-              size_t idx = std::min(edge_idx.fetch_add(4096), edge_num);
-              size_t end = std::min(idx + 4096, edge_num);
-              if (idx == end) {
-                break;
-              }
-              while (idx < end) {
-                auto& edge = edges_[idx];
-                int src_bucket_id = (edge.src - src_begin) / src_chunk;
-                int dst_bucket_id = (edge.dst - dst_begin) / dst_chunk;
-                thread_bucket_edge_nums[tid][src_bucket_id * thread_num +
-                                             dst_bucket_id]++;
-                ++idx;
-              }
-            }
-          },
-          i);
-    }
-    for (auto& t : threads) {
-      t.join();
-    }
     std::vector<size_t> bucket_edge_num(thread_num * thread_num, 0);
-    for (int i = 0; i < thread_num; ++i) {
-      for (int j = 0; j < thread_num * thread_num; ++j) {
-        bucket_edge_num[j] += thread_bucket_edge_nums[i][j];
+    {
+      std::vector<std::vector<size_t>> thread_bucket_edge_nums(thread_num);
+      std::vector<std::thread> threads;
+      std::atomic<size_t> edge_idx(0);
+      size_t edge_num = edges_.size();
+
+      for (int i = 0; i < thread_num; ++i) {
+        threads.emplace_back(
+            [&, this](int tid) {
+              thread_bucket_edge_nums[tid].clear();
+              thread_bucket_edge_nums[tid].resize(thread_num * thread_num, 0);
+              while (true) {
+                size_t idx = std::min(edge_idx.fetch_add(4096), edge_num);
+                size_t end = std::min(idx + 4096, edge_num);
+                if (idx == end) {
+                  break;
+                }
+                while (idx < end) {
+                  auto& edge = edges_[idx];
+                  int src_bucket_id = (edge.src - src_begin) / src_chunk;
+                  int dst_bucket_id = (edge.dst - dst_begin) / dst_chunk;
+                  thread_bucket_edge_nums[tid][src_bucket_id * thread_num +
+                                               dst_bucket_id]++;
+                  ++idx;
+                }
+              }
+            },
+            i);
+      }
+      for (auto& t : threads) {
+        t.join();
+      }
+      for (int i = 0; i < thread_num; ++i) {
+        for (int j = 0; j < thread_num * thread_num; ++j) {
+          bucket_edge_num[j] += thread_bucket_edge_nums[i][j];
+        }
       }
     }
 
     bucket_num_ = thread_num;
     bucket_edge_offsets_.clear();
-    edge_num = 0;
+    size_t edge_num = 0;
     for (size_t i = 0; i < bucket_edge_num.size(); ++i) {
       bucket_edge_offsets_.push_back(edge_num);
       edge_num += bucket_edge_num[i];
     }
     bucket_edge_offsets_.push_back(edge_num);
+    std::sort(edges_.begin(), edges_.end(),
+              [&](const edge_t& a, const edge_t& b) {
+                int src_bucket_id_a = (a.src - src_begin) / src_chunk;
+                int src_bucket_id_b = (b.src - src_begin) / src_chunk;
+
+                if (src_bucket_id_a != src_bucket_id_b) {
+                  return src_bucket_id_a < src_bucket_id_b;
+                }
+
+                int dst_bucket_id_a = (a.dst - dst_begin) / dst_chunk;
+                int dst_bucket_id_b = (b.dst - dst_begin) / dst_chunk;
+
+                return dst_bucket_id_a < dst_bucket_id_b;
+              });
   }
 
   VCPartitioner<oid_t> partitioner_;

From 455192de158eb4f0cb1e62d3833aa294540ade70 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 19 Sep 2024 17:24:43 +0800
Subject: [PATCH 12/15] preallocate cache for comm

---
 .../analytical_apps/pagerank/pagerank_vc.h    |   3 +
 .../parallel/gather_scatter_message_manager.h | 117 +++++++++++++++---
 2 files changed, 101 insertions(+), 19 deletions(-)

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index 871a2ce4..3b869508 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -129,6 +129,9 @@ class PageRankVC
 
   void IncEval(const fragment_t& frag, context_t& ctx,
                message_manager_t& messages) {
+    if (ctx.step == 0) {
+      messages.AllocateGatherBuffers<fragment_t, double>(frag);
+    }
     ++ctx.step;
 
     double base = (1.0 - ctx.delta) / ctx.graph_vnum +
diff --git a/grape/parallel/gather_scatter_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
index 1ed12338..156ef3bf 100644
--- a/grape/parallel/gather_scatter_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
 #define GRAPE_PARALLEL_GATHER_SCATTER_MESSAGE_MANAGER_H_
 
+#include <queue>
+
 #include <mpi.h>
 
 #include "grape/communication/sync_comm.h"
@@ -97,6 +99,39 @@ class GatherScatterMessageManager : public MessageManagerBase {
 
   size_t GetMsgSize() const override { return sent_size_; }
 
+  template <typename FRAG_T, typename MESSAGE_T>
+  void AllocateGatherBuffers(const FRAG_T& frag) {
+    auto output_range = frag.MasterVertices();
+    size_t output_size = output_range.size() * sizeof(MESSAGE_T);
+    for (fid_t i = 1; i < fnum_; ++i) {
+      fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
+      auto src_vertices = frag.GetPartitioner().get_src_vertices(src_fid);
+      if (output_range.IsSubsetOf(src_vertices)) {
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // allocate memory for received messages
+        MemoryTracker::GetInstance().allocate(output_size);
+#endif
+        std::vector<char> buffer(output_size);
+        returnToPool(std::move(buffer));
+        continue;
+      } else {
+        CHECK(!output_range.OverlapWith(src_vertices));
+      }
+      auto dst_vertices = frag.GetPartitioner().get_dst_vertices(src_fid);
+      if (output_range.IsSubsetOf(dst_vertices)) {
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+        // allocate memory for received messages
+        MemoryTracker::GetInstance().allocate(output_size);
+#endif
+        std::vector<char> buffer(output_size);
+        returnToPool(std::move(buffer));
+        continue;
+      } else {
+        CHECK(!output_range.OverlapWith(dst_vertices));
+      }
+    }
+  }
+
   template <typename GRAPH_T, typename MESSAGE_T, typename AGGR_T>
   void GatherMasterVertices(
       const GRAPH_T& frag,
@@ -128,33 +163,47 @@ class GatherScatterMessageManager : public MessageManagerBase {
       }
     }
 
-    std::vector<std::vector<MESSAGE_T>> recv_buffers;
+    std::vector<MESSAGE_T*> recv_buffers;
     const auto& output_range = output.GetVertexRange();
     size_t output_size = output_range.size();
+    std::queue<std::vector<char>> from_pool;
+    std::queue<std::vector<char>> from_heap;
     for (fid_t i = 1; i < fnum_; ++i) {
       fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
       auto src_vertices = partitioner.get_src_vertices(src_fid);
       if (output_range.IsSubsetOf(src_vertices)) {
-        recv_buffers.emplace_back(output_size);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
-        // allocate memory for received messages
-        MemoryTracker::GetInstance().allocate(output_size * sizeof(MESSAGE_T));
-#endif
-        sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
-                                src_fid, 0, comm_, requests);
+        MESSAGE_T* recv_data = nullptr;
+        std::vector<char> buffer;
+        bool is_from_pool =
+            takeFromPool(buffer, output_size * sizeof(MESSAGE_T));
+        recv_data = reinterpret_cast<MESSAGE_T*>(buffer.data());
+        if (is_from_pool) {
+          from_pool.emplace(std::move(buffer));
+        } else {
+          from_heap.emplace(std::move(buffer));
+        }
+        recv_buffers.emplace_back(recv_data);
+        sync_comm::irecv_buffer(recv_data, output_size, src_fid, 0, comm_,
+                                requests);
         continue;
       } else {
         CHECK(!output_range.OverlapWith(src_vertices));
       }
       auto dst_vertices = partitioner.get_dst_vertices(src_fid);
       if (output_range.IsSubsetOf(dst_vertices)) {
-        recv_buffers.emplace_back(output_size);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
-        // allocate memory for received messages
-        MemoryTracker::GetInstance().allocate(output_size * sizeof(MESSAGE_T));
-#endif
-        sync_comm::irecv_buffer(recv_buffers.back().data(), output_size,
-                                src_fid, 0, comm_, requests);
+        MESSAGE_T* recv_data = nullptr;
+        std::vector<char> buffer;
+        bool is_from_pool =
+            takeFromPool(buffer, output_size * sizeof(MESSAGE_T));
+        recv_data = reinterpret_cast<MESSAGE_T*>(buffer.data());
+        if (is_from_pool) {
+          from_pool.emplace(std::move(buffer));
+        } else {
+          from_heap.emplace(std::move(buffer));
+        }
+        recv_buffers.emplace_back(recv_data);
+        sync_comm::irecv_buffer(recv_data, output_size, src_fid, 0, comm_,
+                                requests);
         continue;
       } else {
         CHECK(!output_range.OverlapWith(dst_vertices));
@@ -220,11 +269,15 @@ class GatherScatterMessageManager : public MessageManagerBase {
 
 #ifdef TRACKING_MEMORY_ALLOCATIONS
     // deallocate memory for received messages
-    for (auto& recv_buffer : recv_buffers) {
-      MemoryTracker::GetInstance().deallocate(recv_buffer.size() *
-                                              sizeof(MESSAGE_T));
+    while (!from_heap.empty()) {
+      MemoryTracker::GetInstance().deallocate(from_heap.front().size());
+      from_heap.pop();
     }
 #endif
+    while (!from_pool.empty()) {
+      returnToPool(std::move(from_pool.front()));
+      from_pool.pop();
+    }
 #ifdef PROFILING
     t1_gather_calc_ += GetCurrentTime();
 #endif
@@ -276,7 +329,9 @@ class GatherScatterMessageManager : public MessageManagerBase {
       if (master_vertices.IsSubsetOf(output_range)) {
         MESSAGE_T* output_data = &output[*master_vertices.begin()];
         size_t output_size = master_vertices.size();
-        sync_comm::recv_buffer(output_data, output_size, src_fid, 0, comm_);
+        // sync_comm::recv_buffer(output_data, output_size, src_fid, 0, comm_);
+        sync_comm::irecv_buffer(output_data, output_size, src_fid, 0, comm_,
+                                requests);
       } else {
         CHECK(!master_vertices.OverlapWith(output_range));
       }
@@ -296,6 +351,28 @@ class GatherScatterMessageManager : public MessageManagerBase {
   }
 
  private:
+  bool takeFromPool(std::vector<char>& buffer, size_t size) {
+    for (auto& pair : gather_pools_) {
+      if (pair.first >= size) {
+        if (!pair.second.empty()) {
+          buffer = std::move(pair.second.front());
+          pair.second.pop();
+          return true;
+        }
+      }
+    }
+#ifdef TRACKING_MEMORY_ALLOCATIONS
+    // allocate memory for received messages
+    MemoryTracker::GetInstance().allocate(size);
+#endif
+    buffer.resize(size);
+    return false;
+  }
+
+  void returnToPool(std::vector<char>&& buffer) {
+    gather_pools_[buffer.size()].emplace(std::move(buffer));
+  }
+
   fid_t fid_;
   fid_t fnum_;
   CommSpec comm_spec_;
@@ -309,6 +386,8 @@ class GatherScatterMessageManager : public MessageManagerBase {
   TerminateInfo terminate_info_;
   bool vote_terminate_;
 
+  std::map<size_t, std::queue<std::vector<char>>> gather_pools_;
+
 #ifdef PROFILING
   double t0_gather_comm_ = 0;
   double t1_gather_calc_ = 0;

From 87f395a35d2f35e3d9992324bc4de37743a250f4 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 19 Sep 2024 17:26:27 +0800
Subject: [PATCH 13/15] comment

---
 grape/parallel/gather_scatter_message_manager.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/grape/parallel/gather_scatter_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
index 156ef3bf..f53f0775 100644
--- a/grape/parallel/gather_scatter_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -329,7 +329,6 @@ class GatherScatterMessageManager : public MessageManagerBase {
       if (master_vertices.IsSubsetOf(output_range)) {
         MESSAGE_T* output_data = &output[*master_vertices.begin()];
         size_t output_size = master_vertices.size();
-        // sync_comm::recv_buffer(output_data, output_size, src_fid, 0, comm_);
         sync_comm::irecv_buffer(output_data, output_size, src_fid, 0, comm_,
                                 requests);
       } else {

From 9ed0fab60c42ff77d693094ff6b7c64df35e1db7 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 19 Sep 2024 17:33:53 +0800
Subject: [PATCH 14/15] tracking memory as option

---
 CMakeLists.txt                                  |  6 ++++++
 grape/fragment/basic_vc_ds_fragment_loader.h    | 16 ++++++++--------
 grape/fragment/basic_vc_fragment_loader.h       | 14 +++++++-------
 grape/fragment/immutable_vertexcut_fragment.h   |  4 ++--
 grape/fragment/vc_fragment_loader.h             |  4 ++--
 grape/parallel/gather_scatter_message_manager.h |  8 ++++----
 grape/utils/memory_tracker.cc                   |  2 +-
 grape/utils/memory_tracker.h                    |  3 +--
 8 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b213e967..15b13cfd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,6 +14,7 @@ option(USE_JEMALLOC "Whether to use jemalloc." OFF)
 option(USE_HUGEPAGES "Whether to use hugepages." OFF)
 option(BUILD_SHARED_LIBS "Whether to build libgrape-lite as shared library" ON)
 option(PROFILING "Whether to enable profiling" OFF)
+option(TRACKING_MEMORY "Whether to enable memory tracking" OFF)
 option(WITH_ASAN "Build with Address Sanitizer" OFF)
 option(BUILD_LIBGRAPELITE_DOCS "Build libgrape-lite documentation" ON)
 option(BUILD_LIBGRAPELITE_TESTS "Build libgrape-lite test cases" ON)
@@ -38,6 +39,11 @@ if (PROFILING)
   add_definitions(-DPROFILING)
 endif ()
 
+if (TRACKING_MEMORY)
+  message(STATUS "Enable memory tracking")
+  add_definitions(-DTRACKING_MEMORY)
+endif ()
+
 if (WCC_USE_GID)
   add_definitions(-DWCC_USE_GID)
 endif ()
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index 24bde1cd..382bfff1 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -140,7 +140,7 @@ class BasicVCDSFragmentLoader {
       edge_num += partition_bucket_edge_num_[comm_spec_.fid()][k];
     }
     edges_.resize(edge_num);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate memory for edges
     MemoryTracker::GetInstance().allocate((sizeof(edge_t)) * edge_num);
 #endif
@@ -156,7 +156,7 @@ class BasicVCDSFragmentLoader {
         edges_to_frag_[fid].DisableComm();
       }
     }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate memory for edge shuffle-out buffer
     MemoryTracker::GetInstance().allocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -167,7 +167,7 @@ class BasicVCDSFragmentLoader {
       edge_move_threads_.emplace_back([this] {
         ShuffleBufferTuple<oid_t, oid_t, edata_t> cur;
         std::vector<std::vector<edge_t>> edges_cache(bucket_num_ * bucket_num_);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
         // allocate memory for thread local edge cache
         MemoryTracker::GetInstance().allocate(
             (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -187,7 +187,7 @@ class BasicVCDSFragmentLoader {
               edges_cache[bucket_id].clear();
             }
           });
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
           // deallocate edge shuffle-in buffer
           MemoryTracker::GetInstance().deallocate(
               (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur_size);
@@ -200,7 +200,7 @@ class BasicVCDSFragmentLoader {
                       edges_.begin() + cursor);
           }
         }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
         // deallocate memory for thread local edge cache
         MemoryTracker::GetInstance().deallocate(
             (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -218,7 +218,7 @@ class BasicVCDSFragmentLoader {
     edges_to_frag_[fid].Emplace(src, dst, data);
     if (fid == comm_spec_.fid() &&
         edges_to_frag_[fid].buffers().size() >= shuffle_out_size) {
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // allocate memory for edge shuffle-out(to self) buffer
       MemoryTracker::GetInstance().allocate(
           (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
@@ -237,7 +237,7 @@ class BasicVCDSFragmentLoader {
     edge_recv_thread_.join();
     recv_thread_running_ = false;
     edges_to_frag_.clear();
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // deallocate memory for edge shuffle-out buffer
     MemoryTracker::GetInstance().deallocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -275,7 +275,7 @@ class BasicVCDSFragmentLoader {
         break;
       }
       CHECK_EQ(dst_fid, comm_spec_.fid());
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // allocate memory for edge shuffle-in buffer
       MemoryTracker::GetInstance().allocate(
           (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) *
diff --git a/grape/fragment/basic_vc_fragment_loader.h b/grape/fragment/basic_vc_fragment_loader.h
index cfa81059..fc04523e 100644
--- a/grape/fragment/basic_vc_fragment_loader.h
+++ b/grape/fragment/basic_vc_fragment_loader.h
@@ -50,7 +50,7 @@ class BasicVCFragmentLoader {
       }
     }
 
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate shuffle out buffers
     MemoryTracker::GetInstance().allocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -99,13 +99,13 @@ class BasicVCFragmentLoader {
       got_edges_memory_usage +=
           buf.size() * (sizeof(oid_t) * 2 + sizeof(edata_t));
     }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate edges recv buffer
     MemoryTracker::GetInstance().allocate(got_edges_memory_usage);
 #endif
 
     edges_to_frag_[comm_spec_.fid()].Clear();
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // deallocate shuffle out buffers
     MemoryTracker::GetInstance().deallocate(
         (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * comm_spec_.fnum() *
@@ -163,14 +163,14 @@ class BasicVCFragmentLoader {
 
     std::vector<edge_t> edges;
     edges.resize(edge_num);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate edges buffer
     MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edge_num);
 #endif
 
     {
       static constexpr size_t thread_local_cache_size = 128;
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // allocate thread local cache
       MemoryTracker::GetInstance().allocate(
           sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
@@ -223,14 +223,14 @@ class BasicVCFragmentLoader {
       for (auto& thrd : insert_threads) {
         thrd.join();
       }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // deallocate thread local cache
       MemoryTracker::GetInstance().deallocate(
           sizeof(edge_t) * bucket_num_ * bucket_num_ * thread_local_cache_size *
           load_concurrency_);
 #endif
       got_edges_.clear();
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // deallocate edges recv buffer
       MemoryTracker::GetInstance().deallocate(got_edges_memory_usage);
 #endif
diff --git a/grape/fragment/immutable_vertexcut_fragment.h b/grape/fragment/immutable_vertexcut_fragment.h
index 346c6640..010f7265 100644
--- a/grape/fragment/immutable_vertexcut_fragment.h
+++ b/grape/fragment/immutable_vertexcut_fragment.h
@@ -100,7 +100,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
 
 #ifdef USE_EDGE_ARRAY
     edges_.resize(edges.size());
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // reallocate memory for edges
     MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
     MemoryTracker::GetInstance().deallocate(sizeof(edge_t) * edges_.size());
@@ -212,7 +212,7 @@ class ImmutableVertexcutFragment<int64_t, EmptyType, EmptyType>
 
     if (std::is_pod<edata_t>::value && std::is_pod<oid_t>::value) {
       edges_.resize(edge_num);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       MemoryTracker::GetInstance().allocate(sizeof(edge_t) * edges_.size());
 #endif
       if (edge_num > 0) {
diff --git a/grape/fragment/vc_fragment_loader.h b/grape/fragment/vc_fragment_loader.h
index 17da0ec8..a1149e65 100644
--- a/grape/fragment/vc_fragment_loader.h
+++ b/grape/fragment/vc_fragment_loader.h
@@ -119,7 +119,7 @@ class VCFragmentLoader {
     if (comm_spec_.worker_id() == 0) {
       VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
     }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
             << MemoryTracker::GetInstance().GetMemoryUsageInfo();
 #endif
@@ -223,7 +223,7 @@ class VCFragmentLoader {
     if (comm_spec_.worker_id() == 0) {
       VLOG(1) << "finished constructing fragment, time: " << t2 << " s";
     }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     VLOG(1) << "[frag-" << comm_spec_.fid() << "] after constructing fragment: "
             << MemoryTracker::GetInstance().GetMemoryUsageInfo();
 #endif
diff --git a/grape/parallel/gather_scatter_message_manager.h b/grape/parallel/gather_scatter_message_manager.h
index f53f0775..2ad7e968 100644
--- a/grape/parallel/gather_scatter_message_manager.h
+++ b/grape/parallel/gather_scatter_message_manager.h
@@ -107,7 +107,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
       fid_t src_fid = (fid_ + fnum_ - i) % fnum_;
       auto src_vertices = frag.GetPartitioner().get_src_vertices(src_fid);
       if (output_range.IsSubsetOf(src_vertices)) {
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
         // allocate memory for received messages
         MemoryTracker::GetInstance().allocate(output_size);
 #endif
@@ -119,7 +119,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
       }
       auto dst_vertices = frag.GetPartitioner().get_dst_vertices(src_fid);
       if (output_range.IsSubsetOf(dst_vertices)) {
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
         // allocate memory for received messages
         MemoryTracker::GetInstance().allocate(output_size);
 #endif
@@ -267,7 +267,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
       thrd.join();
     }
 
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // deallocate memory for received messages
     while (!from_heap.empty()) {
       MemoryTracker::GetInstance().deallocate(from_heap.front().size());
@@ -360,7 +360,7 @@ class GatherScatterMessageManager : public MessageManagerBase {
         }
       }
     }
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     // allocate memory for received messages
     MemoryTracker::GetInstance().allocate(size);
 #endif
diff --git a/grape/utils/memory_tracker.cc b/grape/utils/memory_tracker.cc
index bacb3ed3..fc444c6b 100644
--- a/grape/utils/memory_tracker.cc
+++ b/grape/utils/memory_tracker.cc
@@ -17,7 +17,7 @@ limitations under the License.
 
 namespace grape {
 
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
 
 MemoryTracker& MemoryTracker::GetInstance() {
   static MemoryTracker instance;
diff --git a/grape/utils/memory_tracker.h b/grape/utils/memory_tracker.h
index 311a1e9d..7b5dfc93 100644
--- a/grape/utils/memory_tracker.h
+++ b/grape/utils/memory_tracker.h
@@ -20,8 +20,7 @@ limitations under the License.
 
 namespace grape {
 
-#define TRACKING_MEMORY_ALLOCATIONS
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
 
 struct MemoryTracker {
   static MemoryTracker& GetInstance();

From f900f430c30b12b3ae781f4d54d4b15340a5b563 Mon Sep 17 00:00:00 2001
From: luoxiaojian <lxj193371@alibaba-inc.com>
Date: Thu, 19 Sep 2024 17:42:27 +0800
Subject: [PATCH 15/15] fix

---
 examples/analytical_apps/pagerank/pagerank_vc.h         | 4 ++--
 examples/analytical_apps/pagerank/pagerank_vc_context.h | 2 +-
 examples/analytical_apps/run_app_vc.h                   | 4 ++--
 grape/fragment/basic_vc_ds_fragment_loader.h            | 3 +--
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/examples/analytical_apps/pagerank/pagerank_vc.h b/examples/analytical_apps/pagerank/pagerank_vc.h
index 3b869508..77b136a2 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc.h
@@ -55,7 +55,7 @@ class PageRankVC
 #endif
       typename fragment_t::template both_vertex_array_t<int> degree(
           frag.Vertices(), 0);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // allocate degree array for both src and dst vertices
       MemoryTracker::GetInstance().allocate(frag.Vertices().size() *
                                             sizeof(int));
@@ -91,7 +91,7 @@ class PageRankVC
 
       messages.GatherMasterVertices<fragment_t, int, NumericSum<int>>(
           frag, degree, ctx.master_degree);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
       // deallocate degree array for both src and dst vertices
       MemoryTracker::GetInstance().deallocate(frag.Vertices().size() *
                                               sizeof(int));
diff --git a/examples/analytical_apps/pagerank/pagerank_vc_context.h b/examples/analytical_apps/pagerank/pagerank_vc_context.h
index 1dd7f001..00f3dd96 100644
--- a/examples/analytical_apps/pagerank/pagerank_vc_context.h
+++ b/examples/analytical_apps/pagerank/pagerank_vc_context.h
@@ -34,7 +34,7 @@ class PageRankVCContext : public VertexDataContext<FRAG_T, double> {
     next_result.Init(fragment.Vertices());
     master_degree.Init(fragment.MasterVertices());
 
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
     MemoryTracker::GetInstance().allocate(fragment.Vertices().size() *
                                           sizeof(double));
     MemoryTracker::GetInstance().allocate(fragment.Vertices().size() *
diff --git a/examples/analytical_apps/run_app_vc.h b/examples/analytical_apps/run_app_vc.h
index d495be62..6a9a2b5b 100644
--- a/examples/analytical_apps/run_app_vc.h
+++ b/examples/analytical_apps/run_app_vc.h
@@ -41,7 +41,7 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
       ImmutableVertexcutFragment<int64_t, grape::EmptyType, grape::EmptyType>;
   std::shared_ptr<FRAG_T> fragment = LoadVertexcutGraph<FRAG_T>(
       FLAGS_efile, FLAGS_vertex_num, comm_spec, graph_spec);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
   VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after loading graph: "
           << MemoryTracker::GetInstance().GetMemoryUsageInfo();
 #endif
@@ -49,7 +49,7 @@ void CreateAndQueryVC(const CommSpec& comm_spec, const std::string& out_prefix,
   auto app = std::make_shared<AppType>();
   DoQuery<FRAG_T, AppType, Args...>(fragment, app, comm_spec, spec, out_prefix,
                                     args...);
-#ifdef TRACKING_MEMORY_ALLOCATIONS
+#ifdef TRACKING_MEMORY
   VLOG(1) << "[worker-" << comm_spec.worker_id() << "] after query: "
           << MemoryTracker::GetInstance().GetMemoryUsageInfo();
 #endif
diff --git a/grape/fragment/basic_vc_ds_fragment_loader.h b/grape/fragment/basic_vc_ds_fragment_loader.h
index 382bfff1..16a8d615 100644
--- a/grape/fragment/basic_vc_ds_fragment_loader.h
+++ b/grape/fragment/basic_vc_ds_fragment_loader.h
@@ -174,7 +174,6 @@ class BasicVCDSFragmentLoader {
             thread_local_cache_size);
 #endif
         while (got_edges_.Get(cur)) {
-          size_t cur_size = cur.size();
           foreach_rval(cur, [&edges_cache, this](oid_t src, oid_t dst,
                                                  edata_t data) {
             int bucket_id = bucketer_->get_bucket_id(src, dst);
@@ -190,7 +189,7 @@ class BasicVCDSFragmentLoader {
 #ifdef TRACKING_MEMORY
           // deallocate edge shuffle-in buffer
           MemoryTracker::GetInstance().deallocate(
-              (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur_size);
+              (sizeof(oid_t) + sizeof(oid_t) + sizeof(edata_t)) * cur.size());
 #endif
         }
         for (int i = 0; i < bucket_num_ * bucket_num_; ++i) {