[Tolk] Source maps implementation

crypto/fift/lib/Asm.fif

@@ -260,6 +260,11 @@ x{7F} @Defop TRUE
 x{83FF} @Defop PUSHNAN
 { <b x{84} s, swap 1- 8 u, @addopb } : PUSHPOW2DEC
 { <b x{85} s, swap 1- 8 u, @addopb } : PUSHNEGPOW2
+
+// special instruction for debugging purposes
+// doesn't exists in TVM
+{ <b x{F955} s, swap 16 u, @addopb } : DEBUGMARK
+
 //
 // other constants
 x{88} @Defop(ref) PUSHREF

crypto/fift/utils.cpp

@@ -227,7 +227,7 @@ td::Result<td::Ref<vm::Cell>> compile_asm(td::Slice asm_code) {
 
 td::Result<CompiledProgramOutput> compile_asm_program(std::string&& program_code, const std::string& fift_dir) {
   std::string main_fif;
-  main_fif.reserve(program_code.size() + 100);
+  main_fif.reserve(program_code.size() + 200);
   main_fif.append(program_code.data(), program_code.size());
   main_fif.append(R"( dup hashB B>X      $>B "hex" B>file)");   // write codeHashHex to a file
   main_fif.append(R"(     boc>B B>base64 $>B "boc" B>file)");   // write codeBoc64 to a file

tolk/CMakeLists.txt

@@ -31,6 +31,7 @@ set(TOLK_SOURCE
         pipe-ast-to-legacy.cpp
         pipe-find-unused-symbols.cpp
         pipe-generate-fif-output.cpp
+        pipe-generate-source-map.cpp
         type-system.cpp
         smart-casts-cfg.cpp
         generics-helpers.cpp
@@ -43,6 +44,7 @@ set(TOLK_SOURCE
         stack-transform.cpp
         optimize.cpp
         codegen.cpp
+        debug-info.cpp
         tolk.cpp
 )
 

tolk/abscode.cpp

@@ -254,6 +254,10 @@ void Op::show(std::ostream& os, const std::vector<TmpVar>& vars, std::string pfx
       show_var_list(os, left, vars);
       os << " := " << str_const << std::endl;
       break;
+    case _DebugInfo:
+      os << pfx << dis << "DEBUGINFO ";
+      os << source_map_entry_idx << std::endl;
+      break;
     case _Import:
       os << pfx << dis << "IMPORT ";
       show_var_list(os, left, vars);
@@ -394,41 +398,41 @@ void CodeBlob::print(std::ostream& os, int flags) const {
   os << "-------- END ---------\n\n";
 }
 
-std::vector<var_idx_t> CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name) {
+std::vector<var_idx_t> CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name, TypePtr parent_type) {
   std::vector<var_idx_t> ir_idx;
   int stack_w = var_type->get_width_on_stack();
   ir_idx.reserve(stack_w);
   if (const TypeDataStruct* t_struct = var_type->try_as<TypeDataStruct>()) {
     for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) {
       StructFieldPtr field_ref = t_struct->struct_ref->get_field(i);
       std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name;
-      std::vector<var_idx_t> nested = create_var(field_ref->declared_type, loc, std::move(sub_name));
+      std::vector<var_idx_t> nested = create_var(field_ref->declared_type, loc, std::move(sub_name), parent_type);
       ir_idx.insert(ir_idx.end(), nested.begin(), nested.end());
     }
   } else if (const TypeDataTensor* t_tensor = var_type->try_as<TypeDataTensor>()) {
     for (int i = 0; i < t_tensor->size(); ++i) {
       std::string sub_name = name.empty() ? name : name + "." + std::to_string(i);
-      std::vector<var_idx_t> nested = create_var(t_tensor->items[i], loc, std::move(sub_name));
+      std::vector<var_idx_t> nested = create_var(t_tensor->items[i], loc, std::move(sub_name), parent_type);
       ir_idx.insert(ir_idx.end(), nested.begin(), nested.end());
     }
   } else if (const TypeDataAlias* t_alias = var_type->try_as<TypeDataAlias>()) {
-    ir_idx = create_var(t_alias->underlying_type, loc, std::move(name));
+    ir_idx = create_var(t_alias->underlying_type, loc, std::move(name), parent_type);
   } else if (const TypeDataUnion* t_union = var_type->try_as<TypeDataUnion>(); t_union && stack_w != 1) {
     std::string utag_name = name.empty() ? "'UTag" : name + ".UTag";
     if (t_union->or_null) {   // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag"
-      ir_idx = create_var(t_union->or_null, loc, std::move(name));
+      ir_idx = create_var(t_union->or_null, loc, std::move(name), parent_type);
     } else {                  // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag"
       for (int i = 0; i < stack_w - 1; ++i) {
         std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1);
-        ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]);
+        ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name), var_type)[0]);
       }
     }
-    ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]);
+    ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name), parent_type)[0]);
   } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) {
 #ifdef TOLK_DEBUG
     tolk_assert(stack_w == 1);
 #endif
-    vars.emplace_back(var_cnt, var_type, std::move(name), loc);
+    vars.emplace_back(var_cnt, var_type, std::move(name), loc, parent_type);
     ir_idx.emplace_back(var_cnt);
     var_cnt++;
   }

tolk/analyzer.cpp

@@ -303,7 +303,7 @@ bool Op::std_compute_used_vars(bool disabled) {
 bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
   tolk_assert(next);
   const VarDescrList& next_var_info = next->var_info;
-  if (cl == _Nop) {
+  if (cl == _Nop || cl == _DebugInfo) {
     return set_var_info_except(next_var_info, left);
   }
   switch (cl) {
@@ -528,6 +528,7 @@ bool prune_unreachable(std::unique_ptr<Op>& ops) {
     case Op::_UnTuple:
     case Op::_Import:
     case Op::_Let:
+    case Op::_DebugInfo:
       reach = true;
       break;
     case Op::_Return:
@@ -693,6 +694,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
   switch (cl) {
     case _Nop:
     case _Import:
+    case _DebugInfo:
       break;
     case _Return:
       values.set_unreachable();
@@ -887,6 +889,7 @@ bool Op::mark_noreturn() {
     case _SetGlob:
     case _GlobVar:
     case _CallInd:
+    case _DebugInfo:
       return set_noreturn(next->mark_noreturn());
     case _Return:
       return set_noreturn();

tolk/asmops.cpp

@@ -330,6 +330,21 @@ void AsmOpList::show_var_ext(std::ostream& os, std::pair<var_idx_t, const_idx_t>
   }
 }
 
+std::optional<std::tuple<TmpVar, std::string>> AsmOpList::get_var(const std::pair<var_idx_t, const_idx_t>& idx_pair) const {
+  const var_idx_t var_idx = idx_pair.first;
+  const const_idx_t const_idx = idx_pair.second;
+  if (!var_names_ || static_cast<size_t>(var_idx) >= var_names_->size()) {
+    return std::nullopt;
+  }
+  const auto var = var_names_->at(var_idx);
+  if (static_cast<size_t>(const_idx) < constants_.size() && constants_[const_idx].not_null()) {
+    const auto value = constants_[const_idx];
+    const auto value_str = value->to_dec_string();
+    return std::tie(var, value_str);
+  }
+  return std::tie(var, "");
+}
+
 void AsmOpList::out(std::ostream& os, int mode) const {
   std::size_t n = list_.size();
   for (std::size_t i = 0; i < n; i++) {

tolk/ast-stringifier.h

@@ -16,8 +16,6 @@
 */
 #pragma once
 
-#ifdef TOLK_DEBUG
-
 #include "ast.h"
 #include "ast-visitor.h"
 #include "type-system.h"
@@ -379,5 +377,3 @@ class ASTStringifier final : public ASTVisitor {
 };
 
 } // namespace tolk
-
-#endif // TOLK_DEBUG

tolk/codegen.cpp

@@ -276,6 +276,29 @@ void Stack::rearrange_top(SrcLocation loc, var_idx_t top, bool last) {
 }
 
 bool Op::generate_code_step(Stack& stack) {
+  // we need to handle it here to correctly handle case `IFJMP { DROP }`
+  if (cl == _DebugInfo) {
+    std::ostringstream ops;
+    ops << source_map_entry_idx << " DEBUGMARK"; // pseudo instruction
+
+    // Append opcode to a list
+    if (const auto list_size = stack.o.list_.size(); list_size > 0) {
+      stack.o.insert(stack.o.list_.size(), loc, ops.str());
+    }
+
+    if (source_map_entry_idx < G.source_map.size()) {
+      auto& entry = G.source_map.at(source_map_entry_idx);
+
+      // Collect all available variables at this point
+      for (const auto index : stack.s) {
+        if (const auto var = stack.o.get_var(index); var.has_value()) {
+          const auto& [data, value] = *var;
+          entry.vars.push_back({data, value});
+        }
+      }
+    }
+  }
+
   stack.opt_show();
 
   // detect `throw 123` (actually _IntConst 123 + _Call __throw)
@@ -882,6 +905,10 @@ bool Op::generate_code_step(Stack& stack) {
       stack.o << AsmOp::Custom(loc, "TRY");
       return true;
     }
+    case _DebugInfo: {
+      // already handled above
+      return true;
+    }
     default:
       std::cerr << "fatal: unknown operation <??" << cl << ">\n";
       throw ParseError(loc, "unknown operation in generate_code()");

tolk/compiler-state.h

@@ -22,6 +22,7 @@
 #include <functional>
 #include <set>
 #include <string>
+#include <tolk.h>
 
 namespace tolk {
 
@@ -53,9 +54,11 @@ struct CompilerSettings {
   int optimization_level = 2;
   bool stack_layout_comments = true;
   bool tolk_src_as_line_comments = true;
+  bool collect_source_map = false;
 
   std::string output_filename;
   std::string boc_output_filename;
+  std::string source_map_output_filename;
   std::string stdlib_folder;    // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback
 
   FsReadCallback read_callback;
@@ -106,6 +109,8 @@ struct CompilerState {
   std::vector<EnumDefPtr> all_enums;
   AllRegisteredSrcFiles all_src_files;
 
+  std::vector<SourceMapEntry> source_map;
+
   bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; }
 };
 

tolk/debug-info.cpp

@@ -0,0 +1,69 @@
+#include "tolk.h"
+#include <ast.h>
+#include <compiler-state.h>
+#include "ast-stringifier.h"
+
+namespace tolk {
+
+void insert_debug_info(SrcLocation loc, ASTNodeKind kind, CodeBlob& code, bool is_leave, std::string descr) {
+  if (!G.settings.collect_source_map) {
+    return;
+  }
+
+  if (kind == ast_artificial_aux_vertex || kind == ast_throw_statement) {
+    return;
+  }
+
+#ifdef TOLK_DEBUG
+  const auto last_op = std::find_if(code._vector_of_ops.rbegin(), code._vector_of_ops.rend(), [](const auto& it) {
+    return it->cl != Op::_DebugInfo;
+  });
+  const Op* last_op_ptr = last_op != code._vector_of_ops.rend() ? *last_op : nullptr;
+#endif
+
+  auto& op = code.emplace_back(loc, Op::_DebugInfo);
+  op.source_map_entry_idx = G.source_map.size();
+
+  auto info = SourceMapEntry{};
+  info.idx = op.source_map_entry_idx;
+  info.descr = descr;
+  info.is_entry = kind == ast_function_declaration;
+  info.is_leave = is_leave;
+
+#ifdef TOLK_DEBUG
+  if (last_op_ptr) {
+    std::stringstream st;
+    last_op_ptr->show(st, code.vars, "", 4);
+
+    info.opcode = st.str();
+  }
+#endif
+
+  info.ast_kind = ASTStringifier::ast_node_kind_to_string(kind);
+
+  if (const SrcFile* src_file = loc.get_src_file(); src_file != nullptr) {
+    const auto& pos = src_file->convert_offset(loc.get_char_offset());
+
+    info.loc.file = src_file->realpath;
+    info.loc.offset = loc.get_char_offset();
+    info.loc.line = pos.line_no;
+    info.loc.line_offset = is_leave;
+    info.loc.col = pos.char_no - 1;
+    info.loc.length = 1; // Once we have the actual length of node, we should use it here
+  }
+
+  info.func_name = code.fun_ref->name;
+  if (code.name != info.func_name) {
+    // If a function was inlined, `code.name` will contain the name of the function we are inlining into
+    info.inlined_to_func_name = code.name;
+  }
+  info.func_inline_mode = code.fun_ref->inline_mode;
+
+  G.source_map.push_back(info);
+}
+
+void insert_debug_info(AnyV v, CodeBlob& code) {
+  insert_debug_info(v->loc, v->kind, code, 0, "");
+}
+
+}

tolk/pack-unpack-api.cpp

@@ -127,7 +127,7 @@ class PackUnpackAvailabilityChecker {
       }
       return {};
     }
-    
+
     if (const auto* t_union = any_type->try_as<TypeDataUnion>()) {
       // a union can almost always be serialized if every of its variants can:
       // - `T?` is TL/B `(Maybe T)`
@@ -253,6 +253,8 @@ std::vector<var_idx_t> generate_T_toCell(FunctionPtr called_f, CodeBlob& code, S
   FunctionPtr f_beginCell = lookup_function("beginCell");
   FunctionPtr f_endCell = lookup_function("builder.endCell");
   std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b");
+
+  insert_debug_info(loc, ast_function_call, code);
   code.emplace_back(loc, Op::_Call, rvect_builder, std::vector<var_idx_t>{}, f_beginCell);
 
   PackContext ctx(code, loc, rvect_builder, args[1]);
@@ -267,6 +269,7 @@ std::vector<var_idx_t> generate_T_toCell(FunctionPtr called_f, CodeBlob& code, S
 // fun builder.storeAny<T>(mutate self, v: T, options: PackOptions = {}): self
 std::vector<var_idx_t> generate_builder_storeAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector<std::vector<var_idx_t>>& args) {
   TypePtr typeT = called_f->substitutedTs->typeT_at(0);
+  insert_debug_info(loc, ast_function_call, code);
   PackContext ctx(code, loc, args[0], args[2]);   // mutate this builder
   ctx.generate_pack_any(typeT, std::vector(args[1]));
 
@@ -275,6 +278,8 @@ std::vector<var_idx_t> generate_builder_storeAny(FunctionPtr called_f, CodeBlob&
 
 // fun T.fromSlice(rawSlice: slice, options: UnpackOptions): T
 std::vector<var_idx_t> generate_T_fromSlice(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector<std::vector<var_idx_t>>& args) {
+  insert_debug_info(loc, ast_function_call, code);
+
   std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s");
   code.emplace_back(loc, Op::_Let, slice_copy, args[0]);
 
@@ -305,6 +310,8 @@ std::vector<var_idx_t> generate_slice_loadAny(FunctionPtr called_f, CodeBlob& co
 // fun T.fromCell(packedCell: cell, options: UnpackOptions): T
 // fun Cell<T>.load(self, options: UnpackOptions): T
 std::vector<var_idx_t> generate_T_fromCell(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector<std::vector<var_idx_t>>& args) {
+  insert_debug_info(loc, ast_function_call, code);
+
   TypePtr typeT = called_f->substitutedTs->typeT_at(0);
   FunctionPtr f_beginParse = lookup_function("cell.beginParse");
   std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s");
@@ -324,6 +331,7 @@ std::vector<var_idx_t> generate_T_fromCell(FunctionPtr called_f, CodeBlob& code,
 // fun slice.skipAny<T>(mutate self, options: UnpackOptions): self
 std::vector<var_idx_t> generate_slice_skipAny(FunctionPtr called_f, CodeBlob& code, SrcLocation loc, const std::vector<std::vector<var_idx_t>>& args) {
   TypePtr typeT = called_f->substitutedTs->typeT_at(0);
+  insert_debug_info(loc, ast_function_call, code);
   UnpackContext ctx(code, loc, args[0], args[1]);    // mutate this slice
   ctx.generate_skip_any(typeT);
 
@@ -392,6 +400,8 @@ std::vector<var_idx_t> generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation
   StructPtr original_struct = loaded_state->original_struct;
   StructPtr hidden_struct = loaded_state->hidden_struct;
 
+  insert_debug_info(loc, ast_function_call, code);
+
   std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b");
   code.emplace_back(loc, Op::_Call, rvect_builder, std::vector<var_idx_t>{}, lookup_function("beginCell"));