From af1daabfdfad7d0844b8df10bd3fdd12830bd6b9 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 21 Oct 2024 15:37:26 +0200 Subject: [PATCH 1/5] Parse: Correct keyword/ident parsing --- src/lang/passes/parse.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lang/passes/parse.cc b/src/lang/passes/parse.cc index 3cc5090..4986db7 100644 --- a/src/lang/passes/parse.cc +++ b/src/lang/passes/parse.cc @@ -92,25 +92,25 @@ trieste::Parse parser() // Line comment "(?:#[^\\n\\r]*)" >> [](auto&) {}, - "def" >> [](auto& m) { m.seq(Func); }, + "def\\b" >> [](auto& m) { m.seq(Func); }, "\\(" >> [](auto& m) { m.push(Parens); }, "\\)" >> [](auto& m) { m.term({List, Parens}); m.extend(Parens); }, - "return" >> [](auto& m) { m.seq(Return); }, + "return\\b" >> [](auto& m) { m.seq(Return); }, - "for" >> [](auto& m) { m.seq(For); }, - "in" >> + "for\\b" >> [](auto& m) { m.seq(For); }, + "in\\b" >> [](auto& m) { // In should always be in a list from the identifiers. m.term({List}); }, "," >> [](auto& m) { m.seq(List); }, - "if" >> [](auto& m) { m.seq(If); }, - "else" >> [](auto& m) { m.seq(Else); }, + "if\\b" >> [](auto& m) { m.seq(If); }, + "else\\b" >> [](auto& m) { m.seq(Else); }, ":" >> [indent](auto& m) { // Exit conditionals expressions. @@ -151,11 +151,11 @@ trieste::Parse parser() m.push(Block); }, - "drop" >> [](auto& m) { m.add(Drop); }, - "create" >> [](auto& m) { m.add(Create); }, - "freeze" >> [](auto& m) { m.add(Freeze); }, - "region" >> [](auto& m) { m.add(Region); }, - "None" >> [](auto& m) { m.add(Null); }, + "drop\\b" >> [](auto& m) { m.add(Drop); }, + "create\\b" >> [](auto& m) { m.add(Create); }, + "freeze\\b" >> [](auto& m) { m.add(Freeze); }, + "region\\b" >> [](auto& m) { m.add(Region); }, + "None\\b" >> [](auto& m) { m.add(Null); }, "[0-9A-Za-z_]+" >> [](auto& m) { m.add(Ident); }, "\\[" >> [](auto& m) { m.push(Lookup); }, "\\]" >> [](auto& m) { m.term({Lookup}); }, From 0eb4d98df8c3b35c5e032cbb9ecb8a5ff903f292 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 22 Oct 2024 10:50:06 +0200 Subject: [PATCH 2/5] RT: Add null checks --- src/rt/rt.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/rt/rt.cc b/src/rt/rt.cc index 6c46386..8d1265a 100644 --- a/src/rt/rt.cc +++ b/src/rt/rt.cc @@ -85,10 +85,14 @@ namespace rt objects::DynObject* set_prototype(objects::DynObject* obj, objects::DynObject* proto) { - if (proto->is_primitive() != nullptr) + if (proto && proto->is_primitive() != nullptr) { ui::error("Cannot set a primitive as a prototype."); } + if (obj == nullptr) + { + ui::error("Cannot set a prototype on null."); + } if (obj->is_primitive() != nullptr) { ui::error("Cannot set a prototype on a primitive object."); @@ -183,7 +187,7 @@ namespace rt objects::DynObject* iter_next(objects::DynObject* iter) { assert(!iter->is_immutable()); - if (iter->get_prototype() != core::keyIterPrototypeObject()) + if (iter && iter->get_prototype() != core::keyIterPrototypeObject()) { ui::error("Object is not an iterator."); } @@ -193,7 +197,7 @@ namespace rt verona::interpreter::Bytecode* get_bytecode(objects::DynObject* func) { - if (func->get_prototype() == core::bytecodeFuncPrototypeObject()) + if (func && func->get_prototype() == core::bytecodeFuncPrototypeObject()) { return reinterpret_cast(func)->get_bytecode(); } From c256cc85eaea704fccb0de2920a452becdaf1a8f Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 22 Oct 2024 10:50:35 +0200 Subject: [PATCH 3/5] Interpreter: Support recursive calls It turns out python scoping is a bit weird. Top level functions are added to the `globals()` table. Function calls to self or a parent function which is not on the top level are passed as local variables. Here is an example: ```python print("s00 " + str(globals())) print("s00 " + str(locals())) s00 = 0 def scope_01(con): s01 = 1 print("s01" + str(globals())) print("s01 " + str(locals())) def scope_02(con): s02 = 2 print("s02" + str(globals())) print("s02 " + str(locals())) def scope_03(): s03 = 3 print("s03" + str(globals())) print("s03 " + str(locals())) scope_01(False) scope_02(False) if con: scope_03() if con: scope_02(True) scope_01(True) ``` This commit adds a hack that simulates the global namespace. This at least allows simple recursion. --- src/lang/bytecode.h | 8 +++++++ src/lang/interpreter.cc | 24 ++++++++++++++++++++ src/lang/lang.h | 14 +++++++----- src/lang/passes/bytecode.cc | 8 ++++--- src/lang/passes/flatten.cc | 2 +- src/lang/passes/grouping.cc | 16 +++++++++----- src/lang/passes/parse.cc | 18 ++++++++++----- tests/recursive_list.vpy | 44 +++++++++++++++++++++++++++++++++++++ 8 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 tests/recursive_list.vpy diff --git a/src/lang/bytecode.h b/src/lang/bytecode.h index 4faa24f..14c4a63 100644 --- a/src/lang/bytecode.h +++ b/src/lang/bytecode.h @@ -2,8 +2,16 @@ #include "trieste/ast.h" +/// Loads a value with a given name from the current frame. +/// +/// The value name is stored in the location of the node inline const trieste::TokenDef LoadFrame{"load_frame", trieste::flag::print}; inline const trieste::TokenDef StoreFrame{"store_frame", trieste::flag::print}; +/// Loads a value with a given name from the current scope or the global +/// namespace. This should be used for function resolution. +/// +/// The value name is stored in the location of the node +inline const trieste::TokenDef LoadGlobal{"load_global", trieste::flag::print}; inline const trieste::TokenDef LoadField{"load_field"}; inline const trieste::TokenDef StoreField{"store_field"}; inline const trieste::TokenDef CreateObject{"create_object"}; diff --git a/src/lang/interpreter.cc b/src/lang/interpreter.cc index 894cbd4..20fa825 100644 --- a/src/lang/interpreter.cc +++ b/src/lang/interpreter.cc @@ -104,6 +104,10 @@ namespace verona::interpreter { return frame_stack.back()->frame; } + rt::objects::DynObject* global_frame() + { + return frame_stack.front()->frame; + } rt::objects::DynObject* pop(char const* data_info) { @@ -201,6 +205,26 @@ namespace verona::interpreter return ExecNext{}; } + if (node == LoadGlobal) + { + std::string field{node->location().view()}; + + auto v = rt::get(frame(), field); + if (v) + { + rt::add_reference(frame(), v); + } + else + { + v = rt::get(global_frame(), field); + rt::add_reference(global_frame(), v); + } + + stack().push_back(v); + std::cout << "push " << v << std::endl; + return ExecNext{}; + } + if (node == StoreFrame) { auto v = pop("value to store"); diff --git a/src/lang/lang.h b/src/lang/lang.h index 8d6cee6..a685c5c 100644 --- a/src/lang/lang.h +++ b/src/lang/lang.h @@ -36,6 +36,7 @@ namespace verona::wf inline const auto cmp_values = Ident | Lookup | Null; inline const auto key = Ident | Lookup | String; inline const auto operand = Lookup | Call | Method | Ident; + inline const auto Cond = Eq | Neq; inline const auto grouping = (Top <<= File) | (File <<= Body) | (Body <<= Block) | @@ -44,19 +45,21 @@ namespace verona::wf Method)++) | (Assign <<= (Lhs >>= lv) * (Rhs >>= rv)) | (Lookup <<= (Op >>= operand) * (Rhs >>= key)) | (Region <<= Ident) | - (Freeze <<= Ident) | (Create <<= Ident) | (If <<= Eq * Block * Block) | + (Freeze <<= Ident) | (Create <<= Ident) | + (If <<= (Op >>= Cond) * Block * Block) | (For <<= (Key >>= Ident) * (Value >>= Ident) * (Op >>= lv) * Block) | (Eq <<= (Lhs >>= cmp_values) * (Rhs >>= cmp_values)) | + (Neq <<= (Lhs >>= cmp_values) * (Rhs >>= cmp_values)) | (Func <<= Ident * Params * Body) | (Call <<= Ident * List) | (Method <<= Lookup * List) | (ReturnValue <<= rv) | (List <<= rv++) | (Params <<= Ident++); inline const trieste::wf::Wellformed bytecode = (Top <<= Body) | (Body <<= - (LoadFrame | StoreFrame | LoadField | StoreField | Drop | Null | - CreateObject | CreateRegion | FreezeObject | IterNext | Print | Eq | Neq | - Jump | JumpFalse | Label | Call | Return | ReturnValue | ClearStack | - Dup)++) | + (LoadFrame | LoadGlobal | StoreFrame | LoadField | StoreField | Drop | + Null | CreateObject | CreateRegion | FreezeObject | IterNext | Print | + Eq | Neq | Jump | JumpFalse | Label | Call | Return | ReturnValue | + ClearStack | Dup)++) | (CreateObject <<= (Dictionary | String | KeyIter | Proto | Func)) | (Func <<= Body) | (Label <<= Ident)[Ident]; } @@ -67,6 +70,7 @@ inline const auto RV = inline const auto CMP_V = T(Ident, Lookup, Null); inline const auto KEY = T(Ident, Lookup, String); inline const auto OPERAND = T(Lookup, Call, Method, Ident); +inline const auto COND = T(Eq, Neq); // Parsing && AST construction Parse parser(); diff --git a/src/lang/passes/bytecode.cc b/src/lang/passes/bytecode.cc index 0c3dbbc..ba54fee 100644 --- a/src/lang/passes/bytecode.cc +++ b/src/lang/passes/bytecode.cc @@ -36,7 +36,7 @@ PassDef bytecode() ClearStack)[Op] >> [](auto& _) -> Node { return _(Op); }, - T(Compile) << (T(Eq, Neq)[Op] << (Any[Lhs] * Any[Rhs])) >> + T(Compile) << (COND[Op] << (Any[Lhs] * Any[Rhs])) >> [](auto& _) { return Seq << (Compile << _(Lhs)) << (Compile << _(Rhs)) << _(Op)->type(); @@ -104,10 +104,12 @@ PassDef bytecode() << (Call ^ std::to_string(arg_ctn + 1)); }, T(Compile) - << (T(Call)[Call] << (KEY[Op] * (T(List) << Any++[List]) * End)) >> + << (T(Call)[Call] + << (T(Ident)[Ident] * (T(List) << Any++[List]) * End)) >> [](auto& _) { // The print is done by the called function - return Seq << (Compile << _[List]) << (Compile << _(Op)) + return Seq << (Compile << _[List]) + << create_from(LoadGlobal, _(Ident)) << (Call ^ std::to_string(_[List].size())); }, diff --git a/src/lang/passes/flatten.cc b/src/lang/passes/flatten.cc index 3fcbf4a..60a2a33 100644 --- a/src/lang/passes/flatten.cc +++ b/src/lang/passes/flatten.cc @@ -52,7 +52,7 @@ PassDef flatten() In(Body) * (T(Block) << Any++[Block]) >> [](auto& _) { return Seq << _[Block]; }, T(If)[If] - << (T(Eq)[Op] * (T(Block) << Any++[Lhs]) * + << (COND[Op] * (T(Block) << Any++[Lhs]) * (T(Block) << Any++[Rhs])) >> [](auto& _) { auto else_label = new_jump_label(); diff --git a/src/lang/passes/grouping.cc b/src/lang/passes/grouping.cc index 2218863..abd5ca8 100644 --- a/src/lang/passes/grouping.cc +++ b/src/lang/passes/grouping.cc @@ -76,18 +76,20 @@ PassDef grouping() << ((T(Group) << LV[Lhs] * End) * ((T(Group) << (RV[Rhs] * End)) / (RV[Rhs] * End)) * End) >> [](auto& _) { return Assign << _[Lhs] << _[Rhs]; }, - T(Eq) + COND[Op] << ((T(Group) << CMP_V[Lhs] * End) * (T(Group) << CMP_V[Rhs] * End) * End) >> - [](auto& _) { return Eq << _[Lhs] << _[Rhs]; }, + [](auto& _) { + return create_from(_(Op)->type(), _(Op)) << _[Lhs] << _[Rhs]; + }, - (T(If) << (T(Group) * T(Eq)[Eq] * (T(Group) << T(Block)[Block]))) >> - [](auto& _) { return If << _(Eq) << _(Block); }, + (T(If) << (T(Group) * COND[Op] * (T(Group) << T(Block)[Block]))) >> + [](auto& _) { return If << _(Op) << _(Block); }, (T(Else) << (T(Group) * (T(Group) << T(Block)[Block]))) >> [](auto& _) { return Else << _(Block); }, - (T(If)[If] << (T(Eq) * T(Block) * End)) * (T(Else) << T(Block)[Block]) >> + (T(If)[If] << (COND * T(Block) * End)) * (T(Else) << T(Block)[Block]) >> [](auto& _) { return _(If) << _(Block); }, - (T(If)[If] << (T(Eq) * T(Block) * End)) * (--T(Else)) >> + (T(If)[If] << (COND * T(Block) * End)) * (--T(Else)) >> [](auto& _) { // This adds an empty else block, if no else was written return _(If) << Block; @@ -132,6 +134,8 @@ PassDef grouping() T(Return)[Return] << ((T(Group) << End) * (T(Group) << (RV[Rhs] * End)) * End) >> [](auto& _) { return create_from(ReturnValue, _(Return)) << _(Rhs); }, + T(Return)[Return] << ((T(Group) << End) * (RV[Rhs] * End)) >> + [](auto& _) { return create_from(ReturnValue, _(Return)) << _(Rhs); }, }}; diff --git a/src/lang/passes/parse.cc b/src/lang/passes/parse.cc index 4986db7..df3cb1f 100644 --- a/src/lang/passes/parse.cc +++ b/src/lang/passes/parse.cc @@ -10,12 +10,13 @@ namespace verona::wf Group | Assign | If | Else | Block | For | Func | List | Return; inline const auto parser = (Top <<= File) | (File <<= parse_groups++) | - (Assign <<= Group++) | (If <<= Group * Eq * Group) | + (Assign <<= Group++) | (If <<= Group * (Op >>= Cond) * Group) | (Else <<= Group * Group) | (Group <<= (parse_tokens | Block | List)++) | (Block <<= (parse_tokens | parse_groups)++) | (Eq <<= Group * Group) | - (Lookup <<= Group) | (For <<= Group * List * Group * Group) | - (List <<= Group++) | (Parens <<= (Group | List)++) | - (Func <<= Group * Group * Group) | (Return <<= Group++); + (Neq <<= Group * Group) | (Lookup <<= Group) | + (For <<= Group * List * Group * Group) | (List <<= Group++) | + (Parens <<= (Group | List)++) | (Func <<= Group * Group * Group) | + (Return <<= Group++); } struct Indent @@ -109,12 +110,16 @@ trieste::Parse parser() }, "," >> [](auto& m) { m.seq(List); }, - "if\\b" >> [](auto& m) { m.seq(If); }, + "if\\b" >> + [](auto& m) { + m.term(); + m.seq(If); + }, "else\\b" >> [](auto& m) { m.seq(Else); }, ":" >> [indent](auto& m) { // Exit conditionals expressions. - m.term({Eq}); + m.term({Eq, Neq}); Token toc = Empty; if (m.in(If)) @@ -167,6 +172,7 @@ trieste::Parse parser() }, "\"([^\\n\"]+)\"" >> [](auto& m) { m.add(String, 1); }, "==" >> [](auto& m) { m.seq(Eq); }, + "!=" >> [](auto& m) { m.seq(Neq); }, "=" >> [](auto& m) { m.seq(Assign); }, "{}" >> [](auto& m) { m.add(Empty); }, }); diff --git a/tests/recursive_list.vpy b/tests/recursive_list.vpy new file mode 100644 index 0000000..11f4f28 --- /dev/null +++ b/tests/recursive_list.vpy @@ -0,0 +1,44 @@ +# For now separate +def insert(node, item): + if node.next == None: + node.next = {} + node.next.data = item + node.next.prev = node + else: + insert(node.next, item) + +def contains(node, item): + if node.data == item: + return True + if node.next == None: + return False + return find(node.next, item) + +def remove(node, item): + if node.data == item: + node.prev.next = node.next + if node.next != None: + node.next.prev = node.prev + return True + if node.next == None: + return False + return remove(node.next, item) + +def new_list(): + list = {} + list.head = {} + return list + +list = new_list() + +# Required to not leak memory +region list + +value = "x" +insert(list.head, {}) +insert(list.head, {}) +insert(list.head, value) +insert(list.head, {}) +remove(list.head, value) + +drop list From 6339dc58235b355fe12e2c1f0c7886393abd25a2 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 22 Oct 2024 14:47:35 +0200 Subject: [PATCH 4/5] Interpreter: Improve reference counting --- src/lang/interpreter.cc | 18 +++++++++++++++++- tests/recursive_list.vpy | 8 ++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/lang/interpreter.cc b/src/lang/interpreter.cc index 20fa825..0c32940 100644 --- a/src/lang/interpreter.cc +++ b/src/lang/interpreter.cc @@ -199,6 +199,18 @@ namespace verona::interpreter { std::string field{node->location().view()}; auto v = rt::get(frame(), field); + if (!v) + { + if (field == "True") + { + v = rt::get_true(); + } + else if (field == "False") + { + rt::get_false(); + } + } + rt::add_reference(frame(), v); stack().push_back(v); std::cout << "push " << v << std::endl; @@ -308,6 +320,7 @@ namespace verona::interpreter result = rt::get_false(); result_str = "false"; } + rt::add_reference(frame(), result); stack().push_back(result); std::cout << "push " << result << " (" << result_str << ")" << std::endl; @@ -326,6 +339,7 @@ namespace verona::interpreter { auto v = pop("jump condition"); auto jump = (v == rt::get_false()); + rt::remove_reference(frame(), v); if (jump) { return ExecJump{node->location()}; @@ -442,7 +456,9 @@ namespace verona::interpreter // Setup the new frame for (size_t i = 0; i < func.arg_ctn; i++) { - frame->stack.push_back(parent_frame->stack.back()); + auto value = parent_frame->stack.back(); + frame->stack.push_back(value); + rt::move_reference(parent_frame->frame, frame->frame, value); parent_frame->stack.pop_back(); } } diff --git a/tests/recursive_list.vpy b/tests/recursive_list.vpy index 11f4f28..8b9b03a 100644 --- a/tests/recursive_list.vpy +++ b/tests/recursive_list.vpy @@ -35,10 +35,18 @@ list = new_list() region list value = "x" + +# Dyrona doesn't freeze shared objects automatically (yet) +# There is currently a bug, where the LRC gets incorrect, if "[String]" isn't frozen +proto = value["__proto__"] +freeze proto +drop proto + insert(list.head, {}) insert(list.head, {}) insert(list.head, value) insert(list.head, {}) remove(list.head, value) +drop value drop list From 0f2fba1cf7785a035afcfcc8b51c8be67b31ea0c Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 22 Oct 2024 15:07:20 +0200 Subject: [PATCH 5/5] Prototype: Correct LRC tracking for prototypes --- src/lang/passes/flatten.cc | 3 +-- src/rt/objects/dyn_object.h | 5 ++++- tests/recursive_list.vpy | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lang/passes/flatten.cc b/src/lang/passes/flatten.cc index 60a2a33..50210e1 100644 --- a/src/lang/passes/flatten.cc +++ b/src/lang/passes/flatten.cc @@ -52,8 +52,7 @@ PassDef flatten() In(Body) * (T(Block) << Any++[Block]) >> [](auto& _) { return Seq << _[Block]; }, T(If)[If] - << (COND[Op] * (T(Block) << Any++[Lhs]) * - (T(Block) << Any++[Rhs])) >> + << (COND[Op] * (T(Block) << Any++[Lhs]) * (T(Block) << Any++[Rhs])) >> [](auto& _) { auto else_label = new_jump_label(); auto join_label = new_jump_label(); diff --git a/src/rt/objects/dyn_object.h b/src/rt/objects/dyn_object.h index ac6e2b5..fed8b6c 100644 --- a/src/rt/objects/dyn_object.h +++ b/src/rt/objects/dyn_object.h @@ -183,7 +183,10 @@ namespace rt::objects } if (prototype != nullptr) - prototype->change_rc(1); + { + // prototype->change_rc(1); + add_reference(this, prototype); + } std::cout << "Allocate: " << this << std::endl; } diff --git a/tests/recursive_list.vpy b/tests/recursive_list.vpy index 8b9b03a..becc02d 100644 --- a/tests/recursive_list.vpy +++ b/tests/recursive_list.vpy @@ -37,7 +37,6 @@ region list value = "x" # Dyrona doesn't freeze shared objects automatically (yet) -# There is currently a bug, where the LRC gets incorrect, if "[String]" isn't frozen proto = value["__proto__"] freeze proto drop proto