From 13e17f6c8ec10e376980ea7470a048bf3e27ab7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20N=2EC=2E=20van=20=C2=B4t=20Hooft?= Date: Wed, 19 Jul 2023 14:00:47 -0300 Subject: [PATCH] Add erlang-module support The code is deemed a module definition if the first line start with "-module(". In this case the entire code-block is interpreted as erlang-module and if there are no errors the module is compiled and loaded. Basic error handling has been added Basic tests have been added --- lib/livebook/runtime/evaluator.ex | 114 +++++++++++++++++++++++ test/livebook/runtime/evaluator_test.exs | 47 ++++++++++ 2 files changed, 161 insertions(+) diff --git a/lib/livebook/runtime/evaluator.ex b/lib/livebook/runtime/evaluator.ex index 7043eb5b702..e538c25a1a3 100644 --- a/lib/livebook/runtime/evaluator.ex +++ b/lib/livebook/runtime/evaluator.ex @@ -674,7 +674,109 @@ defmodule Livebook.Runtime.Evaluator do {result, code_markers} end + # main entry point to decide between erlang-statements vs module defp eval(:erlang, code, binding, env) do + is_module = String.starts_with?(code, "-module(") + + case is_module do + true -> eval_module(:erlang, code, binding, env) + false -> eval_statements(:erlang, code, binding, env) + end + end + + # Simple Erlang Module - helper functions + # ------------------------------------------------------------------------ + # In order to handle the expression in their forms - separate per {:dot,_} + defp not_dot({:dot, _}) do + false + end + + defp not_dot(_) do + true + end + + # A list of scanned token - must be seperated per dot, in order to feed them + # into the :erl_parse.parse_form function. + defp tokens_to_forms(tokens) do + {:ok, tokens_to_forms(tokens, [])} + end + + defp tokens_to_forms([], acc) do + :lists.reverse(acc) + end + + defp tokens_to_forms(tokens, acc) do + form = :lists.takewhile(¬_dot/1, tokens) + [dot | rest] = :lists.dropwhile(¬_dot/1, tokens) + tokens_to_forms(rest, [{form ++ [dot]}] ++ acc) + end + + defp parse_forms(form_statements) do + try do + res = + Enum.map( + form_statements, + fn {form_statement} -> + case :erl_parse.parse_form(form_statement) do + {:ok, form} -> form + err -> throw({:parse_fail, err}) + end + end + ) + + {:ok, res} + catch + {:parse_fail, err} -> err + end + end + + # Create module - tokens from string + # Based on: https://stackoverflow.com/questions/2160660/how-to-compile-erlang-code-loaded-into-a-string + # The function will first assume that code starting with -module( is a erlang module definition + + # Step 1: Scan the code + # Step 2: Convert to forms + # Step 3: Extract module name + # Step 4: Compile and load + # Step 5: If compile success - calculate md5 and register module + + defp eval_module(:erlang, code, binding, env) do + try do + with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(code), {1, 1}, [:text]), + {:ok, form_statements} <- tokens_to_forms(tokens), + {:ok, forms} <- parse_forms(form_statements) do + # First statement - form = module definition + {:attribute, _, :module, module_name} = hd(forms) + + # Compile the forms from the code-block + {:ok, _, binary_module} = :compile.forms(forms) + {:module, new_module} = :code.load_binary(module_name, ~c"nofile", binary_module) + + # Registration of module + md5 = apply(new_module, :module_info, [:md5]) + # IO.inspect(%{:md5 => md5, :module => new_module}) + + # Add the newly defined erlang module + env = Map.put(env, :versioned_erlang_modules, %{new_module => md5}) + + {{:ok, ~c"erlang module successfully compiled", binding, env}, []} + else + # Tokenizer error - https://www.erlang.org/doc/man/erl_scan.html#string-3 + {:error, {location, module, description}, _end_loc} -> + process_erlang_error(env, code, location, module, description) + + # Parser error - https://www.erlang.org/doc/man/erl_parse.html#parse_form-1 + {:error, {location, module, description}} -> + process_erlang_error(env, code, location, module, description) + end + catch + kind, error -> + stacktrace = prune_stacktrace(:erl_eval, __STACKTRACE__) + {{:error, kind, error, stacktrace}, []} + end + end + + defp eval_statements(:erlang, code, binding, env) do try do erl_binding = Enum.reduce(binding, %{}, fn {name, value}, erl_binding -> @@ -869,6 +971,18 @@ defmodule Livebook.Runtime.Evaluator do do: {{:module, module}, version}, into: identifiers_defined + # Erlang modules defined ? If so register - with md5 + identifiers_defined = + case Map.has_key?(context.env, :versioned_erlang_modules) do + true -> + for {module, version} <- context.env.versioned_erlang_modules, + do: {{:module, module}, version}, + into: identifiers_defined + + false -> + identifiers_defined + end + # Aliases identifiers_used = diff --git a/test/livebook/runtime/evaluator_test.exs b/test/livebook/runtime/evaluator_test.exs index 3d0ac4ec04b..7b7e4fd643c 100644 --- a/test/livebook/runtime/evaluator_test.exs +++ b/test/livebook/runtime/evaluator_test.exs @@ -1227,6 +1227,53 @@ defmodule Livebook.Runtime.EvaluatorTest do assert_receive {:runtime_evaluation_response, :code_1, {:text, "6"}, metadata()} end + test "evaluate erlang-module code", %{evaluator: evaluator} do + Evaluator.evaluate_code( + evaluator, + :erlang, + "-module(tryme). -export([go/0]). go() ->{ok,went}.", + :code_4, + [] + ) + + assert_receive {:runtime_evaluation_response, :code_4, + {:text, "\"erlang module successfully compiled\""}, metadata()} + end + + test "evaluate erlang-module error function already defined", %{evaluator: evaluator} do + Evaluator.evaluate_code( + evaluator, + :erlang, + "-module(tryme). -export([go/0]). go() ->{ok,went}. go() ->{ok,went}.", + :code_4, + [] + ) + + assert_receive {:runtime_evaluation_output, :code_4, + {:stdout, ":1:52: function go/0 already defined\n"}} + end + + test "evaluate erlang-module error - expression after module", %{evaluator: evaluator} do + Evaluator.evaluate_code( + evaluator, + :erlang, + "-module(tryme). -export([go/0]). go() ->{ok,went}. go() ->{ok,went}. A = 1.", + :code_4, + [] + ) + + assert_receive { + :runtime_evaluation_response, + :code_4, + { + :error, + _ErrorText, + :other + }, + %{code_markers: _List} + } + end + test "mixed erlang/elixir bindings", %{evaluator: evaluator} do Evaluator.evaluate_code(evaluator, :elixir, "x = 1", :code_1, []) Evaluator.evaluate_code(evaluator, :erlang, "Y = X.", :code_2, [:code_1])