feat: add external functions (written in host language) into stdlib

lib/02_parsing/Ast.ml

@@ -72,6 +72,10 @@ and expression_desc =
   | Bool of bool
   | Array of expression array
   | Record of ([ `Required | `Optional ] * expression) StringMap.t
+  | ExternalFunction of {
+      parameters : string list;
+      name : string;
+    }
   | Function of {
       parameters : string list;
       body : expression;

lib/02_parsing/Lexer.ml

@@ -684,6 +684,29 @@ let rec scan_block_comment t =
   found
 ;;
 
+let scan_external_function_symbol t =
+  let start_pos = make_position t in
+  eat2 t;
+  let rec loop buf t =
+    match t.current with
+    | `Chr '%' when peek t = `Chr '%' ->
+        eat2 t;
+        Buffer.contents buf
+    | `EOF ->
+        Diagnostics.error
+          (Location.make ~s:start_pos ())
+          "This external function symbol is not terminated. Please add a `%%%%` at the \
+           end."
+    | `Chr c ->
+        eat t;
+        Buffer.add_char buf c;
+        loop buf t
+  in
+  let buf = Buffer.create 512 in
+  let found = loop buf t in
+  found
+;;
+
 let rec scan_template_token ~start_pos t =
   match t.current with
   | `Chr '{' ->
@@ -916,9 +939,12 @@ and scan_normal_token ~start_pos t =
       | _ ->
           eat t;
           Token.PLUS)
-  | `Chr '%' ->
-      eat t;
-      Token.PERCENT
+  | `Chr '%' -> (
+      match peek t with
+      | `Chr '%' -> Token.EXTERNAL_FUNCTION_SYMBOL (scan_external_function_symbol t)
+      | _ ->
+          eat t;
+          Token.PERCENT)
   | `Chr '?' ->
       eat t;
       Token.QUESTIONMARK

lib/02_parsing/Pinc_Parser.ml

@@ -570,7 +570,7 @@ module Rules = struct
               Ast.ForInExpression { index; iterator; reverse; iterable = expr1; body }
               |> Option.some)
       (* PARSING FN EXPRESSION *)
-      | Token.KEYWORD_FN ->
+      | Token.KEYWORD_FN -> (
           let fn_loc = t.token.location in
           next t;
           let open_paren = t |> optional Token.LEFT_PAREN in
@@ -587,11 +587,17 @@ module Rules = struct
               | None ->
                   Diagnostics.error
                     fn_loc
-                    "Expected this function to have exactly one parameter")
+                    "Expected this function to have exactly one parameter,\n\
+                     or a `()` if no parameter is needed.")
           in
           expect Token.ARROW t;
-          let* body = t |> parse_block in
-          Ast.Function { parameters; body } |> Option.some
+          match t.token.typ with
+          | Token.EXTERNAL_FUNCTION_SYMBOL name ->
+              next t;
+              Option.some @@ Ast.ExternalFunction { parameters; name }
+          | _ ->
+              let* body = t |> parse_block in
+              Option.some @@ Ast.Function { parameters; body })
       (* PARSING IF EXPRESSION *)
       | Token.KEYWORD_IF ->
           next t;

lib/02_parsing/Token.ml

@@ -1,6 +1,7 @@
 module Location = Pinc_Diagnostics.Location
 
 type token_type =
+  | EXTERNAL_FUNCTION_SYMBOL of string
   | COMMENT of string
   | IDENT_LOWER of string
   | IDENT_UPPER of string
@@ -159,6 +160,7 @@ let to_string = function
   | HTML_OR_COMPONENT_TAG_SELF_CLOSING -> "/>"
   | HTML_OR_COMPONENT_TAG_END -> "> (TAG END)"
   | END_OF_INPUT -> "(EOF)"
+  | EXTERNAL_FUNCTION_SYMBOL s -> Printf.sprintf "%%%%%s%%%%" s
 ;;
 
 let is_keyword = function
@@ -178,6 +180,7 @@ let is_keyword = function
   | KEYWORD_LIBRARY
   | KEYWORD_PAGE
   | KEYWORD_STORE -> true
+  | EXTERNAL_FUNCTION_SYMBOL _
   | COMMENT _
   | LEFT_PAREN
   | RIGHT_PAREN

lib/02_parsing/Token.mli

@@ -1,6 +1,7 @@
 open Pinc_Diagnostics
 
 type token_type =
+  | EXTERNAL_FUNCTION_SYMBOL of string
   | COMMENT of string
   | IDENT_LOWER of string
   | IDENT_UPPER of string

lib/pinc_backend/Externals.ml

@@ -0,0 +1,53 @@
+module PincArray = struct
+  let length ~arguments state =
+    let array =
+      arguments |> Helpers.Expect.(required (attribute "array" (array any_value)))
+    in
+
+    let result = Array.length array in
+
+    let output = Helpers.Value.int result in
+    State.add_output state ~output
+  ;;
+end
+
+module PincString = struct
+  let length ~arguments state =
+    let str = arguments |> Helpers.Expect.(required (attribute "string" string)) in
+
+    let result =
+      str |> Containers.Utf8_string.of_string_exn |> Containers.Utf8_string.n_chars
+    in
+
+    let output = Helpers.Value.int result in
+    State.add_output state ~output
+  ;;
+
+  let sub ~arguments state =
+    let str = arguments |> Helpers.Expect.(required (attribute "string" string)) in
+    let offset = arguments |> Helpers.Expect.(required (attribute "ofs" int)) in
+    let length = arguments |> Helpers.Expect.(required (attribute "len" int)) in
+
+    let result =
+      str
+      |> Containers.Utf8_string.of_string_exn
+      |> Containers.Utf8_string.to_list
+      |> Containers.List.drop offset
+      |> Containers.List.take length
+      |> Containers.Utf8_string.of_list
+      |> Containers.Utf8_string.to_string
+    in
+
+    let output = Helpers.Value.string result in
+    State.add_output state ~output
+  ;;
+end
+
+let all =
+  StringMap.of_list
+    [
+      ("pinc_array_length", PincArray.length);
+      ("pinc_string_length", PincString.length);
+      ("pinc_string_sub", PincString.sub);
+    ]
+;;

lib/pinc_backend/Interpreter.ml

@@ -149,6 +149,12 @@ and eval_expression ~state expression =
       in
       state |> State.add_output ~output
   | Ast.String template -> eval_string_template ~state template
+  | Ast.ExternalFunction { parameters; name } ->
+      eval_external_function_declaration
+        ~loc:expression.expression_loc
+        ~state
+        ~parameters
+        name
   | Ast.Function { parameters; body } ->
       eval_function_declaration ~loc:expression.expression_loc ~state ~parameters body
   | Ast.FunctionCall { function_definition; arguments } ->
@@ -306,6 +312,32 @@ and eval_string_template ~state template =
          |> String.concat ""
          |> Helpers.Value.string ~loc:(Location.merge ~s:!start_loc ~e:!end_loc ()))
 
+and eval_external_function_declaration ~state ~loc ~parameters name =
+  let ident = state.binding_identifier in
+  let external_function =
+    match Externals.all |> StringMap.find_opt name with
+    | Some fn -> fn
+    | None ->
+        Pinc_Diagnostics.error
+          loc
+          (Printf.sprintf "The external function with name `%s` was not found." name)
+  in
+  let rec exec ~arguments ~state () =
+    let state = external_function ~arguments state in
+    state |> State.get_output
+  and fn = { value_loc = loc; value_desc = Function { parameters; state; exec } } in
+
+  ident
+  |> Option.iter (fun (_, ident) ->
+         state
+         |> State.add_value_to_function_scopes
+              ~ident
+              ~value:fn
+              ~is_optional:false
+              ~is_mutable:false);
+
+  state |> State.add_output ~output:fn
+
 and eval_function_declaration ~state ~loc ~parameters body =
   let ident = state.binding_identifier in
   let rec exec ~arguments ~state () =

lib/pinc_backend/stdlib/Base__Array.pi

@@ -9,13 +9,7 @@ library Base__Array {
     }
   };
 
-  let length = fn (array) -> {
-    let mutable len = 0;
-    for (_ in array) {
-      len := len + 1;
-    };
-    len
-  };
+  let length = fn (array) -> %%pinc_array_length%%;
 
   let first = fn (array) -> {
     array[0]

lib/pinc_backend/stdlib/Base__String.pi

@@ -1,12 +1,5 @@
 library Base__String {
-  let length = fn (string) -> {
-    let mutable len = 0;
-    for (_ in string) {
-      len := len + 1;
-    };
-
-    len
-  };
+  let length = fn (string) -> %%pinc_string_length%%;
 
   let map = fn (string, f) -> {
     let mutable result = "";
@@ -121,20 +114,7 @@ library Base__String {
     }
   };
 
-  let sub = fn (string, ofs, len) -> {
-    if (ofs < 0 || len < 0) {
-      ""
-    } else {
-      let mutable result = "";
-      for (index, char in string) {
-        if(index < ofs) continue;
-        if(index >= ofs + len) continue;
-
-        result := result ++ char;
-      };
-      result
-    }
-  };
+  let sub = fn (string, ofs, len) -> %%pinc_string_sub%%;
 
   let split = fn (string, sep) -> {
     let mutable result = [];