chore: pr review changes

src/lsp/cobol_parser/grammar_tokens_printer.ml

@@ -77,8 +77,10 @@ let pp_token: token Pretty.printer = fun ppf ->
     | FIXEDLIT (i, sep, d) -> print "FIXED[%s%c%s]" i sep d
     | FLOATLIT (i, sep, d, e) -> print "FLOAT[%s%c%sE%s]" i sep d e
     | INTERVENING_ c -> print "<%c>" c
-    | tok when Text_keywords.is_intrinsic_token tok ->
+    | INTRINSIC_FUNC _ as tok ->
         print "INTRINSIC_FUNC[%a]" pp_token_string tok
+    | tok when Text_keywords.is_known_intrinsic_token tok ->
+        print "SPECIALIZED_INTRINSIC_FUNC[%a]" pp_token_string tok
     | EOF -> string "EOF"
     | t -> pp_token_string ppf t
 

src/lsp/cobol_parser/keywords/gen_keywords.ml

@@ -13,6 +13,7 @@
 
 let cmlyname = ref None
 let external_tokens = ref ""
+let with_is_intrinsic = ref false
 
 let usage_msg = Fmt.str "%s [OPTIONS] file.cmly" Sys.argv.(0)
 let anon str = match !cmlyname with
@@ -24,6 +25,8 @@ let () =
     Arg.[
       ("--external-tokens", Set_string external_tokens,
        "<module> Import token type definition from <module>");
+      ("--with-is-intrinsic", Set with_is_intrinsic,
+       "<module> Import token type definition from <module>");
     ]
     anon usage_msg
 
@@ -175,16 +178,15 @@ let emit_intrinsic_functions_list ppf =
   Fmt.pf ppf "@]@\n]@."
 
 let emit_is_intrinsic ppf =
-  if String.equal cmlyname "grammar.cmly" then
+  if !with_is_intrinsic then
   let is_intrinsic t =
     intrinsic (Terminal.attributes t) |> Option.is_some
   in
-  Fmt.pf ppf "@[<2>let is_intrinsic_token = %s.(function@." tokens_module;
+  Fmt.pf ppf "@[<2>let is_known_intrinsic_token = %s.(function@." tokens_module;
   Terminal.iter begin fun t ->
     if is_intrinsic t
     then Fmt.pf ppf "| %a@." pp_terminal t
   end;
-  Fmt.pf ppf "| INTRINSIC_FUNC _ -> true@.";
   Fmt.pf ppf "| _ -> false@]\n)@."
 
 

src/lsp/cobol_parser/package.toml

@@ -102,7 +102,7 @@ dune-trailer = """
   (action
     (with-stdout-to %{targets}
        (run %{exe:./keywords/gen_keywords.exe} %{deps}
-            --external-tokens Grammar_tokens))))
+            --external-tokens Grammar_tokens --with-is-intrinsic))))
 
 (rule
   (targets grammar_expect.ml)

src/lsp/cobol_parser/text_keywords.mli

@@ -29,4 +29,4 @@ val silenced_keywords: string list
 (** Mapping from punctuations to their respective tokens *)
 val puncts: (string * Grammar_tokens.token) list
 
-val is_intrinsic_token: Grammar_tokens.token -> bool
+val is_known_intrinsic_token: Grammar_tokens.token -> bool

src/lsp/cobol_parser/text_tokenizer.ml

@@ -510,13 +510,17 @@ let reword_intrinsics s : tokens -> tokens =
      [Disabled_intrinsics] does not occur on a `FUNCTION` keyword (but that's
      unlikely). *)
   let keyword_of_token = Hashtbl.find Text_lexer.word_of_token in
+  let is_intrinsic_token = function
+    | INTRINSIC_FUNC _ -> true
+    | t when Text_keywords.is_known_intrinsic_token t -> true
+    | _ -> false in
   let rec aux rev_prefix suffix =
     match suffix with
     | [] ->
         List.rev rev_prefix
     | ({ payload = k1_token; _ } as k1) ::
       ({ payload = k2_token; _ } as k2) :: tl
-      when k1_token <> FUNCTION && Text_keywords.is_intrinsic_token k2_token ->
+      when k1_token <> FUNCTION && is_intrinsic_token k2_token ->
         aux (EzList.tail_map distinguish_words @@
              retokenize s (keyword_of_token k2_token &@<- k2) @
              k1 :: rev_prefix) tl