diff --git a/.drom b/.drom index 95b8daaa..f2fc42e0 100644 --- a/.drom +++ b/.drom @@ -5,7 +5,7 @@ version:0.9.0 # hash of toml configuration files # used for generation of all files -8c51879620df5ee78e682121fa0f6c23:. +718cb8b13f0858c7f365dd1cc0efd9bd:. # end context for . # begin context for .github/workflows/workflow.yml @@ -346,7 +346,7 @@ de6c46a271140f4f52b2580e0d876351:src/lsp/cobol_lsp/version.mlt # begin context for src/lsp/cobol_parser/dune # file src/lsp/cobol_parser/dune -44b9b76fe2777a06cbe5a7eb212f7b16:src/lsp/cobol_parser/dune +0bdd09727ecc330598da0a5c0582a262:src/lsp/cobol_parser/dune # end context for src/lsp/cobol_parser/dune # begin context for src/lsp/cobol_parser/version.mlt diff --git a/src/lsp/cobol_parser/dune b/src/lsp/cobol_parser/dune index 4d6744a0..d5a5d7f5 100644 --- a/src/lsp/cobol_parser/dune +++ b/src/lsp/cobol_parser/dune @@ -43,7 +43,7 @@ (action (with-stdout-to %{targets} (run %{exe:./keywords/gen_keywords.exe} %{deps} - --external-tokens Grammar_tokens)))) + --external-tokens Grammar_tokens --with-is-intrinsic)))) (rule (targets grammar_expect.ml) diff --git a/src/lsp/cobol_parser/grammar_tokens_printer.ml b/src/lsp/cobol_parser/grammar_tokens_printer.ml index d557b8a6..ffeb61de 100644 --- a/src/lsp/cobol_parser/grammar_tokens_printer.ml +++ b/src/lsp/cobol_parser/grammar_tokens_printer.ml @@ -77,8 +77,10 @@ let pp_token: token Pretty.printer = fun ppf -> | FIXEDLIT (i, sep, d) -> print "FIXED[%s%c%s]" i sep d | FLOATLIT (i, sep, d, e) -> print "FLOAT[%s%c%sE%s]" i sep d e | INTERVENING_ c -> print "<%c>" c - | tok when Text_keywords.is_intrinsic_token tok -> + | INTRINSIC_FUNC _ as tok -> print "INTRINSIC_FUNC[%a]" pp_token_string tok + | tok when Text_keywords.is_known_intrinsic_token tok -> + print "SPECIALIZED_INTRINSIC_FUNC[%a]" pp_token_string tok | EOF -> string "EOF" | t -> pp_token_string ppf t diff --git a/src/lsp/cobol_parser/keywords/gen_keywords.ml b/src/lsp/cobol_parser/keywords/gen_keywords.ml index 561efe33..c8820bac 100644 --- a/src/lsp/cobol_parser/keywords/gen_keywords.ml +++ b/src/lsp/cobol_parser/keywords/gen_keywords.ml @@ -13,6 +13,7 @@ let cmlyname = ref None let external_tokens = ref "" +let with_is_intrinsic = ref false let usage_msg = Fmt.str "%s [OPTIONS] file.cmly" Sys.argv.(0) let anon str = match !cmlyname with @@ -24,6 +25,8 @@ let () = Arg.[ ("--external-tokens", Set_string external_tokens, " Import token type definition from "); + ("--with-is-intrinsic", Set with_is_intrinsic, + " Import token type definition from "); ] anon usage_msg @@ -175,16 +178,15 @@ let emit_intrinsic_functions_list ppf = Fmt.pf ppf "@]@\n]@." let emit_is_intrinsic ppf = - if String.equal cmlyname "grammar.cmly" then + if !with_is_intrinsic then let is_intrinsic t = intrinsic (Terminal.attributes t) |> Option.is_some in - Fmt.pf ppf "@[<2>let is_intrinsic_token = %s.(function@." tokens_module; + Fmt.pf ppf "@[<2>let is_known_intrinsic_token = %s.(function@." tokens_module; Terminal.iter begin fun t -> if is_intrinsic t then Fmt.pf ppf "| %a@." pp_terminal t end; - Fmt.pf ppf "| INTRINSIC_FUNC _ -> true@."; Fmt.pf ppf "| _ -> false@]\n)@." diff --git a/src/lsp/cobol_parser/package.toml b/src/lsp/cobol_parser/package.toml index 557c6e09..283620d3 100644 --- a/src/lsp/cobol_parser/package.toml +++ b/src/lsp/cobol_parser/package.toml @@ -102,7 +102,7 @@ dune-trailer = """ (action (with-stdout-to %{targets} (run %{exe:./keywords/gen_keywords.exe} %{deps} - --external-tokens Grammar_tokens)))) + --external-tokens Grammar_tokens --with-is-intrinsic)))) (rule (targets grammar_expect.ml) diff --git a/src/lsp/cobol_parser/text_keywords.mli b/src/lsp/cobol_parser/text_keywords.mli index 5b8f0759..bb3e7fb1 100644 --- a/src/lsp/cobol_parser/text_keywords.mli +++ b/src/lsp/cobol_parser/text_keywords.mli @@ -29,4 +29,4 @@ val silenced_keywords: string list (** Mapping from punctuations to their respective tokens *) val puncts: (string * Grammar_tokens.token) list -val is_intrinsic_token: Grammar_tokens.token -> bool +val is_known_intrinsic_token: Grammar_tokens.token -> bool diff --git a/src/lsp/cobol_parser/text_tokenizer.ml b/src/lsp/cobol_parser/text_tokenizer.ml index e81b2b26..feb40585 100644 --- a/src/lsp/cobol_parser/text_tokenizer.ml +++ b/src/lsp/cobol_parser/text_tokenizer.ml @@ -510,13 +510,17 @@ let reword_intrinsics s : tokens -> tokens = [Disabled_intrinsics] does not occur on a `FUNCTION` keyword (but that's unlikely). *) let keyword_of_token = Hashtbl.find Text_lexer.word_of_token in + let is_intrinsic_token = function + | INTRINSIC_FUNC _ -> true + | t when Text_keywords.is_known_intrinsic_token t -> true + | _ -> false in let rec aux rev_prefix suffix = match suffix with | [] -> List.rev rev_prefix | ({ payload = k1_token; _ } as k1) :: ({ payload = k2_token; _ } as k2) :: tl - when k1_token <> FUNCTION && Text_keywords.is_intrinsic_token k2_token -> + when k1_token <> FUNCTION && is_intrinsic_token k2_token -> aux (EzList.tail_map distinguish_words @@ retokenize s (keyword_of_token k2_token &@<- k2) @ k1 :: rev_prefix) tl