From 87f10ffdb573241bef67ad44b031f531d3e437d9 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Thu, 22 Aug 2024 22:07:56 -0700 Subject: [PATCH 1/5] feat: add raw identifier syntax --- .../reason_declarative_lexer.mll | 5 +- src/reason-parser/reason_pprint_ast.ml | 55 +++++++++++++------ test/raw-identifiers.t/input.re | 28 ++++++++++ test/raw-identifiers.t/run.t | 35 ++++++++++++ 4 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 test/raw-identifiers.t/input.re create mode 100644 test/raw-identifiers.t/run.t diff --git a/src/reason-parser/reason_declarative_lexer.mll b/src/reason-parser/reason_declarative_lexer.mll index 94dcf55a7..8fae46a53 100644 --- a/src/reason-parser/reason_declarative_lexer.mll +++ b/src/reason-parser/reason_declarative_lexer.mll @@ -385,6 +385,7 @@ let hex_float_literal = (['p' 'P'] ['+' '-']? ['0'-'9'] ['0'-'9' '_']* )? let literal_modifier = ['G'-'Z' 'g'-'z'] +let raw_ident_escape = "\\#" rule token state = parse | "\\" newline { @@ -408,6 +409,8 @@ rule token state = parse { QUESTION } | "=?" { set_lexeme_length lexbuf 1; EQUAL } + | raw_ident_escape (lowercase identchar * as name) + { LIDENT name } | lowercase identchar * { let s = Lexing.lexeme lexbuf in try Hashtbl.find keyword_table s @@ -521,7 +524,7 @@ rule token state = parse { CHAR (char_for_decimal_code lexbuf 2) } | "'\\" 'x' ['0'-'9' 'a'-'f' 'A'-'F'] ['0'-'9' 'a'-'f' 'A'-'F'] "'" { CHAR (char_for_hexadecimal_code lexbuf 3) } - | "'" (("\\" _) as esc) + | "'" (("\\" [^ '#']) as esc) { raise_error (Location.curr lexbuf) (Illegal_escape esc); token state lexbuf } diff --git a/src/reason-parser/reason_pprint_ast.ml b/src/reason-parser/reason_pprint_ast.ml index 6dd185832..b471852a1 100644 --- a/src/reason-parser/reason_pprint_ast.ml +++ b/src/reason-parser/reason_pprint_ast.ml @@ -2062,6 +2062,14 @@ let createFormatter () = settings.funcApplicationLabelStyle typeApplicationItems + let add_raw_identifier_prefix txt = + let prefix = + match Hashtbl.find Reason_declarative_lexer.keyword_table txt with + | _ -> "\\#" + | exception Not_found -> "" + in + prefix ^ txt + (* add parentheses to binders when they are in fact infix or prefix operators *) let protectIdentifier txt = @@ -2069,7 +2077,7 @@ let createFormatter () = let txt = if Reason_syntax_util.is_andop txt || Reason_syntax_util.is_letop txt then Reason_syntax_util.compress_letop_identifier txt - else txt + else txt |> add_raw_identifier_prefix in if not needs_parens then atom txt @@ -2926,7 +2934,9 @@ let createFormatter () = let itm = self#formatOneTypeDef prepend - (atom ~loc:td.ptype_name.loc td.ptype_name.txt) + (atom + ~loc:td.ptype_name.loc + (add_raw_identifier_prefix td.ptype_name.txt)) (atom eq_symbol) td in @@ -3412,6 +3422,9 @@ let createFormatter () = let variant_helper i rf = match rf.prf_desc with | Rtag (label, opt_ampersand, ctl) -> + let label = + { label with txt = add_raw_identifier_prefix label.txt } + in let pcd_args = Pcstr_tuple ctl in let all_attrs = List.concat [ pcd_attributes; rf.prf_attributes ] @@ -5007,7 +5020,7 @@ let createFormatter () = ~polyVariant:true ~arityIsClear:true stdAttrs - (atom ("`" ^ l)) + (atom ("`" ^ add_raw_identifier_prefix l)) eo ] (* TODO: Should protect this identifier *) @@ -7712,7 +7725,9 @@ let createFormatter () = (self#core_type ct) ]) | Pexp_variant (l, None) -> - Some (ensureSingleTokenSticksToLabel (atom ("`" ^ l))) + Some + (ensureSingleTokenSticksToLabel + (atom ("`" ^ add_raw_identifier_prefix l))) | Pexp_record (l, eo) -> Some (self#unparseRecord l eo) | Pexp_array l -> Some (self#unparseSequence ~construct:`Array l) | Pexp_let _ | Pexp_sequence _ | Pexp_letmodule _ @@ -8233,12 +8248,13 @@ let createFormatter () = in let upToName = + let name = add_raw_identifier_prefix pci_name.txt in if ls == [] - then label ~space:true (atom opener) (atom pci_name.txt) + then label ~space:true (atom opener) (atom name) else label ~space:true - (label ~space:true (atom opener) (atom pci_name.txt)) + (label ~space:true (atom opener) (atom name)) (self#class_params_def ls) in let includingEqual = @@ -8807,14 +8823,17 @@ let createFormatter () = () method modtype x ~delim = - let name = atom x.pmtd_name.txt in - let letPattern = - makeList ~postSpace:true [ atom "module type"; name; atom delim ] - in + let name = atom (add_raw_identifier_prefix x.pmtd_name.txt) in let main = match x.pmtd_type with | None -> makeList ~postSpace:true [ atom "module type"; name ] - | Some mt -> self#module_type letPattern mt + | Some mt -> + let letPattern = + makeList + ~postSpace:true + [ atom "module type"; name; atom delim ] + in + self#module_type letPattern mt in let { Reason_attributes.stdAttrs; docAttrs } = Reason_attributes.partitionAttributes @@ -9381,6 +9400,7 @@ let createFormatter () = , None ) method class_opening class_keyword name pci_virt ls = + let name = add_raw_identifier_prefix name in let firstToken = if class_keyword then "class" else "and" in match pci_virt, ls with (* When no class params, it's a very simple formatting for the @@ -9436,14 +9456,17 @@ let createFormatter () = self#attach_std_item_attrs binding.pmb_attributes module_binding | Pstr_open od -> self#pstr_open od | Pstr_modtype x -> - let name = atom x.pmtd_name.txt in - let letPattern = - makeList ~postSpace:true [ atom "module type"; name; atom "=" ] - in + let name = atom (add_raw_identifier_prefix x.pmtd_name.txt) in let main = match x.pmtd_type with | None -> makeList ~postSpace:true [ atom "module type"; name ] - | Some mt -> self#module_type letPattern mt + | Some mt -> + let letPattern = + makeList + ~postSpace:true + [ atom "module type"; name; atom "=" ] + in + self#module_type letPattern mt in self#attach_std_item_attrs x.pmtd_attributes main | Pstr_class l -> self#class_declaration_list l diff --git a/test/raw-identifiers.t/input.re b/test/raw-identifiers.t/input.re new file mode 100644 index 000000000..b70308fc2 --- /dev/null +++ b/test/raw-identifiers.t/input.re @@ -0,0 +1,28 @@ + +let \#let = 2; +let \#let = \#let +and \#and = \#let; + +/* labeled arguments */ +let \#let = (~\#let) => { + \#let; +}; + +let \#let = (~\#let: \#let = \#let) => { + \#let; +}; + +/* Types */ +type \#type = \#type; + +module type \#module = \#module; + +class \#class = \#class; + +class type \#class = \#class; + +type x = [ | `\#module ] +type y = [ | \#module ] +let x = `\#module + +external \#external: unit => unit = "external"; diff --git a/test/raw-identifiers.t/run.t b/test/raw-identifiers.t/run.t new file mode 100644 index 000000000..ac7a6f9d4 --- /dev/null +++ b/test/raw-identifiers.t/run.t @@ -0,0 +1,35 @@ +Test raw identifiers in Reason syntax + + $ refmt ./input.re | tee input2.re + let \#let = 2; + let \#let = \#let + and \#and = \#let; + + /* labeled arguments */ + let \#let = (~\#let) => { + \#let; + }; + + let \#let = (~\#let: \#let=\#let) => { + \#let; + }; + + /* Types */ + type \#type = \#type; + + module type \#module = \#module; + + class \#class = class \#class; + + class type \#class = \#class; + + type x = [ | `\#module]; + type y = [ \#module]; + let x = `\#module; + + external \#external: unit => unit = "external"; + +Check idempotency + + $ refmt ./input2.re > out.re + From c6a6581d7d994ce41b6058985f3d828d407d8560 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Fri, 23 Aug 2024 17:37:15 -0700 Subject: [PATCH 2/5] fix: record labels --- src/reason-parser/reason_pprint_ast.ml | 5 ++--- test/raw-identifiers.t/input.re | 10 ++++++++++ test/raw-identifiers.t/run.t | 10 ++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/reason-parser/reason_pprint_ast.ml b/src/reason-parser/reason_pprint_ast.ml index b471852a1..f6d0bb637 100644 --- a/src/reason-parser/reason_pprint_ast.ml +++ b/src/reason-parser/reason_pprint_ast.ml @@ -3109,9 +3109,8 @@ let createFormatter () = let recordRow pld = let hasPunning = recordRowIsPunned pld in let name = - if hasPunning - then [ atom pld.pld_name.txt ] - else [ atom pld.pld_name.txt; atom ":" ] + let name = add_raw_identifier_prefix pld.pld_name.txt in + if hasPunning then [ atom name ] else [ atom name; atom ":" ] in let name = source_map ~loc:pld.pld_name.loc (makeList name) in let withMutable = diff --git a/test/raw-identifiers.t/input.re b/test/raw-identifiers.t/input.re index b70308fc2..db1158e0e 100644 --- a/test/raw-identifiers.t/input.re +++ b/test/raw-identifiers.t/input.re @@ -26,3 +26,13 @@ type y = [ | \#module ] let x = `\#module external \#external: unit => unit = "external"; + +type \#rec = { + \#type: \#type, + \#module: module_ +}; + +let \#rec = { + \#type: \#type, + \#module: module_ +} diff --git a/test/raw-identifiers.t/run.t b/test/raw-identifiers.t/run.t index ac7a6f9d4..e4189a3e8 100644 --- a/test/raw-identifiers.t/run.t +++ b/test/raw-identifiers.t/run.t @@ -28,6 +28,16 @@ Test raw identifiers in Reason syntax let x = `\#module; external \#external: unit => unit = "external"; + + type \#rec = { + \#type, + \#module: module_, + }; + + let \#rec = { + \#type, + \#module: module_, + }; Check idempotency From 79469d344fcdcd34c32732ef68ae61a8098d3133 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 10 Nov 2024 18:31:05 -0800 Subject: [PATCH 3/5] true / false --- src/reason-parser/reason_pprint_ast.ml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/reason-parser/reason_pprint_ast.ml b/src/reason-parser/reason_pprint_ast.ml index f6d0bb637..cf0fb0884 100644 --- a/src/reason-parser/reason_pprint_ast.ml +++ b/src/reason-parser/reason_pprint_ast.ml @@ -718,11 +718,15 @@ type construct = | `normal | `simple of Longident.t | `tuple + | `btrue + | `bfalse ] let view_expr x = match x.pexp_desc with | Pexp_construct ({ txt = Lident "()" }, _) -> `tuple + | Pexp_construct ({ txt = Lident "true"; _ }, _) -> `btrue + | Pexp_construct ({ txt = Lident "false"; _ }, _) -> `bfalse | Pexp_construct ({ txt = Lident "[]" }, _) -> `nil | Pexp_construct ({ txt = Lident "::" }, Some _) -> let rec loop exp acc = @@ -745,7 +749,7 @@ let is_simple_list_expr x = match view_expr x with `list _ | `cons _ -> true | _ -> false let is_simple_construct : construct -> bool = function - | `nil | `tuple | `list _ | `simple _ | `cons _ -> true + | `nil | `tuple | `list _ | `simple _ | `btrue | `bfalse | `cons _ -> true | `normal -> false let uncurriedTable = Hashtbl.create 42 @@ -3611,6 +3615,9 @@ let createFormatter () = ~arityIsClear:true) | Ppat_lazy p -> label (atom "lazy") (formatPrecedence (self#simple_pattern p)) + | Ppat_construct + ({ txt = Lident (("true" | "false") as txt); _ }, None) -> + atom txt | Ppat_construct (({ txt } as li), po) when not (txt = Lident "::") -> (* FIXME The third field always false *) @@ -3759,8 +3766,9 @@ let createFormatter () = else let itm = match x.ppat_desc with - | Ppat_construct ({ loc; txt = Lident (("()" | "[]") as x) }, _) - -> + | Ppat_construct + ( { loc; txt = Lident (("()" | "[]" | "true" | "false") as x) } + , _ ) -> (* Patterns' locations might include a leading bar depending on the * context it was parsed in. Therefore, we need to include further * information about the contents of the pattern such as tokens etc, @@ -7669,6 +7677,8 @@ let createFormatter () = (match view_expr x with | `nil -> atom "[]" | `tuple -> atom "()" + | `btrue -> atom "true" + | `bfalse -> atom "false" | `list xs -> (* LIST EXPRESSION *) self#unparseSequence ~construct:`List xs From eb57930fe53dfe161886b2e1f9d8dfac4273b0a1 Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 10 Nov 2024 19:21:47 -0800 Subject: [PATCH 4/5] more testing --- test/raw-identifiers.t/input.re | 5 +++++ test/raw-identifiers.t/run.t | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/test/raw-identifiers.t/input.re b/test/raw-identifiers.t/input.re index db1158e0e..a08cf5b9a 100644 --- a/test/raw-identifiers.t/input.re +++ b/test/raw-identifiers.t/input.re @@ -36,3 +36,8 @@ let \#rec = { \#type: \#type, \#module: module_ } + +let true = x => x; + +let \#true = x => x; + diff --git a/test/raw-identifiers.t/run.t b/test/raw-identifiers.t/run.t index e4189a3e8..b432d7680 100644 --- a/test/raw-identifiers.t/run.t +++ b/test/raw-identifiers.t/run.t @@ -38,6 +38,10 @@ Test raw identifiers in Reason syntax \#type, \#module: module_, }; + + let true = x => x; + + let \#true = x => x; Check idempotency From f990d99c3e401fe276d078a4013bba0f28c1287e Mon Sep 17 00:00:00 2001 From: Antonio Nuno Monteiro Date: Sun, 10 Nov 2024 20:10:41 -0800 Subject: [PATCH 5/5] add changelog entry --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3638c5547..30c7f7e63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ [#2800](https://github.com/reasonml/reason/pull/2800)) - Fix: don't print all extension strings as quoted extensions (@anmonteiro, [#2809](https://github.com/reasonml/reason/pull/2809)) +- Add support for raw identifier syntax (@anmonteiro, + [#2796](https://github.com/reasonml/reason/pull/2796)) ## 3.13.0