diff --git a/src/dune_rules/cram/cram_exec.ml b/src/dune_rules/cram/cram_exec.ml index f1876be5748..e1e1abb4e45 100644 --- a/src/dune_rules/cram/cram_exec.ml +++ b/src/dune_rules/cram/cram_exec.ml @@ -54,7 +54,8 @@ module For_tests = struct let dyn_of_block = function | Cram_lexer.Comment lines -> Dyn.variant "Comment" [ Dyn.list Dyn.string lines ] - | Command lines -> Dyn.variant "Command" [ Dyn.list Dyn.string lines ] + | Command (loc, lines) -> + Dyn.variant "Command" [ Loc.to_dyn loc; Dyn.list Dyn.string lines ] ;; end @@ -178,10 +179,10 @@ let read_and_attach_exit_codes (sh_script : sh_script) | [], [] -> List.rev acc | (Cram_lexer.Comment _ as comment) :: blocks, _ -> loop (comment :: acc) entries blocks - | Command block_result :: blocks, metadata_entry :: entries -> - loop (Command (block_result, Present metadata_entry) :: acc) entries blocks - | Cram_lexer.Command block_result :: blocks, [] -> - loop (Command (block_result, Missing_unreachable) :: acc) entries blocks + | Command (loc, block_result) :: blocks, metadata_entry :: entries -> + loop (Command (loc, (block_result, Present metadata_entry)) :: acc) entries blocks + | Cram_lexer.Command (loc, block_result) :: blocks, [] -> + loop (Command (loc, (block_result, Missing_unreachable)) :: acc) entries blocks | [], _ :: _ -> Code_error.raise "more blocks than metadata" [] in loop [] metadata_entries sh_script.cram_to_output @@ -238,7 +239,7 @@ let sanitize ~parent_script cram_to_output : command_out Cram_lexer.block list = List.map cram_to_output ~f:(fun (t : (block_result * _) Cram_lexer.block) -> match t with | Cram_lexer.Comment t -> Cram_lexer.Comment t - | Command (block_result, metadata) -> + | Command (loc, (block_result, metadata)) -> let output = match metadata with | Missing_unreachable -> "***** UNREACHABLE *****" @@ -250,7 +251,7 @@ let sanitize ~parent_script cram_to_output : command_out Cram_lexer.block list = ~command_script:block_result.script build_path_prefix_map in - Command { command = block_result.command; metadata; output }) + Command (loc, { command = block_result.command; metadata; output })) ;; (* Compose user written cram stanzas to output *) @@ -267,7 +268,7 @@ let compose_cram_output (cram_to_output : _ Cram_lexer.block list) = List.iter cram_to_output ~f:(fun block -> match (block : _ Cram_lexer.block) with | Comment lines -> List.iter lines ~f:add_line - | Command { command; metadata; output } -> + | Command (_loc, { command; metadata; output }) -> List.iteri command ~f:(fun i line -> let line = sprintf "%c %s" (if i = 0 then '$' else '>') line in add_line_prefixed_with_two_space line); @@ -314,7 +315,7 @@ let create_sh_script cram_stanzas ~temp_dir : sh_script Fiber.t = let loop block = match (block : _ Cram_lexer.block) with | Comment _ as comment -> Fiber.return comment - | Command lines -> + | Command (loc, lines) -> incr i; let i = !i in let file ~ext = file (sprintf "%d%s" i ext) in @@ -340,10 +341,11 @@ let create_sh_script cram_stanzas ~temp_dir : sh_script Fiber.t = Dune_util.Build_path_prefix_map._BUILD_PATH_PREFIX_MAP metadata_file_sh_path; Cram_lexer.Command - { command = lines - ; output_file = user_shell_code_output_file - ; script = user_shell_code_file - } + ( loc + , { command = lines + ; output_file = user_shell_code_output_file + ; script = user_shell_code_file + } ) in fprln oc "trap 'exit 0' EXIT"; let+ cram_to_output = Fiber.sequential_map ~f:loop cram_stanzas in @@ -433,7 +435,7 @@ let run_cram_test env ~src ~script ~cram_stanzas ~temp_dir ~cwd ~timeout = let command_blocks_only = List.filter_map sh_script.cram_to_output ~f:(function | Cram_lexer.Comment _ -> None - | Cram_lexer.Command block_result -> Some block_result) + | Cram_lexer.Command (_, block_result) -> Some block_result) in let total_commands = List.length command_blocks_only in if completed_count < total_commands @@ -486,7 +488,7 @@ let run_and_produce_output ~src ~env ~dir:cwd ~script ~dst ~timeout = let+ commands = run_cram_test env ~src ~script ~cram_stanzas ~temp_dir ~cwd ~timeout >>| List.filter_map ~f:(function - | Cram_lexer.Command c -> Some c + | Cram_lexer.Command (_, c) -> Some c | Comment _ -> None) in let dst = Path.build dst in @@ -552,7 +554,7 @@ module Make_script = struct |> cram_stanzas |> List.filter_map ~f:(function | Cram_lexer.Comment _ -> None - | Command s -> Some s) + | Command (_, s) -> Some s) |> cram_commmands in Io.write_file ~binary:false (Path.build dst) commands; @@ -596,10 +598,11 @@ module Diff = struct | [] -> acc | Cram_lexer.Comment x :: current -> loop (Cram_lexer.Comment x :: acc) current expected - | Command _ :: current -> + | Command (loc, _) :: current -> (match expected with | [] -> acc - | out :: expected -> loop (Cram_lexer.Command out :: acc) current expected) + | out :: expected -> + loop (Cram_lexer.Command (loc, out) :: acc) current expected) in loop [] current_stanzas out |> List.rev in diff --git a/src/dune_rules/cram/cram_lexer.mli b/src/dune_rules/cram/cram_lexer.mli index 660a2d83567..3517ce9d473 100644 --- a/src/dune_rules/cram/cram_lexer.mli +++ b/src/dune_rules/cram/cram_lexer.mli @@ -1,8 +1,10 @@ +open Import + (** .t file parser *) (** A command or comment. Output blocks are skipped *) type 'command block = - | Command of 'command + | Command of Loc.t * 'command | Comment of string list val block : Lexing.lexbuf -> string list block option diff --git a/src/dune_rules/cram/cram_lexer.mll b/src/dune_rules/cram/cram_lexer.mll index 35846d97bd5..4f8a325f11a 100644 --- a/src/dune_rules/cram/cram_lexer.mll +++ b/src/dune_rules/cram/cram_lexer.mll @@ -1,67 +1,107 @@ { +open Import + type 'command block = - | Command of 'command + | Command of Loc.t * 'command | Comment of string list } -let eol = '\n' | eof +let nonspace = [^' ' '\n'] +let not_nl = [^'\n'] -let blank = [' ' '\t' '\r' '\012'] +rule eol = parse + | '\n' { Lexing.new_line lexbuf; true } + | eof { false } -rule block = parse +and block = parse | eof { None } - | " $ " ([^'\n']* as str) eol - { Some (command_cont [str] lexbuf) } - | " " [^'\n']* eol - { output [] lexbuf } - | ' '? as str eol - { comment [str] lexbuf } - | ' '? [^' ' '\n'] [^'\n']* as str eol - { comment [str] lexbuf } + | " $ " ([^'\n']* as str) + { let start = Lexing.lexeme_start_p lexbuf in + let start = { start with pos_cnum = start.pos_cnum + 2 } in + let stop0 = Lexing.lexeme_end_p lexbuf in + match eol lexbuf with + | true -> Some (command_cont start stop0 [ str ] lexbuf) + | false -> Some (Command (Loc.create ~start ~stop:stop0, [ str ])) } + | " > " ([^'\n']* as str) + { ignore (eol lexbuf); + comment [ " > " ^ str ] lexbuf } + | " >" + { ignore (eol lexbuf); + comment [ " >" ] lexbuf } + | " " [^'\n']* + { ignore (eol lexbuf); + output [] lexbuf } + | ' ' ((nonspace not_nl*) as rest) + { ignore (eol lexbuf); + comment [ " " ^ rest ] lexbuf } + | ' ' '\n' + { Lexing.new_line lexbuf; + comment [ " " ] lexbuf } + | ' ' + { comment [ " " ] lexbuf } + | '\n' + { Lexing.new_line lexbuf; + comment [ "" ] lexbuf } + | nonspace not_nl* as str + { ignore (eol lexbuf); + comment [str] lexbuf } and comment acc = parse | eof { match acc with - | [] -> None - | _ -> Some (Comment (List.rev acc)) - } - | ' '? as str eol - { comment (str :: acc) lexbuf } - | ' '? [^' ' '\n'] [^'\n']* as str eol - { comment (str :: acc) lexbuf } - | "" - { Some (Comment (List.rev acc)) } + | [] -> None + | _ -> Some (Comment (List.rev acc)) } + | ' ' ((nonspace not_nl*) as rest) + { ignore (eol lexbuf); + comment ((" " ^ rest) :: acc) lexbuf } + | ' ' '\n' + { Lexing.new_line lexbuf; + comment (" " :: acc) lexbuf } + | '\n' + { Lexing.new_line lexbuf; + comment ("" :: acc) lexbuf } + | nonspace not_nl* as str + { ignore (eol lexbuf); + comment (str :: acc) lexbuf } + | "" { Some (Comment (List.rev acc)) } and output maybe_comment = parse | eof { match maybe_comment with - | [] -> None - | l -> Some (Comment (List.rev l)) - } - | ' ' eof - { Some (Comment (List.rev (" " :: maybe_comment))) } - | " "? eof - { None } - | " " eol - { output [] lexbuf } - | ' '? as s eol - { output (s :: maybe_comment) lexbuf } - | " $" eol - { output [] lexbuf } - | " " '$' [^' ' '\n'] [^'\n']* eol - { output [] lexbuf } - | " " [^'$' '\n'] [^'\n']* eol - { output [] lexbuf } + | [] -> None + | l -> Some (Comment (List.rev l)) } + | " $ " ([^'\n']* as str) + { let start = Lexing.lexeme_start_p lexbuf in + let start = { start with pos_cnum = start.pos_cnum + 2 } in + let stop0 = Lexing.lexeme_end_p lexbuf in + match eol lexbuf with + | true -> Some (command_cont start stop0 [ str ] lexbuf) + | false -> Some (Command (Loc.create ~start ~stop:stop0, [ str ])) } + | ' ' ((nonspace not_nl*) as rest) + { match eol lexbuf with + | true -> output ((" " ^ rest) :: maybe_comment) lexbuf + | false -> Some (Comment (List.rev ((" " ^ rest) :: maybe_comment))) } + | ' ' '\n' + { Lexing.new_line lexbuf; + output (" " :: maybe_comment) lexbuf } + | " " [^'\n']* + { ignore (eol lexbuf); + output maybe_comment lexbuf } | "" { match maybe_comment with - | [] -> block lexbuf - | l -> comment l lexbuf - } + | [] -> block lexbuf + | l -> comment l lexbuf } -and command_cont acc = parse - | " > " ([^'\n']* as str) eol - { command_cont (str :: acc) lexbuf } - | " >" eol - { command_cont ("" :: acc) lexbuf } +and command_cont start last_stop acc = parse + | " > " ([^'\n']* as str) + { let stop0 = Lexing.lexeme_end_p lexbuf in + match eol lexbuf with + | true -> command_cont start stop0 (str :: acc) lexbuf + | false -> Command (Loc.create ~start ~stop:stop0, List.rev (str :: acc)) } + | " >" + { let stop0 = Lexing.lexeme_end_p lexbuf in + match eol lexbuf with + | true -> command_cont start stop0 ("" :: acc) lexbuf + | false -> Command (Loc.create ~start ~stop:stop0, List.rev ("" :: acc)) } | "" - { Command (List.rev acc) } + { Command (Loc.create ~start ~stop:last_stop, List.rev acc) } diff --git a/test/expect-tests/dune_rules/cram_parsing_tests.ml b/test/expect-tests/dune_rules/cram_parsing_tests.ml index fa40a57e531..cea48e9907e 100644 --- a/test/expect-tests/dune_rules/cram_parsing_tests.ml +++ b/test/expect-tests/dune_rules/cram_parsing_tests.ml @@ -4,8 +4,9 @@ open Dune_rules.For_tests.Cram_exec.For_tests let () = Dune_tests_common.init () let test content = - Lexing.from_string content - |> cram_stanzas + let lexbuf = Lexing.from_string ~with_positions:true content in + Stdlib.Lexing.set_filename lexbuf "test"; + cram_stanzas lexbuf |> Dyn.list dyn_of_block |> Dyn.pp |> List.singleton @@ -14,39 +15,129 @@ let test content = let%expect_test "simple single line command" = test " $ echo 'Hello world'"; - [%expect {| [ Command [ "echo 'Hello world'" ] ] |}]; + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 22 } + }, + [ "echo 'Hello world'" ]) + ] + |}]; test " $ echo 'Hello world'\n"; - [%expect {| [ Command [ "echo 'Hello world'" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 22 } + }, + [ "echo 'Hello world'" ]) + ] + |}] ;; let%expect_test "command with single continuation" = test " $ echo 'Hello'\n > echo 'World!'"; - [%expect {| [ Command [ "echo 'Hello'"; "echo 'World!'" ] ] |}]; + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 2; pos_bol = 17; pos_cnum = 34 } + }, + [ "echo 'Hello'"; "echo 'World!'" ]) + ] + |}]; test " $ echo 'Hello'\n > echo 'World!'\n"; - [%expect {| [ Command [ "echo 'Hello'"; "echo 'World!'" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 2; pos_bol = 17; pos_cnum = 34 } + }, + [ "echo 'Hello'"; "echo 'World!'" ]) + ] + |}] ;; let%expect_test "command with empty continutation" = test " $ echo 'Hello..'\n >"; - [%expect {| [ Command [ "echo 'Hello..'"; "" ] ] |}]; + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 2; pos_bol = 19; pos_cnum = 22 } + }, + [ "echo 'Hello..'"; "" ]) + ] + |}]; test " $ echo 'Hello..'\n >\n"; - [%expect {| [ Command [ "echo 'Hello..'"; "" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 2; pos_bol = 19; pos_cnum = 22 } + }, + [ "echo 'Hello..'"; "" ]) + ] + |}] ;; (* Output lines are not parsed. They are skipped. When we run a cram test, we re-output the results and reassemble the cram test. *) let%expect_test "command followed by output lines" = test " $ echo 'Hi'\n Hi"; - [%expect {| [ Command [ "echo 'Hi'" ] ] |}]; + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 13 } + }, + [ "echo 'Hi'" ]) + ] + |}]; test " $ echo 'Hi'\n Hi\n"; - [%expect {| [ Command [ "echo 'Hi'" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 13 } + }, + [ "echo 'Hi'" ]) + ] + |}] ;; let%expect_test "empty output following a command" = test " $ echo 'Hi'\n "; - [%expect {| [ Command [ "echo 'Hi'" ] ] |}]; + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 13 } + }, + [ "echo 'Hi'" ]) + ] + |}]; test " $ echo 'Hi'\n \n"; - [%expect {| [ Command [ "echo 'Hi'" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 1; pos_bol = 0; pos_cnum = 2 } + ; stop = { pos_lnum = 1; pos_bol = 0; pos_cnum = 13 } + }, + [ "echo 'Hi'" ]) + ] + |}] ;; let%expect_test "loose output lines only" = @@ -89,14 +180,43 @@ let%expect_test "group consecutive comment lines" = let%expect_test "comment then command" = test "Intro text\n $ cmd"; - [%expect {| [ Comment [ "Intro text" ]; Command [ "cmd" ] ] |}]; + [%expect + {| + [ Comment [ "Intro text" ] + ; Command + ({ pos_fname = "test" + ; start = { pos_lnum = 2; pos_bol = 11; pos_cnum = 13 } + ; stop = { pos_lnum = 2; pos_bol = 11; pos_cnum = 18 } + }, + [ "cmd" ]) + ] + |}]; test "Intro text\n $ cmd\n"; - [%expect {| [ Comment [ "Intro text" ]; Command [ "cmd" ] ] |}] + [%expect + {| + [ Comment [ "Intro text" ] + ; Command + ({ pos_fname = "test" + ; start = { pos_lnum = 2; pos_bol = 11; pos_cnum = 13 } + ; stop = { pos_lnum = 2; pos_bol = 11; pos_cnum = 18 } + }, + [ "cmd" ]) + ] + |}] ;; let%expect_test "orphan output lines before command" = test " ignored\n also ignored\n $ real\n"; - [%expect {| [ Command [ "real" ] ] |}] + [%expect + {| + [ Command + ({ pos_fname = "test" + ; start = { pos_lnum = 3; pos_bol = 25; pos_cnum = 27 } + ; stop = { pos_lnum = 3; pos_bol = 25; pos_cnum = 33 } + }, + [ "real" ]) + ] + |}] ;; let%expect_test "mixed document" = @@ -104,9 +224,19 @@ let%expect_test "mixed document" = [%expect {| [ Comment [ "Doc line A"; "Doc line B" ] - ; Command [ "echo one" ] + ; Command + ({ pos_fname = "test" + ; start = { pos_lnum = 3; pos_bol = 22; pos_cnum = 24 } + ; stop = { pos_lnum = 3; pos_bol = 22; pos_cnum = 34 } + }, + [ "echo one" ]) ; Comment [ "" ] - ; Command [ "echo two" ] + ; Command + ({ pos_fname = "test" + ; start = { pos_lnum = 6; pos_bol = 42; pos_cnum = 44 } + ; stop = { pos_lnum = 6; pos_bol = 42; pos_cnum = 54 } + }, + [ "echo two" ]) ] |}] ;; @@ -115,3 +245,18 @@ let%expect_test "comments separated by new line" = test "Comment 1\n\nComment 2\n"; [%expect {| [ Comment [ "Comment 1"; ""; "Comment 2" ] ] |}] ;; + +let%expect_test "stray command continuation is a comment" = + test " > stray\n $ cmd"; + [%expect + {| + [ Comment [ " > stray" ] + ; Command + ({ pos_fname = "test" + ; start = { pos_lnum = 2; pos_bol = 10; pos_cnum = 12 } + ; stop = { pos_lnum = 2; pos_bol = 10; pos_cnum = 17 } + }, + [ "cmd" ]) + ] + |}] +;;