mirror of
https://github.com/ocaml-multicore/eio.git
synced 2025-08-29 00:03:47 -04:00
Add Buf_read.{seq,lines} and Dir.{load,save} convenience functions
This makes it easy to load and save whole files, or to read a file line by line. Also, adds `at_end_of_input` and renames `eof` to `end_of_input` to match Angstrom. Also, fix a bug in `traceln` where partially applying it resulted in multiple uses sharing the same buffer.
This commit is contained in:
Thomas Leonard
parent
3498a14593
commit
f157ed3309
10
fuzz/test.ml
10
fuzz/test.ml
@ -111,8 +111,13 @@ module Model = struct
|
||||
);
|
||||
consume t n
|
||||
|
||||
let eof t =
|
||||
let end_of_input t =
|
||||
if !t <> "" then failwith "not eof"
|
||||
|
||||
let rec lines t =
|
||||
match line t with
|
||||
| line -> line :: lines t
|
||||
| exception End_of_file -> []
|
||||
end
|
||||
|
||||
type op = Op : 'a Crowbar.printer * 'a Buf_read.parser * (Model.t -> 'a) -> op
|
||||
@ -137,7 +142,8 @@ let op =
|
||||
"take_while digit", Crowbar.const @@ Op (Fmt.Dump.string, Buf_read.take_while digit, Model.take_while digit);
|
||||
"skip_while digit", Crowbar.const @@ Op (unit, Buf_read.skip_while digit, Model.skip_while digit);
|
||||
"skip", Crowbar.(map [int]) (fun n -> Op (unit, Buf_read.skip n, Model.skip n));
|
||||
"eof", Crowbar.const @@ Op (unit, Buf_read.eof, Model.eof);
|
||||
"end_of_input", Crowbar.const @@ Op (unit, Buf_read.end_of_input, Model.end_of_input);
|
||||
"lines", Crowbar.const @@ Op (Fmt.Dump.(list string), (Buf_read.(map List.of_seq lines)), Model.lines);
|
||||
]
|
||||
|
||||
let catch f x =
|
||||
|
||||
@ -252,14 +252,18 @@ let line t =
|
||||
consume t (nl + 1);
|
||||
line
|
||||
|
||||
let eof t =
|
||||
if t.len = 0 && eof_seen t then ()
|
||||
let at_end_of_input t =
|
||||
if t.len = 0 && eof_seen t then true
|
||||
else (
|
||||
match ensure t 1 with
|
||||
| () -> failwith "Unexpected data after parsing"
|
||||
| exception End_of_file -> ()
|
||||
| () -> false
|
||||
| exception End_of_file -> true
|
||||
)
|
||||
|
||||
let end_of_input t =
|
||||
if not (at_end_of_input t) then
|
||||
failwith "Unexpected data after parsing"
|
||||
|
||||
let pp_pos f t =
|
||||
Fmt.pf f "at offset %d" (consumed_bytes t)
|
||||
|
||||
@ -272,4 +276,13 @@ let format_errors p t =
|
||||
|
||||
let parse ?initial_size ~max_size p flow =
|
||||
let buf = of_flow flow ?initial_size ~max_size in
|
||||
format_errors (p <* eof) buf
|
||||
format_errors (p <* end_of_input) buf
|
||||
|
||||
let seq ?(stop=at_end_of_input) p t =
|
||||
let rec aux () =
|
||||
if stop t then Seq.Nil
|
||||
else Seq.Cons (p t, aux)
|
||||
in
|
||||
aux
|
||||
|
||||
let lines t = seq line t
|
||||
|
||||
@ -45,3 +45,17 @@ let with_open_out ?append ~create (t:#t) path fn =
|
||||
|
||||
let with_open_dir (t:#t) path fn =
|
||||
Switch.run @@ fun sw -> fn (open_dir ~sw t path)
|
||||
|
||||
let with_lines (t:#t) path fn =
|
||||
with_open_in t path @@ fun flow ->
|
||||
let buf = Buf_read.of_flow flow ~max_size:max_int in
|
||||
fn (Buf_read.lines buf)
|
||||
|
||||
let load (t:#t) path =
|
||||
with_open_in t path @@ fun flow ->
|
||||
let buf = Buf_read.of_flow flow ~max_size:max_int in
|
||||
Buf_read.take_all buf
|
||||
|
||||
let save ?append ~create (t:#t) path data =
|
||||
with_open_out ?append ~create t path @@ fun flow ->
|
||||
Flow.copy_string data flow
|
||||
|
||||
@ -525,12 +525,16 @@ module Buf_read : sig
|
||||
(** {2 Reading data} *)
|
||||
|
||||
val line : string parser
|
||||
(** [line t] parses one line.
|
||||
(** [line] parses one line.
|
||||
Lines can be terminated by either LF or CRLF.
|
||||
The returned string does not include the terminator.
|
||||
If [End_of_file] is reached after seeing some data but before seeing a line
|
||||
terminator, the data seen is returned as the last line. *)
|
||||
|
||||
val lines : string Seq.t parser
|
||||
(** [lines] returns a sequence that lazily reads the next line until the end of the input is reached.
|
||||
[lines = seq line ~stop:at_end_of_input] *)
|
||||
|
||||
val char : char -> unit parser
|
||||
(** [char c] checks that the next byte is [c] and consumes it.
|
||||
@raise Failure if the next byte is not [c] *)
|
||||
@ -573,12 +577,22 @@ module Buf_read : sig
|
||||
except that the number of skipped bytes may be larger than the buffer (it will not grow).
|
||||
Note: if [End_of_file] is raised, all bytes in the stream will have been consumed. *)
|
||||
|
||||
val eof : unit parser
|
||||
(** [eof] checks that there are no further bytes in the stream.
|
||||
val at_end_of_input : bool parser
|
||||
(** [at_end_of_input] returns [true] when at the end of the stream, or
|
||||
[false] if there is at least one more byte to be read. *)
|
||||
|
||||
val end_of_input : unit parser
|
||||
(** [end_of_input] checks that there are no further bytes in the stream.
|
||||
@raise Failure if there are further bytes *)
|
||||
|
||||
(** {2 Combinators} *)
|
||||
|
||||
val seq : ?stop:bool parser -> 'a parser -> 'a Seq.t parser
|
||||
(** [seq p] is a sequence that uses [p] to get the next item.
|
||||
@param stop This is used before parsing each item.
|
||||
The sequence ends if this returns [true].
|
||||
The default is {!at_end_of_input}. *)
|
||||
|
||||
val pair : 'a parser -> 'b parser -> ('a * 'b) parser
|
||||
(** [pair a b] is a parser that first uses [a] to parse a value [x],
|
||||
then uses [b] to parse a value [y], then returns [(x, y)].
|
||||
@ -830,6 +844,14 @@ module Dir : sig
|
||||
method virtual close : unit
|
||||
end
|
||||
|
||||
val load : #t -> path -> string
|
||||
(** [load t path] returns the contents of the given file.
|
||||
This is a convenience wrapper around {!with_open_in}. *)
|
||||
|
||||
val save : ?append:bool -> create:create -> #t -> path -> string -> unit
|
||||
(** [save t path data ~create] writes [data] to [path].
|
||||
This is a convenience wrapper around {!with_open_out}. *)
|
||||
|
||||
val open_in : sw:Switch.t -> #t -> path -> <Flow.source; Flow.close>
|
||||
(** [open_in ~sw t path] opens [t/path] for reading.
|
||||
Note: files are always opened in binary mode. *)
|
||||
@ -838,6 +860,9 @@ module Dir : sig
|
||||
(** [with_open_in] is like [open_in], but calls [fn flow] with the new flow and closes
|
||||
it automatically when [fn] returns (if it hasn't already been closed by then). *)
|
||||
|
||||
val with_lines : #t -> path -> (string Seq.t -> 'a) -> 'a
|
||||
(** [with_lines t path fn] is a convenience function for streaming the lines of the file. *)
|
||||
|
||||
val open_out :
|
||||
sw:Switch.t ->
|
||||
?append:bool ->
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
let mutex = Mutex.create ()
|
||||
|
||||
let default_traceln ?__POS__:pos fmt =
|
||||
let b = Buffer.create 512 in
|
||||
let k f =
|
||||
let k go =
|
||||
let b = Buffer.create 512 in
|
||||
let f = Format.formatter_of_buffer b in
|
||||
go f;
|
||||
Option.iter (fun (file, lnum, _, _) -> Format.fprintf f " [%s:%d]" file lnum) pos;
|
||||
Format.pp_close_box f ();
|
||||
Format.pp_print_flush f ();
|
||||
@ -14,4 +16,4 @@ let default_traceln ?__POS__:pos fmt =
|
||||
List.iter (Printf.eprintf "+%s\n") lines;
|
||||
flush stderr
|
||||
in
|
||||
Format.kfprintf k (Format.formatter_of_buffer b) ("@[" ^^ fmt)
|
||||
Format.kdprintf k ("@[" ^^ fmt)
|
||||
|
||||
@ -439,12 +439,12 @@ Exception: Failure "Expected 'a' but got 'b'".
|
||||
+mock_flow returning 3 bytes
|
||||
- : (string, [> `Msg of string ]) result = Ok "abc"
|
||||
|
||||
# test ["abc"] R.(format_errors (take 2 <* eof));;
|
||||
# test ["abc"] R.(format_errors (take 2 <* end_of_input));;
|
||||
+mock_flow returning 3 bytes
|
||||
- : (string, [> `Msg of string ]) result =
|
||||
Error (`Msg "Unexpected data after parsing (at offset 2)")
|
||||
|
||||
# test ["abc"] R.(format_errors (take 4 <* eof));;
|
||||
# test ["abc"] R.(format_errors (take 4 <* end_of_input));;
|
||||
+mock_flow returning 3 bytes
|
||||
+mock_flow returning Eof
|
||||
- : (string, [> `Msg of string ]) result =
|
||||
@ -455,3 +455,27 @@ Error (`Msg "Unexpected end-of-file at offset 3")
|
||||
- : (string, [> `Msg of string ]) result =
|
||||
Error (`Msg "Buffer size limit exceeded when reading at offset 0")
|
||||
```
|
||||
|
||||
## Sequences
|
||||
|
||||
```ocaml
|
||||
# test ["one"; "\ntwo\n"; "three"] R.lines |> Seq.iter (traceln "%S");;
|
||||
+mock_flow returning 3 bytes
|
||||
+mock_flow returning 5 bytes
|
||||
+"one"
|
||||
+"two"
|
||||
+mock_flow returning 5 bytes
|
||||
+mock_flow returning Eof
|
||||
+"three"
|
||||
- : unit = ()
|
||||
|
||||
# test ["abcd1234"] R.(seq (take 2)) |> List.of_seq |> String.concat ",";;
|
||||
+mock_flow returning 8 bytes
|
||||
+mock_flow returning Eof
|
||||
- : string = "ab,cd,12,34"
|
||||
|
||||
# test ["abcd123"] R.(seq (take 2)) |> List.of_seq |> String.concat ",";;
|
||||
+mock_flow returning 7 bytes
|
||||
+mock_flow returning Eof
|
||||
Exception: End_of_file.
|
||||
```
|
||||
|
||||
@ -21,23 +21,11 @@ let run (fn : Eio.Stdenv.t -> unit) =
|
||||
Eio_main.run @@ fun env ->
|
||||
fn env
|
||||
|
||||
let read_all flow =
|
||||
let b = Buffer.create 100 in
|
||||
Eio.Flow.copy flow (Eio.Flow.buffer_sink b);
|
||||
Buffer.contents b
|
||||
|
||||
let write_file ~create ?append dir path content =
|
||||
Eio.Dir.with_open_out ~create ?append dir path @@ fun flow ->
|
||||
Eio.Flow.copy_string content flow
|
||||
|
||||
let try_write_file ~create ?append dir path content =
|
||||
match write_file ~create ?append dir path content with
|
||||
match Eio.Dir.save ~create ?append dir path content with
|
||||
| () -> traceln "write %S -> ok" path
|
||||
| exception ex -> traceln "write %S -> %a" path Fmt.exn ex
|
||||
|
||||
let read_file dir path =
|
||||
Eio.Dir.with_open_in dir path read_all
|
||||
|
||||
let try_mkdir dir path =
|
||||
match Eio.Dir.mkdir dir path ~perm:0o700 with
|
||||
| () -> traceln "mkdir %S -> ok" path
|
||||
@ -54,8 +42,8 @@ Creating a file and reading it back:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Exclusive 0o666) cwd "test-file" "my-data";
|
||||
traceln "Got %S" @@ read_file cwd "test-file";;
|
||||
Eio.Dir.save ~create:(`Exclusive 0o666) cwd "test-file" "my-data";
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "test-file";;
|
||||
+Got "my-data"
|
||||
- : unit = ()
|
||||
```
|
||||
@ -73,7 +61,7 @@ Trying to use cwd to access a file outside of that subtree fails:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Exclusive 0o666) cwd "../test-file" "my-data";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o666) cwd "../test-file" "my-data";
|
||||
failwith "Should have failed";;
|
||||
Exception: Eio.Dir.Permission_denied ("../test-file", _)
|
||||
```
|
||||
@ -82,7 +70,7 @@ Trying to use cwd to access an absolute path fails:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Exclusive 0o666) cwd "/tmp/test-file" "my-data";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o666) cwd "/tmp/test-file" "my-data";
|
||||
failwith "Should have failed";;
|
||||
Exception: Eio.Dir.Permission_denied ("/tmp/test-file", _)
|
||||
```
|
||||
@ -93,8 +81,8 @@ Exclusive create fails if already exists:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Exclusive 0o666) cwd "test-file" "first-write";
|
||||
write_file ~create:(`Exclusive 0o666) cwd "test-file" "first-write";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o666) cwd "test-file" "first-write";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o666) cwd "test-file" "first-write";
|
||||
failwith "Should have failed";;
|
||||
Exception: Eio.Dir.Already_exists ("test-file", _)
|
||||
```
|
||||
@ -103,9 +91,9 @@ If-missing create succeeds if already exists:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`If_missing 0o666) cwd "test-file" "1st-write-original";
|
||||
write_file ~create:(`If_missing 0o666) cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ read_file cwd "test-file";;
|
||||
Eio.Dir.save ~create:(`If_missing 0o666) cwd "test-file" "1st-write-original";
|
||||
Eio.Dir.save ~create:(`If_missing 0o666) cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "test-file";;
|
||||
+Got "2nd-write-original"
|
||||
- : unit = ()
|
||||
```
|
||||
@ -114,9 +102,9 @@ Truncate create succeeds if already exists, and truncates:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Or_truncate 0o666) cwd "test-file" "1st-write-original";
|
||||
write_file ~create:(`Or_truncate 0o666) cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ read_file cwd "test-file";;
|
||||
Eio.Dir.save ~create:(`Or_truncate 0o666) cwd "test-file" "1st-write-original";
|
||||
Eio.Dir.save ~create:(`Or_truncate 0o666) cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "test-file";;
|
||||
+Got "2nd-write"
|
||||
- : unit = ()
|
||||
# Unix.unlink "test-file";;
|
||||
@ -127,8 +115,8 @@ Error if no create and doesn't exist:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:`Never cwd "test-file" "1st-write-original";
|
||||
traceln "Got %S" @@ read_file cwd "test-file";;
|
||||
Eio.Dir.save ~create:`Never cwd "test-file" "1st-write-original";
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "test-file";;
|
||||
Exception: Eio.Dir.Not_found ("test-file", _)
|
||||
```
|
||||
|
||||
@ -136,9 +124,9 @@ Appending to an existing file:
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
write_file ~create:(`Or_truncate 0o666) cwd "test-file" "1st-write-original";
|
||||
write_file ~create:`Never ~append:true cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ read_file cwd "test-file";;
|
||||
Eio.Dir.save ~create:(`Or_truncate 0o666) cwd "test-file" "1st-write-original";
|
||||
Eio.Dir.save ~create:`Never ~append:true cwd "test-file" "2nd-write";
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "test-file";;
|
||||
+Got "1st-write-original2nd-write"
|
||||
- : unit = ()
|
||||
# Unix.unlink "test-file";;
|
||||
@ -152,7 +140,7 @@ Appending to an existing file:
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
try_mkdir cwd "subdir";
|
||||
try_mkdir cwd "subdir/nested";
|
||||
write_file ~create:(`Exclusive 0o600) cwd "subdir/nested/test-file" "data";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o600) cwd "subdir/nested/test-file" "data";
|
||||
();;
|
||||
+mkdir "subdir" -> ok
|
||||
+mkdir "subdir/nested" -> ok
|
||||
@ -196,9 +184,9 @@ Create a sandbox, write a file with it, then read it from outside:
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
try_mkdir cwd "sandbox";
|
||||
let subdir = Eio.Dir.open_dir ~sw cwd "sandbox" in
|
||||
write_file ~create:(`Exclusive 0o600) subdir "test-file" "data";
|
||||
Eio.Dir.save ~create:(`Exclusive 0o600) subdir "test-file" "data";
|
||||
try_mkdir subdir "../new-sandbox";
|
||||
traceln "Got %S" @@ read_file cwd "sandbox/test-file";;
|
||||
traceln "Got %S" @@ Eio.Dir.load cwd "sandbox/test-file";;
|
||||
+mkdir "sandbox" -> ok
|
||||
+mkdir "../new-sandbox" -> Eio.Dir.Permission_denied ("../new-sandbox", _)
|
||||
+Got "data"
|
||||
@ -248,3 +236,18 @@ Can use `fs` to access absolute paths:
|
||||
+Trying with cwd instead fails:
|
||||
Exception: Eio.Dir.Permission_denied ("/dev/null", _)
|
||||
```
|
||||
|
||||
## Streamling lines
|
||||
|
||||
```ocaml
|
||||
# run @@ fun env ->
|
||||
let cwd = Eio.Stdenv.cwd env in
|
||||
Eio.Dir.save ~create:(`Exclusive 0o600) cwd "test-data" "one\ntwo\nthree";
|
||||
Eio.Dir.with_lines cwd "test-data" (fun lines ->
|
||||
Seq.iter (traceln "Line: %s") lines
|
||||
);;
|
||||
+Line: one
|
||||
+Line: two
|
||||
+Line: three
|
||||
- : unit = ()
|
||||
```
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user