From 4e43e63aa1a4c57279943906dc41c0a2e06e8422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Sat, 28 May 2022 18:12:07 -0700 Subject: [PATCH 01/84] feat: Add a JSON serializer/deserializer --- src/BoundedInts.dfy | 30 ++- src/JSON/Deserializer.dfy | 431 ++++++++++++++++++++++++++++++++++++++ src/JSON/Grammar.dfy | 165 +++++++++++++++ src/JSON/Lexers.dfy | 51 +++++ src/JSON/Parsers.dfy | 57 +++++ src/JSON/Stacks.dfy | 133 ++++++++++++ src/JSON/Tests.dfy | 56 +++++ src/JSON/Views.dfy | 80 +++++++ 8 files changed, 998 insertions(+), 5 deletions(-) create mode 100644 src/JSON/Deserializer.dfy create mode 100644 src/JSON/Grammar.dfy create mode 100644 src/JSON/Lexers.dfy create mode 100644 src/JSON/Parsers.dfy create mode 100644 src/JSON/Stacks.dfy create mode 100644 src/JSON/Tests.dfy create mode 100644 src/JSON/Views.dfy diff --git a/src/BoundedInts.dfy b/src/BoundedInts.dfy index 088fe15b..c32b7754 100644 --- a/src/BoundedInts.dfy +++ b/src/BoundedInts.dfy @@ -2,7 +2,6 @@ module BoundedInts { const TWO_TO_THE_0: int := 1 - const TWO_TO_THE_1: int := 2 const TWO_TO_THE_2: int := 4 const TWO_TO_THE_4: int := 16 @@ -17,15 +16,14 @@ module BoundedInts { const TWO_TO_THE_64: int := 0x1_00000000_00000000 const TWO_TO_THE_128: int := 0x1_00000000_00000000_00000000_00000000 const TWO_TO_THE_256: int := 0x1_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000 - const TWO_TO_THE_512: int := - 0x1_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; + const TWO_TO_THE_512: int := 0x1_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000_00000000; - newtype uint8 = x: int | 0 <= x < TWO_TO_THE_8 + newtype uint8 = x: int | 0 <= x < TWO_TO_THE_8 newtype uint16 = x: int | 0 <= x < TWO_TO_THE_16 newtype uint32 = x: int | 0 <= x < TWO_TO_THE_32 newtype uint64 = x: int | 0 <= x < TWO_TO_THE_64 - newtype int8 = x: int | -0x80 <= x < 0x80 + newtype int8 = x: int | -0x80 <= x < 0x80 newtype int16 = x: int | -0x8000 <= x < 0x8000 newtype int32 = x: int | -0x8000_0000 <= x < 0x8000_0000 newtype int64 = x: int | -0x8000_0000_0000_0000 <= x < 0x8000_0000_0000_0000 @@ -35,4 +33,26 @@ module BoundedInts { newtype nat32 = x: int | 0 <= x < 0x8000_0000 newtype nat64 = x: int | 0 <= x < 0x8000_0000_0000_0000 + const UINT8_MAX: uint8 := 0xFF + const UINT16_MAX: uint16 := 0xFFFF + const UINT32_MAX: uint32 := 0xFFFF_FFFF + const UINT64_MAX: uint64 := 0xFFFF_FFFF_FFFF_FFFF + + const INT8_MIN: int8 := -0x80 + const INT8_MAX: int8 := 0x7F + const INT16_MIN: int16 := -0x8000 + const INT16_MAX: int16 := 0x7FFF + const INT32_MIN: int32 := -0x8000_0000 + const INT32_MAX: int32 := 0x7FFFFFFF + const INT64_MIN: int64 := -0x8000_0000_0000_0000 + const INT64_MAX: int64 := 0x7FFFFFFF_FFFFFFFF + + const NAT8_MAX: nat8 := 0x7F + const NAT16_MAX: nat16 := 0x7FFF + const NAT32_MAX: nat32 := 0x7FFFFFFF + const NAT64_MAX: nat64 := 0x7FFFFFFF_FFFFFFFF + + type byte = uint8 + type bytes = seq + newtype opt_byte = c: int | -1 <= c < TWO_TO_THE_8 } diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy new file mode 100644 index 00000000..12a7fe7e --- /dev/null +++ b/src/JSON/Deserializer.dfy @@ -0,0 +1,431 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "Grammar.dfy" +include "Parsers.dfy" + +module {:options "-functionSyntax:4"} JSON.Deserializer { + module Core { + import opened BoundedInts + import opened Wrappers + + import opened Cursors + import opened Parsers + import opened Grammar + + datatype JSONError = + | LeadingSeparator // "Separator not allowed before first item" + | UnterminatedSequence // "Unterminated sequence." + | EmptyNumber // "Number must contain at least one digit" + | ExpectingEOF // "Expecting EOF" + { + function ToString() : string { + match this + case LeadingSeparator => "Separator not allowed before first item" + case UnterminatedSequence => "Unterminated sequence." + case EmptyNumber => "Number must contain at least one digit" + case ExpectingEOF => "Expecting EOF" + } + } + type ParseError = CursorError + type ParseResult<+T> = SplitResult + type Parser<+T> = Parsers.Parser + type SubParser<+T> = Parsers.SubParser + + // FIXME make more things opaque + + function {:opaque} Get(ps: FreshCursor, err: JSONError): (pp: ParseResult) + ensures pp.Success? ==> pp.value.t.Length() == 1 + ensures pp.Success? ==> pp.value.ps.StrictlySplitFrom?(ps) // FIXME splitfrom should be on pp + { + var ps :- ps.Get(err); + Success(ps.Split()) + } + + function {:opaque} WS(ps: FreshCursor): (pp: Split) + ensures pp.ps.SplitFrom?(ps) + { + ps.SkipWhile(Blank?).Split() + } + + function Structural(ps: FreshCursor, parser: Parser) + : (ppr: ParseResult>) + requires forall ps :: parser.fn.requires(ps) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var SP(before, ps) := WS(ps); + var SP(val, ps) :- parser.fn(ps); + var SP(after, ps) := WS(ps); + Success(SP(Grammar.Structural(before, val, after), ps)) + } + } + + abstract module SequenceParams { + import opened BoundedInts + + import opened Grammar + import opened Cursors + import opened Core + + const OPEN: byte + const CLOSE: byte + const SEPARATOR: byte + + type TItem + function Item(ps: FreshCursor, json: SubParser) + : (ppr: ParseResult) + requires ps.StrictlySplitFrom?(json.ps) + decreases ps.Length() + ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + } + + abstract module Sequences { + import opened Wrappers + import opened BoundedInts + import opened Params: SequenceParams + + import opened Vs = Views.Core + import opened Grammar + import opened Cursors + import Parsers + import opened Core + + type jopen = v: View | v.Byte?(OPEN) witness View.OfBytes([OPEN]) + type jclose = v: View | v.Byte?(CLOSE) witness View.OfBytes([CLOSE]) + type jsep = v: View | v.Byte?(SEPARATOR) witness View.OfBytes([SEPARATOR]) + type TSeq = Bracketed + + function {:tailrecursion} SeparatorPrefixedItems( + ps: FreshCursor, json: SubParser, + open: Structural, + items: PrefixedSequence + ): (ppr: ParseResult) + requires ps.StrictlySplitFrom?(json.ps) + requires NoLeadingPrefix(items) + decreases ps.Length() + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var SP(sep, ps) :- Core.Structural(ps, Parsers.Parser(ps => Get(ps, UnterminatedSequence))); + var s0 := sep.t.At(0); + if s0 == CLOSE then + Success(SP(Grammar.Bracketed(open, items, sep), ps)) + else if s0 == SEPARATOR then + :- Need(|items| > 0, OtherError(LeadingSeparator)); + var SP(item, ps) :- Item(ps, json); + var items := items + [Prefixed(NonEmpty(sep), item)]; + ghost var ppr' := SeparatorPrefixedItems(ps, json, open, items); // DISCUSS: Why is this needed? + assert ppr'.Success? ==> ppr'.value.ps.StrictlySplitFrom?(ps); + SeparatorPrefixedItems(ps, json, open, items) + else + Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0 as opt_byte)) + } + + function Open(ps: FreshCursor) + : (ppr: ParseResult) + { + var ps :- ps.AssertByte(OPEN); + Success(ps.Split()) + } + + function Close(ps: FreshCursor) + : (ppr: ParseResult) + { + var ps :- ps.AssertByte(CLOSE); + Success(ps.Split()) + } + + function Items(ps: FreshCursor, json: SubParser, open: Structural) + : (ppr: ParseResult) + requires ps.StrictlySplitFrom?(json.ps) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + if ps.Peek() == CLOSE as opt_byte then + var SP(close, ps) :- Core.Structural(ps, Parsers.Parser(Close)); + Success(SP(Grammar.Bracketed(open, [], close), ps)) + else + var SP(item, ps) :- Item(ps, json); + var items := [Prefixed(Empty(), item)]; + ghost var ppr' := SeparatorPrefixedItems(ps, json, open, items); // DISCUSS: Why is this needed? + assert ppr'.Success? ==> ppr'.value.ps.StrictlySplitFrom?(ps); + SeparatorPrefixedItems(ps, json, open, items) + } + + lemma Valid(x: TSeq) + ensures x.l.t.Byte?(OPEN) + ensures x.r.t.Byte?(CLOSE) + ensures NoLeadingPrefix(x.data) + ensures forall pf | pf in x.data :: + pf.before.NonEmpty? ==> pf.before.t.t.Byte?(SEPARATOR) + { // DISCUSS: Why is this lemma needed? Why does it require a body? + var xlt: jopen := x.l.t; + var xrt: jclose := x.r.t; + forall pf | pf in x.data + ensures pf.before.NonEmpty? ==> pf.before.t.t.Byte?(SEPARATOR) + { + if pf.before.NonEmpty? { + var xtt := pf.before.t.t; + } + } + } + + function Bracketed(ps: FreshCursor, json: SubParser) + : (ppr: ParseResult) + requires ps.SplitFrom?(json.ps) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + // ensures ppr.Success? ==> Valid?(ppr.value.t) + { + var SP(open, ps) :- Core.Structural(ps, Parsers.Parser(Open)); + ghost var ppr := Items(ps, json, open); // DISCUSS: Why is this needed? + assert ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps); + Items(ps, json, open) + } + } + + module Top { + import opened Wrappers + + import opened Vs = Views.Core + import opened Grammar + import opened Cursors + import opened Core + import Values + + function JSON(ps: FreshCursor) : (ppr: ParseResult) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + Core.Structural(ps, Parsers.Parser(Values.Value)) + } + + function Text(v: View) : (pr: Result) { + var SP(text, ps) :- JSON(Cursor.OfView(v)); + :- Need(ps.EOF?, OtherError(ExpectingEOF)); + Success(text) + } + } + + module Values { + import opened BoundedInts + import opened Wrappers + + import opened Grammar + import opened Cursors + import opened Core + + import Strings + import Numbers + import Objects + import Arrays + import Constants + + function Value(ps: FreshCursor) : (ppr: ParseResult) + decreases ps.Length(), 1 + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var c := ps.Peek(); + if c == '{' as opt_byte then + var SP(obj, ps) :- Objects.Bracketed(ps, ValueParser(ps)); + Objects.Valid(obj); + Success(SP(Grammar.Object(obj), ps)) + else if c == '[' as opt_byte then + var SP(arr, ps) :- Arrays.Bracketed(ps, ValueParser(ps)); + Arrays.Valid(arr); + Success(SP(Grammar.Array(arr), ps)) + else if c == '\"' as opt_byte then + var SP(str, ps) :- Strings.String(ps); + Success(SP(Grammar.String(str), ps)) + else if c == 't' as opt_byte then + var SP(cst, ps) :- Constants.Constant(ps, TRUE); + Success(SP(Grammar.Bool(cst), ps)) + else if c == 'f' as opt_byte then + var SP(cst, ps) :- Constants.Constant(ps, FALSE); + Success(SP(Grammar.Bool(cst), ps)) + else if c == 'n' as opt_byte then + var SP(cst, ps) :- Constants.Constant(ps, NULL); + Success(SP(Grammar.Null(cst), ps)) + else + Numbers.Number(ps) + } + + function ValueParser(ps: FreshCursor) : (p: SubParser) + decreases ps.Length(), 0 + ensures ps.SplitFrom?(p.ps) + { + var pre := (ps': FreshCursor) => ps'.Length() < ps.Length(); + var fn := (ps': FreshCursor) requires pre(ps') => Value(ps'); + Parsers.SubParser(ps, pre, fn) + } + } + + module Constants { + import opened BoundedInts + import opened Wrappers + + import opened Grammar + import opened Core + import opened Cursors + + function Constant(ps: FreshCursor, expected: bytes) : (ppr: ParseResult) + requires |expected| < TWO_TO_THE_32 + ensures ppr.Success? ==> ppr.value.t.Bytes() == expected + { + var ps :- ps.AssertBytes(expected); + Success(ps.Split()) + } + } + + module Strings { + import opened Wrappers + + import opened Grammar + import opened Cursors + import opened LC = Lexers.Core + import opened Lexers.Strings + import opened Parsers + import opened Core + + function String(ps: FreshCursor): (ppr: ParseResult) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var ps :- ps.AssertChar('\"'); + var ps :- ps.SkipWhileLexer(StringBody, Partial(StringBodyLexerStart)); + var ps :- ps.AssertChar('\"'); + Success(ps.Split()) + } + } + + module Numbers { + import opened BoundedInts + import opened Wrappers + + import opened Grammar + import opened Cursors + import opened Core + + function Digits(ps: FreshCursor) + : (pp: Split) + ensures pp.ps.SplitFrom?(ps) + { + ps.SkipWhile(Digit?).Split() + } + + function NonEmptyDigits(ps: FreshCursor) : (ppr: ParseResult) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var pp := Digits(ps); + :- Need(!pp.t.Empty?, OtherError(EmptyNumber)); + Success(pp) + } + + function Minus(ps: FreshCursor) : (pp: Split) + ensures pp.ps.SplitFrom?(ps) + { + ps.SkipIf(c => c == '-' as byte).Split() + } + + function Sign(ps: FreshCursor) : (pp: Split) + ensures pp.ps.SplitFrom?(ps) + { + ps.SkipIf(c => c == '-' as byte || c == '+' as byte).Split() + } + + function TrimmedInt(ps: FreshCursor) : (ppr: ParseResult) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + { + var pp := ps.SkipIf(c => c == '0' as byte).Split(); + if pp.t.Empty? then NonEmptyDigits(ps) + else Success(pp) + } + + function Exp(ps: FreshCursor) : (ppr: ParseResult>) + ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + { + var SP(e, ps) := + ps.SkipIf(c => c == 'e' as byte || c == 'E' as byte).Split(); + if e.Empty? then + Success(SP(Empty(), ps)) + else + assert e.Char?('e') || e.Char?('E'); + var SP(sign, ps) := Sign(ps); + var SP(num, ps) :- NonEmptyDigits(ps); + Success(SP(NonEmpty(JExp(e, sign, num)), ps)) + } + + function Frac(ps: FreshCursor) : (ppr: ParseResult>) + ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + { + var SP(period, ps) := + ps.SkipIf(c => c == '.' as byte).Split(); + if period.Empty? then + Success(SP(Empty(), ps)) + else + var SP(num, ps) :- NonEmptyDigits(ps); + Success(SP(NonEmpty(JFrac(period, num)), ps)) + } + + function Number(ps: FreshCursor) : (ppr: ParseResult) + ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + ensures ppr.Success? ==> ppr.value.t.Number? + { + var SP(minus, ps) := Minus(ps); + var SP(num, ps) :- TrimmedInt(ps); + var SP(frac, ps) :- Frac(ps); + var SP(exp, ps) :- Exp(ps); + Success(SP(Grammar.Number(minus, num, frac, exp), ps)) + } + } + + module ArrayParams refines SequenceParams { + import opened Strings + import opened Wrappers + + type TItem = Value + + const OPEN := '[' as byte + const CLOSE := ']' as byte + const SEPARATOR: byte := ',' as byte + + function Item(ps: FreshCursor, json: SubParser) + : (ppr: ParseResult) + { + assert ps.StrictlySplitFrom?(json.ps); + assert json.Valid?(); + assert json.Valid?(); + assert json.fn.requires(ps); + json.fn(ps) + } + } + + module Arrays refines Sequences { + import opened Params = ArrayParams + } + + module ObjectParams refines SequenceParams { + import opened Wrappers + import Strings + + type TItem = jkv + + const OPEN := '{' as byte + const CLOSE := '}' as byte + const SEPARATOR: byte := ',' as byte + + function Colon(ps: FreshCursor) + : (ppr: ParseResult) + { + var ps :- ps.AssertChar(':'); + Success(ps.Split()) + } + + function Item(ps: FreshCursor, json: SubParser) + : (ppr: ParseResult) + { + var SP(k, ps) :- Strings.String(ps); + var SP(colon, ps) :- Core.Structural(ps, Parsers.Parser(Colon)); + var SP(v, ps) :- json.fn(ps); + Success(SP(KV(k, colon, v), ps)) + } + } + + module Objects refines Sequences { + import opened Params = ObjectParams + } +} diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy new file mode 100644 index 00000000..71ddd6e3 --- /dev/null +++ b/src/JSON/Grammar.dfy @@ -0,0 +1,165 @@ +include "../BoundedInts.dfy" +include "Views.dfy" + +module {:options "-functionSyntax:4"} JSON.Grammar { + import opened BoundedInts + import opened Views.Core + + type jchar = v: View | v.Length() == 1 witness View.OfBytes(['b' as byte]) + type jperiod = v: View | v.Char?('.') witness View.OfBytes(['.' as byte]) + type je = v: View | v.Char?('e') || v.Char?('E') witness View.OfBytes(['e' as byte]) + type jcolon = v: View | v.Char?(':') witness View.OfBytes([':' as byte]) + type jcomma = v: View | v.Char?(',') witness View.OfBytes([',' as byte]) + type jlbrace = v: View | v.Char?('{') witness View.OfBytes(['{' as byte]) + type jrbrace = v: View | v.Char?('}') witness View.OfBytes(['}' as byte]) + type jlbracket = v: View | v.Char?('[') witness View.OfBytes(['[' as byte]) + type jrbracket = v: View | v.Char?(']') witness View.OfBytes([']' as byte]) + type jminus = v: View | v.Char?('-') || v.Empty? witness View.OfBytes([]) + type jsign = v: View | v.Char?('-') || v.Char?('+') || v.Empty? witness View.OfBytes([]) + + predicate Blank?(b: byte) { b == 0x20 || b == 0x09 || b == 0x0A || b == 0x0D } + ghost predicate Blanks?(v: View) { forall b | b in v.Bytes() :: Blank?(b) } + type jblanks = v: View | Blanks?(v) witness View.OfBytes([]) + + datatype Structural<+T> = + Structural(before: jblanks, t: T, after: jblanks) + { + function Bytes(pt: T -> bytes): bytes { // LATER: Trait + before.Bytes() + pt(t) + after.Bytes() + } + } + + function ConcatBytes(ts: seq, pt: T --> bytes) : bytes + requires forall d | d in ts :: pt.requires(d) + { + if |ts| == 0 then [] + else pt(ts[0]) + ConcatBytes(ts[1..], pt) + } + + // FIXME remove empty + datatype Maybe<+T> = Empty() | NonEmpty(t: T) { + function Bytes(pt: T --> bytes): bytes + requires NonEmpty? ==> pt.requires(t) + { // LATER: Trait + match this + case Empty() => [] + case NonEmpty(t) => pt(t) + } + } + + datatype Prefixed<+S, +T> = + Prefixed(before: Maybe>, t: T) + { + function Bytes(psep: S -> bytes, pt: T --> bytes): bytes + requires pt.requires(t) + { // LATER: Trait + before.Bytes((s: Structural) => s.Bytes(psep)) + pt(t) + } + } + + type PrefixedSequence<+S, +D> = s: seq> | NoLeadingPrefix(s) + ghost predicate NoLeadingPrefix(s: seq>) { + forall idx | 0 <= idx < |s| :: s[idx].before.Empty? <==> idx == 0 + } + + datatype Bracketed<+L, +S, +D, +R> = + Bracketed(l: Structural, data: PrefixedSequence, r: Structural) + { + function Bytes(pl: L -> bytes, pdatum: Prefixed --> bytes, pr: R -> bytes): bytes + requires forall d | d in data :: pdatum.requires(d) + { // LATER: Trait + l.Bytes(pl) + + ConcatBytes(data, pdatum) + + r.Bytes(pr) + } + } + + function BytesOfString(str: string) : bytes + requires forall c | c in str :: c as int < 256 + { + seq(|str|, idx requires 0 <= idx < |str| => assert str[idx] in str; str[idx] as byte) + } + + const NULL: bytes := BytesOfString("null") + const TRUE: bytes := BytesOfString("true") + const FALSE: bytes := BytesOfString("false") + + ghost predicate Null?(v: View) { v.Bytes() == NULL } + ghost predicate Bool?(v: View) { v.Bytes() in {TRUE, FALSE} } + predicate Digit?(b: byte) { '0' as byte <= b <= '9' as byte } + ghost predicate Digits?(v: View) { forall b | b in v.Bytes() :: Digit?(b) } + ghost predicate Num?(v: View) { Digits?(v) && !v.Empty? } + ghost predicate Int?(v: View) { v.Char?('0') || (Num?(v) && v.At(0) != '0' as byte) } + + type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce correct escaping + type jnull = v: View | Null?(v) witness View.OfBytes(NULL) + type jbool = v: View | Bool?(v) witness View.OfBytes(TRUE) + type jdigits = v: View | Digits?(v) witness View.OfBytes([]) + type jnum = v: View | Num?(v) witness View.OfBytes(['0' as byte]) + type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) + + datatype jkv = KV(k: jstring, colon: Structural, v: Value) { + function Bytes(): bytes { + k.Bytes() + colon.Bytes((c: jcolon) => c.Bytes()) + v.Bytes() + } + } + + type jobject = Bracketed + type jarray = Bracketed + + datatype jfrac = JFrac(period: jperiod, num: jnum) { + function Bytes(): bytes { + period.Bytes() + num.Bytes() + } + } + datatype jexp = JExp(e: je, sign: jsign, num: jnum) { + function Bytes(): bytes { + e.Bytes() + sign.Bytes() + num.Bytes() + } + } + + datatype Value = + | Null(n: jnull) + | Bool(b: jbool) + | String(str: jstring) + | Number(minus: jminus, num: jnum, frac: Maybe, exp: Maybe) + | Object(obj: jobject) + | Array(arr: jarray) + { + function Bytes(): bytes { + match this { + case Null(n) => + n.Bytes() + case Bool(b) => + b.Bytes() + case String(str) => + str.Bytes() + case Number(minus, num, frac, exp) => + minus.Bytes() + num.Bytes() + + frac.Bytes((f: jfrac) => f.Bytes()) + + exp.Bytes((e: jexp) => e.Bytes()) + case Object(o) => + o.Bytes((l: jlbrace) => l.Bytes(), + (d: Prefixed) requires d in o.data => + d.Bytes((s: jcomma) => s.Bytes(), // BUG(https://github.com/dafny-lang/dafny/issues/2170) + var pt := (kv: jkv) requires kv == d.t => d.t.Bytes(); + assert pt.requires(d.t); + pt), + (r: jrbrace) => r.Bytes()) + case Array(a) => + a.Bytes((l: jlbracket) => l.Bytes(), + (d: Prefixed) requires d in a.data => + d.Bytes((s: jcomma) => s.Bytes(), + var pt := (v: Value) requires v == d.t => v.Bytes(); + assert pt.requires(d.t); + pt), + (r: jrbracket) => r.Bytes()) + } + } + } + + type JSON = Structural + function Bytes(js: JSON) : bytes { // TODO: TR + js.Bytes((v: Value) => v.Bytes()) + } +} diff --git a/src/JSON/Lexers.dfy b/src/JSON/Lexers.dfy new file mode 100644 index 00000000..4faf4cc7 --- /dev/null +++ b/src/JSON/Lexers.dfy @@ -0,0 +1,51 @@ +include "../Wrappers.dfy" +include "../BoundedInts.dfy" + +module {:options "-functionSyntax:4"} Lexers { + module Core { + import opened Wrappers + import opened BoundedInts + + datatype LexerState<+T, +R> = Accept | Reject(err: R) | Partial(st: T) + + type Lexer = (LexerState, opt_byte) -> LexerState + } + + module Strings { + import opened Core + import opened BoundedInts + + type StringBodyLexerState = /* escaped: */ bool + const StringBodyLexerStart: StringBodyLexerState := false; + + function StringBody(st: LexerState, byte: opt_byte) + : LexerState + { + match st + case Partial(escaped) => + if byte == '\\' as opt_byte then Partial(!escaped) + else if byte == '\"' as opt_byte && !escaped then Accept + else Partial(false) + case _ => st + } + + datatype StringLexerState = Start | Body(escaped: bool) | End + const StringLexerStart: StringLexerState := Start; + + function String(st: LexerState, byte: opt_byte) + : LexerState + { + match st + case Partial(Start()) => + if byte == '\"' as opt_byte then Partial(Body(false)) + else Reject("String must start with double quote") + case Partial(End()) => + Accept + case Partial(Body(escaped)) => + if byte == '\\' as opt_byte then Partial(Body(!escaped)) + else if byte == '\"' as opt_byte && !escaped then Partial(End) + else Partial(Body(false)) + case _ => st + } + } +} diff --git a/src/JSON/Parsers.dfy b/src/JSON/Parsers.dfy new file mode 100644 index 00000000..28730380 --- /dev/null +++ b/src/JSON/Parsers.dfy @@ -0,0 +1,57 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "Cursors.dfy" + +module {:options "-functionSyntax:4"} Parsers { + import opened BoundedInts + import opened Wrappers + + import opened Views.Core + import opened Cursors + + type SplitResult<+T, +R> = CursorResult, R> + + type Parser<+T, +R> = p: Parser_ | p.Valid?() + // BUG(https://github.com/dafny-lang/dafny/issues/2103) + witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + datatype Parser_<+T, +R> = Parser(fn: FreshCursor -> SplitResult) { + ghost predicate Valid?() { + forall ps': FreshCursor :: fn(ps').Success? ==> fn(ps').value.ps.StrictlySplitFrom?(ps') + } + } + + function {:opaque} ParserWitness(): (p: Parser_) + ensures p.Valid?() + { + Parser(_ => Failure(EOF)) + } + + // BUG(): It would be much nicer if `SubParser` was a special case of + // `Parser`, but that would require making `fn` in parser a partial + // function `-->`. The problem with that is that we would then have to + // restrict the `Valid?` clause of `Parser` on `fn.requires()`, thus + // making it unprovable in the `SubParser` case (`fn` for subparsers is + // typically a lambda, and the `requires` of lambdas are essentially + // uninformative/opaque). + datatype SubParser_<+T, +R> = SubParser( + ghost ps: Cursor, + ghost pre: FreshCursor -> bool, + fn: FreshCursor --> SplitResult) + { + ghost predicate Valid?() { + && (forall ps': FreshCursor | pre(ps') :: fn.requires(ps')) + && (forall ps': FreshCursor | ps'.StrictlySplitFrom?(ps) :: pre(ps')) + && (forall ps': FreshCursor | pre(ps') :: fn(ps').Success? ==> fn(ps').value.ps.StrictlySplitFrom?(ps')) + } + } + type SubParser<+T, +R> = p: SubParser_ | p.Valid?() + witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + + function {:opaque} SubParserWitness(): (subp: SubParser_) + ensures subp.Valid?() + { + SubParser(Cursor([], 0, 0, 0), + (ps: FreshCursor) => false, + (ps: FreshCursor) => Failure(EOF)) + } +} diff --git a/src/JSON/Stacks.dfy b/src/JSON/Stacks.dfy new file mode 100644 index 00000000..0ba59a7f --- /dev/null +++ b/src/JSON/Stacks.dfy @@ -0,0 +1,133 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" + +module {:options "-functionSyntax:4"} Stacks { + import opened BoundedInts + import opened Wrappers + + datatype Error = OutOfMemory + + class Stack { + ghost var Repr : seq + + const a: A + var size: uint32 + var capacity: uint32 + var data: array + + const MAX_CAPACITY: uint32 := UINT32_MAX + const MAX_CAPACITY_BEFORE_DOUBLING: uint32 := UINT32_MAX / 2 + + ghost predicate Valid?() + reads this, data + { + && capacity != 0 + && data.Length == capacity as int + && size <= capacity + && Repr == data[..size] + } + + constructor(a0: A, initial_capacity: uint32 := 8) + requires initial_capacity > 0 + ensures Valid?() + { + a := a0; + Repr := []; + size := 0; + capacity := initial_capacity; + data := new A[initial_capacity](_ => a0); + } + + method At(idx: uint32) returns (a: A) + requires idx < size + requires Valid?() + ensures a == data[idx] == Repr[idx] + { + return data[idx]; + } + + method Blit(new_data: array, count: uint32) + requires count as int <= new_data.Length + requires count <= capacity + requires data.Length == capacity as int + modifies data + ensures data[..count] == new_data[..count] + ensures data[count..] == old(data[count..]) + { + for idx: uint32 := 0 to count + invariant idx <= capacity + invariant data.Length == capacity as int + invariant data[..idx] == new_data[..idx] + invariant data[count..] == old(data[count..]) + { + data[idx] := new_data[idx]; + } + } + + method Realloc(new_capacity: uint32) + requires Valid?() + requires new_capacity > capacity + modifies this, data + ensures Valid?() + ensures Repr == old(Repr) + ensures size == old(size) + ensures capacity == new_capacity + ensures fresh(data) + { + var old_data, old_capacity := data, capacity; + data, capacity := new A[new_capacity](_ => a), new_capacity; + Blit(old_data, old_capacity); + } + + method PopFast(a: A) + requires Valid?() + requires size > 0 + modifies this, data + ensures Valid?() + ensures size == old(size) - 1 + ensures Repr == old(Repr[..|Repr| - 1]) + { + size := size - 1; + Repr := Repr[..|Repr| - 1]; + } + + method PushFast(a: A) + requires Valid?() + requires size < capacity + modifies this, data + ensures Valid?() + ensures size == old(size) + 1 + ensures Repr == old(Repr) + [a] + { + data[size] := a; + size := size + 1; + Repr := Repr + [a]; + } + + method Push(a: A) returns (o: Outcome) + requires Valid?() + modifies this, data + ensures Valid?() + ensures o.Fail? ==> + && unchanged(this) + && unchanged(data) + ensures o.Pass? ==> + && old(size) < MAX_CAPACITY + && size == old(size) + 1 + && Repr == old(Repr) + [a] + { + if size == capacity { + if capacity < MAX_CAPACITY_BEFORE_DOUBLING { + Realloc(2 * capacity); + } else { + if capacity == MAX_CAPACITY { + return Fail(OutOfMemory); + } + Realloc(MAX_CAPACITY); + } + } + PushFast(a); + return Pass; + } + } +} diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy new file mode 100644 index 00000000..acb1828d --- /dev/null +++ b/src/JSON/Tests.dfy @@ -0,0 +1,56 @@ +include "Grammar.dfy" +include "Deserializer.dfy" +include "../Collections/Sequences/Seq.dfy" + +import opened BoundedInts + +import opened Vs = Views.Core + +import opened JSON.Grammar +import opened JSON.Deserializer + +function method bytes_of_ascii(s: string) : bytes { + Seq.Map((c: char) requires c in s => if c as int < 256 then c as byte else 0 as byte, s) +} + +function method ascii_of_bytes(bs: bytes) : string { + Seq.Map((b: byte) requires b in bs => b as char, bs) +} + +const VECTORS := [ + "true", + "false", + "null", + "\"\"", + "\"string\"", + "[\"A\"]", + "-123.456e-18", + "[]", + "[1]", + "[1, 2]", + "{}", + "{ \"a\": 1 }", + "{ \"a\": \"b\" }", + "{ \"some\" : \"string\", \"and\": [ \"a number\", -123.456e-18 ] }", + "[true, false , null, { \"some\" : \"string\", \"and\": [ \"a number\", -123.456e-18 ] } ] " +]; + +method Main() { + for i := 0 to |VECTORS| { + var input := VECTORS[i]; + expect |input| < 0x1_00000000; + + print input, "\n"; + var bytes := bytes_of_ascii(input); + match Deserializer.Top.Text(View.OfBytes(bytes)) { + case Failure(msg) => + print "Parse error: " + msg.ToString((e: Deserializer.Core.JSONError) => e.ToString()) + "\n"; + expect false; + case Success(js) => + var bytes' := Grammar.Bytes(js); + print "=> " + ascii_of_bytes(bytes') + "\n"; + expect bytes' == bytes; + } + print "\n"; + } +} diff --git a/src/JSON/Views.dfy b/src/JSON/Views.dfy new file mode 100644 index 00000000..e03f9b0c --- /dev/null +++ b/src/JSON/Views.dfy @@ -0,0 +1,80 @@ +include "../BoundedInts.dfy" + +module {:options "-functionSyntax:4"} Views.Core { + import opened BoundedInts + + type View = v: View_ | v.Valid? witness View([], 0, 0) + datatype View_ = View(s: bytes, beg: uint32, end: uint32) { + ghost const Valid?: bool := + 0 <= beg as int <= end as int <= |s| < TWO_TO_THE_32; + + const Empty? := + beg == end + + function Length(): uint32 requires Valid? { + end - beg + } + + function Bytes(): bytes requires Valid? { + s[beg..end] + } + + static function OfBytes(bs: bytes) : (v: View) + requires |bs| < TWO_TO_THE_32 + ensures v.Bytes() == bs + { + View(bs, 0 as uint32, |bs| as uint32) + } + + ghost predicate Byte?(c: byte) + requires Valid? + { + Bytes() == [c] + } + + ghost predicate Char?(c: char) + requires Valid? + requires c as int < 256 + { + Byte?(c as byte) + } + + ghost predicate ValidIndex?(idx: uint32) { + beg as int + idx as int < end as int + } + + function At(idx: uint32) : byte + requires Valid? + requires ValidIndex?(idx) + { + s[beg + idx] + } + + method Blit(bs: array, start: uint32 := 0) + requires Valid? + requires start as int + Length() as int <= bs.Length + requires start as int + Length() as int < TWO_TO_THE_32 + modifies bs + ensures bs[start..start + Length()] == Bytes() + ensures bs[start + Length()..] == old(bs[start + Length()..]) + { + for idx := 0 to Length() + invariant bs[start..start + idx] == Bytes()[..idx] + invariant bs[start + Length()..] == old(bs[start + Length()..]) + { + bs[start + idx] := s[beg + idx]; + } + } + } + + predicate Adjacent(lv: View, rv: View) { + lv.s == rv.s && + lv.end == rv.beg + } + + function Merge(lv: View, rv: View) : View + requires Adjacent(lv, rv) + { + lv.(end := rv.end) + } +} From f611179829558bf1672a85e312e1a91493db2e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Sun, 29 May 2022 13:44:38 -0700 Subject: [PATCH 02/84] json: Add an efficient serializer --- src/JSON/Cursors.dfy | 269 ++++++++++++++++++++++++++++++++++++ src/JSON/Serializer.dfy | 272 +++++++++++++++++++++++++++++++++++++ src/JSON/Views.Writers.dfy | 113 +++++++++++++++ 3 files changed, 654 insertions(+) create mode 100644 src/JSON/Cursors.dfy create mode 100644 src/JSON/Serializer.dfy create mode 100644 src/JSON/Views.Writers.dfy diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy new file mode 100644 index 00000000..ca4260f4 --- /dev/null +++ b/src/JSON/Cursors.dfy @@ -0,0 +1,269 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "Views.dfy" +include "Lexers.dfy" + +module {:options "-functionSyntax:4"} Cursors { + import opened BoundedInts + import opened Wrappers + + import opened Vs = Views.Core + import opened Lx = Lexers.Core + + datatype Split<+T> = SP(t: T, ps: FreshCursor) + + // LATER: Make this a newtype and put members here instead of requiring `Valid?` everywhere + type Cursor = ps: Cursor_ | ps.Valid? + witness Cursor([], 0, 0, 0) + type FreshCursor = ps: Cursor | ps.BOF? + witness Cursor([], 0, 0, 0) + + datatype CursorError<+R> = + | EOF + | ExpectingByte(expected: byte, b: opt_byte) + | ExpectingAnyByte(expected_sq: seq, b: opt_byte) + | OtherError(err: R) + { + function ToString(pr: R -> string) : string { + match this + case EOF => "Reached EOF" + case ExpectingByte(b0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + "Expecting '" + [b0 as char] + "', read " + c + case ExpectingAnyByte(bs0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); + "Expecting one of '" + c0s + "', read " + c + case OtherError(err) => pr(err) + } + } + type CursorResult<+T, +R> = Result> + + datatype Cursor_ = Cursor(s: bytes, beg: uint32, point: uint32, end: uint32) { + ghost const Valid?: bool := + 0 <= beg as int <= point as int <= end as int <= |s| < TWO_TO_THE_32; + + const BOF? := + point == beg + + const EOF? := + point == end + + static function OfView(v: View) : Cursor { + Cursor(v.s, v.beg, v.beg, v.end) + } + + function Bytes() : bytes + requires Valid? + { + s[beg..end] + } + + ghost function StrictlyAdvancedFrom?(other: Cursor): (b: bool) + requires Valid? + ensures b ==> + SuffixLength() < other.SuffixLength() + ensures b ==> + beg == other.beg && end == other.end ==> + forall idx | beg <= idx < point :: s[idx] == other.s[idx] + { + && s == other.s + && beg == other.beg + && end == other.end + && point > other.point + } + + ghost predicate AdvancedFrom?(other: Cursor) + requires Valid? + { + || this == other + || StrictlyAdvancedFrom?(other) + } + + ghost predicate StrictSuffixOf?(other: Cursor) + requires Valid? + ensures StrictSuffixOf?(other) ==> + Length() < other.Length() + { + && s == other.s + && beg > other.beg + && end == other.end + } + + ghost predicate SuffixOf?(other: Cursor) + requires Valid? + { + || this == other + || StrictSuffixOf?(other) + } + + ghost predicate StrictlySplitFrom?(other: Cursor) + requires Valid? + { + && BOF? + && StrictSuffixOf?(other) + } + + ghost predicate SplitFrom?(other: Cursor) + requires Valid? + { + || this == other + || StrictlySplitFrom?(other) + } + + function Prefix() : View requires Valid? { + View(s, beg, point) + } + + function Suffix() : Cursor requires Valid? { + this.(beg := point) + } + + function Split() : Split requires Valid? { + SP(this.Prefix(), this.Suffix()) + } + + function PrefixLength() : uint32 requires Valid? { + point - beg + } + + function SuffixLength() : uint32 requires Valid? { + end - point + } + + function Length() : uint32 requires Valid? { + end - beg + } + + ghost predicate ValidIndex?(idx: uint32) { + beg as int + idx as int < end as int + } + + function At(idx: uint32) : byte + requires Valid? + requires ValidIndex?(idx) + { + s[beg + idx] + } + + ghost predicate ValidSuffixIndex?(idx: uint32) { + point as int + idx as int < end as int + } + + function SuffixAt(idx: uint32) : byte + requires Valid? + requires ValidSuffixIndex?(idx) + { + s[point + idx] + } + + function Peek(): (r: opt_byte) + requires Valid? + ensures r < 0 <==> EOF? + { + if EOF? then -1 + else SuffixAt(0) as opt_byte + } + + function LookingAt(c: char): (b: bool) + requires Valid? + requires c as int < 256 + ensures b <==> !EOF? && SuffixAt(0) == c as byte + { + Peek() == c as opt_byte + } + + function Skip(n: uint32): (ps: Cursor) + requires Valid? + requires point as int + n as int <= end as int + ensures n == 0 ==> ps == this + ensures n > 0 ==> ps.StrictlyAdvancedFrom?(this) + { + this.(point := point + n) + } + + function Unskip(n: uint32): Cursor + requires Valid? + requires beg as int <= point as int - n as int + { + this.(point := point - n) + } + + function Get(err: R): (ppr: CursorResult) + requires Valid? + ensures ppr.Success? ==> ppr.value.StrictlyAdvancedFrom?(this) + { + if EOF? then Failure(OtherError(err)) + else Success(Skip(1)) + } + + function AssertByte(b: byte): (pr: CursorResult) + requires Valid? + ensures pr.Success? ==> !EOF? + ensures pr.Success? ==> s[point] == b + ensures pr.Success? ==> pr.value.StrictlyAdvancedFrom?(this) + { + var nxt := Peek(); + if nxt == b as opt_byte then Success(Skip(1)) + else Failure(ExpectingByte(b, nxt)) + } + + function {:tailrecursion} AssertBytes(bs: bytes, offset: uint32 := 0): (pr: CursorResult) + requires Valid? + requires |bs| < TWO_TO_THE_32 + requires offset <= |bs| as uint32 + requires forall b | b in bs :: b as int < 256 + decreases SuffixLength() + ensures pr.Success? ==> pr.value.AdvancedFrom?(this) + ensures pr.Success? && offset < |bs| as uint32 ==> pr.value.StrictlyAdvancedFrom?(this) + ensures pr.Success? ==> s[point..pr.value.point] == bs[offset..] + { + if offset == |bs| as uint32 then Success(this) + else + var ps :- AssertByte(bs[offset] as byte); + ps.AssertBytes(bs, offset + 1) + } + + function AssertChar(c0: char): (pr: CursorResult) + requires Valid? + requires c0 as int < 256 + ensures pr.Success? ==> pr.value.StrictlyAdvancedFrom?(this) + { + AssertByte(c0 as byte) + } + + function SkipIf(p: byte -> bool): (ps: Cursor) + requires Valid? + decreases SuffixLength() + ensures ps.AdvancedFrom?(this) + ensures !EOF? && p(SuffixAt(0)) ==> ps.StrictlyAdvancedFrom?(this) + { + if EOF? || !p(SuffixAt(0)) then this + else Skip(1) + } + + function SkipWhile(p: byte -> bool): (ps: Cursor) + requires Valid? + decreases SuffixLength() + ensures ps.AdvancedFrom?(this) + ensures forall idx | point <= idx < ps.point :: p(ps.s[idx]) + { + if EOF? || !p(SuffixAt(0)) then this + else Skip(1).SkipWhile(p) + } + + function SkipWhileLexer(step: Lexer, st: LexerState) + : (pr: CursorResult) + requires Valid? + decreases SuffixLength() + ensures pr.Success? ==> pr.value.AdvancedFrom?(this) + { + match step(st, Peek()) + case Accept => Success(this) + case Reject(err) => Failure(OtherError(err)) + case partial => + if EOF? then Failure(EOF) + else Skip(1).SkipWhileLexer(step, partial) + } + } +} diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy new file mode 100644 index 00000000..7dd54d7d --- /dev/null +++ b/src/JSON/Serializer.dfy @@ -0,0 +1,272 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "Grammar.dfy" +include "Views.Writers.dfy" +include "Stacks.dfy" + +module {:options "-functionSyntax:4"} JSON.Serializer { + import opened BoundedInts + import opened Wrappers + + import opened Grammar + import opened Views.Writers + import opened Vs = Views.Core // DISCUSS: Module naming convention? + + datatype Error = OutOfMemory + + method Serialize(js: JSON) returns (bsr: Result, Error>) + ensures bsr.Success? ==> bsr.value[..] == Bytes(js) + { + var writer := JSON(js); + :- Need(writer.Unsaturated?, OutOfMemory); + var bs := writer.ToArray(); + return Success(bs); + } + + function {:opaque} JSON(js: JSON, writer: Writer := Writer.Empty) : (wr: Writer) + ensures wr.Bytes() == writer.Bytes() + Bytes(js) + { + // DISCUSS: This doesn't work: + // writer + // .Append(js.before) + // .Then(wr => Value(js.t, wr)) + // .Append(js.after) + // … but this does: + var wr := writer; + var wr := wr.Append(js.before); + var wr := Value(js.t, wr); + var wr := wr.Append(js.after); + wr + } + + function {:opaque} Value(v: Value, writer: Writer) : (wr: Writer) + decreases v, 4 + ensures wr.Bytes() == writer.Bytes() + v.Bytes() + { + match v + case Null(n) => + writer.Append(n) + case Bool(b) => + writer.Append(b) + case String(str) => + writer.Append(str) + case Number(minus, num, frac, exp) => + Number(v, writer) + case Object(obj) => + Object(v, obj, writer) + case Array(arr) => + Array(v, arr, writer) + } + + function {:opaque} Number(v: Value, writer: Writer) : (wr: Writer) + requires v.Number? + decreases v, 0 + ensures wr.Bytes() == writer.Bytes() + v.Bytes() + { + var writer := writer.Append(v.minus).Append(v.num); + var writer := if v.frac.NonEmpty? then + writer.Append(v.frac.t.period).Append(v.frac.t.num) + else writer; + var writer := if v.exp.NonEmpty? then + writer.Append(v.exp.t.e).Append(v.exp.t.sign).Append(v.exp.t.num) + else writer; + writer + } + + // DISCUSS: Can't be opaque, due to the lambda + function Structural(st: Structural, writer: Writer) : (wr: Writer) + // FIXME ensures writer.Length() + |st.Bytes((v: View) => v.Bytes())| < TWO_TO_THE_32 ==> rwr.Success? + ensures wr.Bytes() == writer.Bytes() + st.Bytes((v: View) => v.Bytes()) + { + writer.Append(st.before).Append(st.t).Append(st.after) + } + + lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS + ensures forall + pl0: L -> bytes, pd0: Prefixed --> bytes, pr0: R -> bytes, + pl1: L -> bytes, pd1: Prefixed --> bytes, pr1: R -> bytes + | && (forall d | d in bracketed.data :: pd0.requires(d)) + && (forall d | d in bracketed.data :: pd1.requires(d)) + && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + && (forall l :: pl0(l) == pl1(l)) + && (forall r :: pr0(r) == pr1(r)) + :: bracketed.Bytes(pl0, pd0, pr0) == bracketed.Bytes(pl1, pd1, pr1) + { + forall pl0: L -> bytes, pd0: Prefixed --> bytes, pr0: R -> bytes, + pl1: L -> bytes, pd1: Prefixed --> bytes, pr1: R -> bytes + | && (forall d | d in bracketed.data :: pd0.requires(d)) + && (forall d | d in bracketed.data :: pd1.requires(d)) + && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + && (forall l :: pl0(l) == pl1(l)) + && (forall r :: pr0(r) == pr1(r)) + { + calc { + bracketed.Bytes(pl0, pd0, pr0); + { ConcatBytes_Morphism(bracketed.data, pd0, pd1); } + bracketed.Bytes(pl1, pd1, pr1); + } + } + } + + lemma {:induction ts} ConcatBytes_Morphism(ts: seq, pt0: T --> bytes, pt1: T --> bytes) + requires forall d | d in ts :: pt0.requires(d) + requires forall d | d in ts :: pt1.requires(d) + requires forall d | d in ts :: pt0(d) == pt1(d) + ensures ConcatBytes(ts, pt0) == ConcatBytes(ts, pt1) + {} + + lemma {:induction ts0} ConcatBytes_Linear(ts0: seq, ts1: seq, pt: T --> bytes) + requires forall d | d in ts0 :: pt.requires(d) + requires forall d | d in ts1 :: pt.requires(d) + ensures ConcatBytes(ts0 + ts1, pt) == ConcatBytes(ts0, pt) + ConcatBytes(ts1, pt) + { + if |ts0| == 0 { + assert [] + ts1 == ts1; + } else { + assert ts0 + ts1 == [ts0[0]] + (ts0[1..] + ts1); + } + } + + lemma {:axiom} Assume(b: bool) ensures b + + function {:opaque} Object(v: Value, obj: jobject, writer: Writer) : (wr: Writer) + requires v.Object? && obj == v.obj + decreases v, 3 + ensures wr.Bytes() == writer.Bytes() + v.Bytes() + { + var writer := Structural(obj.l, writer); + var writer := Members(v, obj, writer); + var writer := Structural(obj.r, writer); + + Bracketed_Morphism(obj); // DISCUSS + assert v.Bytes() == obj.Bytes(((l: jlbrace) => l.Bytes()), Grammar.Member, ((r: jrbrace) => r.Bytes())); + + writer + } + + function {:opaque} Array(v: Value, arr: jarray, writer: Writer) : (wr: Writer) + requires v.Array? && arr == v.arr + decreases v, 3 + ensures wr.Bytes() == writer.Bytes() + v.Bytes() + { + var writer := Structural(arr.l, writer); + var writer := Items(v, arr, writer); + var writer := Structural(arr.r, writer); + + Bracketed_Morphism(arr); // DISCUSS + assert v.Bytes() == arr.Bytes(((l: jlbracket) => l.Bytes()), Grammar.Item, ((r: jrbracket) => r.Bytes())); + + writer + } + + function {:opaque} Members(ghost v: Value, obj: jobject, writer: Writer) : (wr: Writer) + requires obj < v + decreases v, 2 + ensures wr.Bytes() == writer.Bytes() + ConcatBytes(obj.data, Grammar.Member) + ensures wr == MembersSpec(v, obj.data, writer) + { + MembersSpec(v, obj.data, writer) + } by method { + wr := MembersImpl(v, obj, writer); + Assume(false); // FIXME + } + + ghost function MembersSpec(v: Value, members: seq, writer: Writer) : (wr: Writer) + requires forall j | 0 <= j < |members| :: members[j] < v + decreases v, 1, members + ensures wr.Bytes() == writer.Bytes() + ConcatBytes(members, Grammar.Member) + { // TR elimination doesn't work for mutually recursive methods, so this + // function is only used as a spec for Members. + if members == [] then writer + else + var writer := MembersSpec(v, members[..|members|-1], writer); + assert members == members[..|members|-1] + [members[|members|-1]]; + ConcatBytes_Linear(members[..|members|-1], [members[|members|-1]], Grammar.Member); + Member(v, members[|members|-1], writer) + } // No by method block here, because the loop invariant in the method version + // needs to call MembersSpec and the termination checker gets confused by + // that. Instead, see Members above. // DISCUSS + + method MembersImpl(ghost v: Value, obj: jobject, writer: Writer) returns (wr: Writer) + requires obj < v + decreases v, 1 + ensures wr == MembersSpec(v, obj.data, writer); + { + wr := writer; + var members := obj.data; + assert wr == MembersSpec(v, members[..0], writer); + for i := 0 to |members| // FIXME uint32 + invariant wr == MembersSpec(v, members[..i], writer) + { + assert members[..i+1][..i] == members[..i]; + wr := Member(v, members[i], wr); + } + assert members[..|members|] == members; + } + + function {:opaque} Member(ghost v: Value, m: jmember, writer: Writer) : (wr: Writer) + requires m < v + decreases v, 0 + ensures wr.Bytes() == writer.Bytes() + Grammar.Member(m) + { + var writer := if m.prefix.NonEmpty? then Structural(m.prefix.t, writer) else writer; + var writer := writer.Append(m.t.k); + var writer := Structural(m.t.colon, writer); + Value(m.t.v, writer) + } + function {:opaque} Items(ghost v: Value, arr: jarray, writer: Writer) : (wr: Writer) + requires arr < v + decreases v, 2 + ensures wr.Bytes() == writer.Bytes() + ConcatBytes(arr.data, Grammar.Item) + ensures wr == ItemsSpec(v, arr.data, writer) + { + ItemsSpec(v, arr.data, writer) + } by method { + wr := ItemsImpl(v, arr, writer); + Assume(false); // FIXME + } + + ghost function ItemsSpec(v: Value, items: seq, writer: Writer) : (wr: Writer) + requires forall j | 0 <= j < |items| :: items[j] < v + decreases v, 1, items + ensures wr.Bytes() == writer.Bytes() + ConcatBytes(items, Grammar.Item) + { // TR elimination doesn't work for mutually recursive methods, so this + // function is only used as a spec for Items. + if items == [] then writer + else + var writer := ItemsSpec(v, items[..|items|-1], writer); + assert items == items[..|items|-1] + [items[|items|-1]]; + ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], Grammar.Item); + Item(v, items[|items|-1], writer) + } // No by method block here, because the loop invariant in the method version + // needs to call ItemsSpec and the termination checker gets confused by + // that. Instead, see Items above. // DISCUSS + + method ItemsImpl(ghost v: Value, arr: jarray, writer: Writer) returns (wr: Writer) + requires arr < v + decreases v, 1 + ensures wr == ItemsSpec(v, arr.data, writer); + { + wr := writer; + var items := arr.data; + assert wr == ItemsSpec(v, items[..0], writer); + for i := 0 to |items| // FIXME uint32 + invariant wr == ItemsSpec(v, items[..i], writer) + { + assert items[..i+1][..i] == items[..i]; + wr := Item(v, items[i], wr); + } + assert items[..|items|] == items; + } + + function {:opaque} Item(ghost v: Value, m: jitem, writer: Writer) : (wr: Writer) + requires m < v + decreases v, 0 + ensures wr.Bytes() == writer.Bytes() + Grammar.Item(m) + { + var writer := if m.prefix.NonEmpty? then Structural(m.prefix.t, writer) else writer; + Value(m.t, writer) + } + + +} diff --git a/src/JSON/Views.Writers.dfy b/src/JSON/Views.Writers.dfy new file mode 100644 index 00000000..f5aa6378 --- /dev/null +++ b/src/JSON/Views.Writers.dfy @@ -0,0 +1,113 @@ +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "Views.dfy" + +module {:options "-functionSyntax:4"} Views.Writers { + import opened BoundedInts + import opened Wrappers + + import opened Core + + // export + // reveals Error, Writer + // provides Core, Wrappers + // provides Writer_, Writer_.Append, Writer_.Empty, Writer_.Valid? + + datatype Chain = + | Empty + | Chain(previous: Chain, v: View) + { + function Length() : nat { + if Empty? then 0 + else previous.Length() + v.Length() as int + } + + function Count() : nat { + if Empty? then 0 + else previous.Count() + 1 + } + + function Bytes() : (bs: bytes) + ensures |bs| == Length() + { + if Empty? then [] + else previous.Bytes() + v.Bytes() + } + + function Append(v': View): (c: Chain) + ensures c.Bytes() == Bytes() + v'.Bytes() + { + if Chain? && Adjacent(v, v') then + Chain(previous, Merge(v, v')) + else + Chain(this, v') + } + + method {:tailrecursion} Blit(bs: array, end: uint32) + requires end as int == Length() <= bs.Length + modifies bs + ensures bs[..end] == Bytes() + ensures bs[end..] == old(bs[end..]) + { + if Chain? { + var end := end - v.Length(); + v.Blit(bs, end); + previous.Blit(bs, end); + } + } + } + + type Writer = w: Writer_ | w.Valid? witness Writer(0, Chain.Empty) + datatype Writer_ = Writer(length: uint32, chain: Chain) + { + static const Empty: Writer := Writer(0, Chain.Empty) + + const Empty? := chain.Empty? + const Unsaturated? := length != UINT32_MAX + + ghost function Length() : nat { chain.Length() } + + ghost const Valid? := + length == // length is a saturating counter + if chain.Length() >= TWO_TO_THE_32 then UINT32_MAX + else chain.Length() as uint32 + + function Bytes() : (bs: bytes) + ensures |bs| == Length() + { + chain.Bytes() + } + + static function SaturatedAddU32(a: uint32, b: uint32): uint32 { + if a <= UINT32_MAX - b then a + b + else UINT32_MAX + } + + function {:opaque} Append(v': View): (rw: Writer) + requires Valid? + ensures rw.Unsaturated? <==> v'.Length() < UINT32_MAX - length + ensures rw.Bytes() == Bytes() + v'.Bytes() + { + Writer(SaturatedAddU32(length, v'.Length()), + chain.Append(v')) + } + + function Then(fn: Writer ~> Writer) : Writer + reads fn.reads + requires Valid? + requires fn.requires(this) + { + fn(this) + } + + method ToArray() returns (bs: array) + requires Valid? + requires Unsaturated? + ensures fresh(bs) + ensures bs[..] == Bytes() + { + bs := new byte[length]; + chain.Blit(bs, length); + } + } +} From e3d8e8362060f637df94a489f330329dc5b5ea9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Sun, 29 May 2022 19:19:00 -0700 Subject: [PATCH 03/84] json: Move away from member functions to reduce the number of lambdas https://github.com/dafny-lang/dafny/issues/2179 --- src/JSON/Grammar.dfy | 165 -------------- ...Deserializer.dfy => JSON.Deserializer.dfy} | 143 ++++++------ src/JSON/JSON.Grammar.dfy | 78 +++++++ .../{Serializer.dfy => JSON.Serializer.dfy} | 203 +++++++----------- src/JSON/JSON.Spec.dfy | 85 ++++++++ src/JSON/JSON.SpecProperties.dfy | 48 +++++ src/JSON/Tests.dfy | 20 +- 7 files changed, 374 insertions(+), 368 deletions(-) delete mode 100644 src/JSON/Grammar.dfy rename src/JSON/{Deserializer.dfy => JSON.Deserializer.dfy} (69%) create mode 100644 src/JSON/JSON.Grammar.dfy rename src/JSON/{Serializer.dfy => JSON.Serializer.dfy} (55%) create mode 100644 src/JSON/JSON.Spec.dfy create mode 100644 src/JSON/JSON.SpecProperties.dfy diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy deleted file mode 100644 index 71ddd6e3..00000000 --- a/src/JSON/Grammar.dfy +++ /dev/null @@ -1,165 +0,0 @@ -include "../BoundedInts.dfy" -include "Views.dfy" - -module {:options "-functionSyntax:4"} JSON.Grammar { - import opened BoundedInts - import opened Views.Core - - type jchar = v: View | v.Length() == 1 witness View.OfBytes(['b' as byte]) - type jperiod = v: View | v.Char?('.') witness View.OfBytes(['.' as byte]) - type je = v: View | v.Char?('e') || v.Char?('E') witness View.OfBytes(['e' as byte]) - type jcolon = v: View | v.Char?(':') witness View.OfBytes([':' as byte]) - type jcomma = v: View | v.Char?(',') witness View.OfBytes([',' as byte]) - type jlbrace = v: View | v.Char?('{') witness View.OfBytes(['{' as byte]) - type jrbrace = v: View | v.Char?('}') witness View.OfBytes(['}' as byte]) - type jlbracket = v: View | v.Char?('[') witness View.OfBytes(['[' as byte]) - type jrbracket = v: View | v.Char?(']') witness View.OfBytes([']' as byte]) - type jminus = v: View | v.Char?('-') || v.Empty? witness View.OfBytes([]) - type jsign = v: View | v.Char?('-') || v.Char?('+') || v.Empty? witness View.OfBytes([]) - - predicate Blank?(b: byte) { b == 0x20 || b == 0x09 || b == 0x0A || b == 0x0D } - ghost predicate Blanks?(v: View) { forall b | b in v.Bytes() :: Blank?(b) } - type jblanks = v: View | Blanks?(v) witness View.OfBytes([]) - - datatype Structural<+T> = - Structural(before: jblanks, t: T, after: jblanks) - { - function Bytes(pt: T -> bytes): bytes { // LATER: Trait - before.Bytes() + pt(t) + after.Bytes() - } - } - - function ConcatBytes(ts: seq, pt: T --> bytes) : bytes - requires forall d | d in ts :: pt.requires(d) - { - if |ts| == 0 then [] - else pt(ts[0]) + ConcatBytes(ts[1..], pt) - } - - // FIXME remove empty - datatype Maybe<+T> = Empty() | NonEmpty(t: T) { - function Bytes(pt: T --> bytes): bytes - requires NonEmpty? ==> pt.requires(t) - { // LATER: Trait - match this - case Empty() => [] - case NonEmpty(t) => pt(t) - } - } - - datatype Prefixed<+S, +T> = - Prefixed(before: Maybe>, t: T) - { - function Bytes(psep: S -> bytes, pt: T --> bytes): bytes - requires pt.requires(t) - { // LATER: Trait - before.Bytes((s: Structural) => s.Bytes(psep)) + pt(t) - } - } - - type PrefixedSequence<+S, +D> = s: seq> | NoLeadingPrefix(s) - ghost predicate NoLeadingPrefix(s: seq>) { - forall idx | 0 <= idx < |s| :: s[idx].before.Empty? <==> idx == 0 - } - - datatype Bracketed<+L, +S, +D, +R> = - Bracketed(l: Structural, data: PrefixedSequence, r: Structural) - { - function Bytes(pl: L -> bytes, pdatum: Prefixed --> bytes, pr: R -> bytes): bytes - requires forall d | d in data :: pdatum.requires(d) - { // LATER: Trait - l.Bytes(pl) + - ConcatBytes(data, pdatum) + - r.Bytes(pr) - } - } - - function BytesOfString(str: string) : bytes - requires forall c | c in str :: c as int < 256 - { - seq(|str|, idx requires 0 <= idx < |str| => assert str[idx] in str; str[idx] as byte) - } - - const NULL: bytes := BytesOfString("null") - const TRUE: bytes := BytesOfString("true") - const FALSE: bytes := BytesOfString("false") - - ghost predicate Null?(v: View) { v.Bytes() == NULL } - ghost predicate Bool?(v: View) { v.Bytes() in {TRUE, FALSE} } - predicate Digit?(b: byte) { '0' as byte <= b <= '9' as byte } - ghost predicate Digits?(v: View) { forall b | b in v.Bytes() :: Digit?(b) } - ghost predicate Num?(v: View) { Digits?(v) && !v.Empty? } - ghost predicate Int?(v: View) { v.Char?('0') || (Num?(v) && v.At(0) != '0' as byte) } - - type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce correct escaping - type jnull = v: View | Null?(v) witness View.OfBytes(NULL) - type jbool = v: View | Bool?(v) witness View.OfBytes(TRUE) - type jdigits = v: View | Digits?(v) witness View.OfBytes([]) - type jnum = v: View | Num?(v) witness View.OfBytes(['0' as byte]) - type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) - - datatype jkv = KV(k: jstring, colon: Structural, v: Value) { - function Bytes(): bytes { - k.Bytes() + colon.Bytes((c: jcolon) => c.Bytes()) + v.Bytes() - } - } - - type jobject = Bracketed - type jarray = Bracketed - - datatype jfrac = JFrac(period: jperiod, num: jnum) { - function Bytes(): bytes { - period.Bytes() + num.Bytes() - } - } - datatype jexp = JExp(e: je, sign: jsign, num: jnum) { - function Bytes(): bytes { - e.Bytes() + sign.Bytes() + num.Bytes() - } - } - - datatype Value = - | Null(n: jnull) - | Bool(b: jbool) - | String(str: jstring) - | Number(minus: jminus, num: jnum, frac: Maybe, exp: Maybe) - | Object(obj: jobject) - | Array(arr: jarray) - { - function Bytes(): bytes { - match this { - case Null(n) => - n.Bytes() - case Bool(b) => - b.Bytes() - case String(str) => - str.Bytes() - case Number(minus, num, frac, exp) => - minus.Bytes() + num.Bytes() + - frac.Bytes((f: jfrac) => f.Bytes()) + - exp.Bytes((e: jexp) => e.Bytes()) - case Object(o) => - o.Bytes((l: jlbrace) => l.Bytes(), - (d: Prefixed) requires d in o.data => - d.Bytes((s: jcomma) => s.Bytes(), // BUG(https://github.com/dafny-lang/dafny/issues/2170) - var pt := (kv: jkv) requires kv == d.t => d.t.Bytes(); - assert pt.requires(d.t); - pt), - (r: jrbrace) => r.Bytes()) - case Array(a) => - a.Bytes((l: jlbracket) => l.Bytes(), - (d: Prefixed) requires d in a.data => - d.Bytes((s: jcomma) => s.Bytes(), - var pt := (v: Value) requires v == d.t => v.Bytes(); - assert pt.requires(d.t); - pt), - (r: jrbracket) => r.Bytes()) - } - } - } - - type JSON = Structural - function Bytes(js: JSON) : bytes { // TODO: TR - js.Bytes((v: Value) => v.Bytes()) - } -} diff --git a/src/JSON/Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy similarity index 69% rename from src/JSON/Deserializer.dfy rename to src/JSON/JSON.Deserializer.dfy index 12a7fe7e..4fd23605 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/JSON.Deserializer.dfy @@ -1,6 +1,4 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" -include "Grammar.dfy" +include "JSON.Grammar.dfy" include "Parsers.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { @@ -13,10 +11,10 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Grammar datatype JSONError = - | LeadingSeparator // "Separator not allowed before first item" - | UnterminatedSequence // "Unterminated sequence." - | EmptyNumber // "Number must contain at least one digit" - | ExpectingEOF // "Expecting EOF" + | LeadingSeparator + | UnterminatedSequence + | EmptyNumber + | ExpectingEOF { function ToString() : string { match this @@ -33,24 +31,27 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { // FIXME make more things opaque - function {:opaque} Get(ps: FreshCursor, err: JSONError): (pp: ParseResult) - ensures pp.Success? ==> pp.value.t.Length() == 1 - ensures pp.Success? ==> pp.value.ps.StrictlySplitFrom?(ps) // FIXME splitfrom should be on pp + function {:opaque} Get(ps: FreshCursor, err: JSONError): (pr: ParseResult) + ensures pr.Success? ==> pr.value.t.Length() == 1 + ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) // FIXME splitfrom should be on pp { var ps :- ps.Get(err); Success(ps.Split()) } function {:opaque} WS(ps: FreshCursor): (pp: Split) + ensures ps.Bytes() == pp.t.Bytes() + pp.ps.Bytes() ensures pp.ps.SplitFrom?(ps) { ps.SkipWhile(Blank?).Split() } function Structural(ps: FreshCursor, parser: Parser) - : (ppr: ParseResult>) + : (pr: ParseResult>) requires forall ps :: parser.fn.requires(ps) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + // PROOF: pass spec in here ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var SP(before, ps) := WS(ps); var SP(val, ps) :- parser.fn(ps); @@ -70,12 +71,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const CLOSE: byte const SEPARATOR: byte - type TItem - function Item(ps: FreshCursor, json: SubParser) - : (ppr: ParseResult) + type TElement // LATER: With traits, make sure that TElement is convertible to bytes + function Element(ps: FreshCursor, json: SubParser) + : (pr: ParseResult) requires ps.StrictlySplitFrom?(json.ps) decreases ps.Length() - ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + // PROOF pass spec here + ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) } abstract module Sequences { @@ -92,17 +94,18 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { type jopen = v: View | v.Byte?(OPEN) witness View.OfBytes([OPEN]) type jclose = v: View | v.Byte?(CLOSE) witness View.OfBytes([CLOSE]) type jsep = v: View | v.Byte?(SEPARATOR) witness View.OfBytes([SEPARATOR]) - type TSeq = Bracketed + type TSeq = Bracketed - function {:tailrecursion} SeparatorPrefixedItems( + function {:tailrecursion} SeparatorPrefixedElements( ps: FreshCursor, json: SubParser, open: Structural, - items: PrefixedSequence - ): (ppr: ParseResult) + items: PrefixedSequence + ): (pr: ParseResult) requires ps.StrictlySplitFrom?(json.ps) requires NoLeadingPrefix(items) decreases ps.Length() - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + // PROOF ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var SP(sep, ps) :- Core.Structural(ps, Parsers.Parser(ps => Get(ps, UnterminatedSequence))); var s0 := sep.t.At(0); @@ -110,43 +113,43 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(Grammar.Bracketed(open, items, sep), ps)) else if s0 == SEPARATOR then :- Need(|items| > 0, OtherError(LeadingSeparator)); - var SP(item, ps) :- Item(ps, json); + var SP(item, ps) :- Element(ps, json); var items := items + [Prefixed(NonEmpty(sep), item)]; - ghost var ppr' := SeparatorPrefixedItems(ps, json, open, items); // DISCUSS: Why is this needed? - assert ppr'.Success? ==> ppr'.value.ps.StrictlySplitFrom?(ps); - SeparatorPrefixedItems(ps, json, open, items) + ghost var pr' := SeparatorPrefixedElements(ps, json, open, items); // DISCUSS: Why is this needed? + assert pr'.Success? ==> pr'.value.ps.StrictlySplitFrom?(ps); + SeparatorPrefixedElements(ps, json, open, items) else Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0 as opt_byte)) } function Open(ps: FreshCursor) - : (ppr: ParseResult) + : (pr: ParseResult) { var ps :- ps.AssertByte(OPEN); Success(ps.Split()) } function Close(ps: FreshCursor) - : (ppr: ParseResult) + : (pr: ParseResult) { var ps :- ps.AssertByte(CLOSE); Success(ps.Split()) } - function Items(ps: FreshCursor, json: SubParser, open: Structural) - : (ppr: ParseResult) + function Elements(ps: FreshCursor, json: SubParser, open: Structural) + : (pr: ParseResult) requires ps.StrictlySplitFrom?(json.ps) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { if ps.Peek() == CLOSE as opt_byte then var SP(close, ps) :- Core.Structural(ps, Parsers.Parser(Close)); Success(SP(Grammar.Bracketed(open, [], close), ps)) else - var SP(item, ps) :- Item(ps, json); + var SP(item, ps) :- Element(ps, json); var items := [Prefixed(Empty(), item)]; - ghost var ppr' := SeparatorPrefixedItems(ps, json, open, items); // DISCUSS: Why is this needed? - assert ppr'.Success? ==> ppr'.value.ps.StrictlySplitFrom?(ps); - SeparatorPrefixedItems(ps, json, open, items) + ghost var pr' := SeparatorPrefixedElements(ps, json, open, items); // DISCUSS: Why is this needed? + assert pr'.Success? ==> pr'.value.ps.StrictlySplitFrom?(ps); + SeparatorPrefixedElements(ps, json, open, items) } lemma Valid(x: TSeq) @@ -154,29 +157,29 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { ensures x.r.t.Byte?(CLOSE) ensures NoLeadingPrefix(x.data) ensures forall pf | pf in x.data :: - pf.before.NonEmpty? ==> pf.before.t.t.Byte?(SEPARATOR) + pf.prefix.NonEmpty? ==> pf.prefix.t.t.Byte?(SEPARATOR) { // DISCUSS: Why is this lemma needed? Why does it require a body? var xlt: jopen := x.l.t; var xrt: jclose := x.r.t; forall pf | pf in x.data - ensures pf.before.NonEmpty? ==> pf.before.t.t.Byte?(SEPARATOR) + ensures pf.prefix.NonEmpty? ==> pf.prefix.t.t.Byte?(SEPARATOR) { - if pf.before.NonEmpty? { - var xtt := pf.before.t.t; + if pf.prefix.NonEmpty? { + var xtt := pf.prefix.t.t; } } } function Bracketed(ps: FreshCursor, json: SubParser) - : (ppr: ParseResult) + : (pr: ParseResult) requires ps.SplitFrom?(json.ps) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) - // ensures ppr.Success? ==> Valid?(ppr.value.t) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + // ensures pr.Success? ==> Valid?(pr.value.t) { var SP(open, ps) :- Core.Structural(ps, Parsers.Parser(Open)); - ghost var ppr := Items(ps, json, open); // DISCUSS: Why is this needed? - assert ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps); - Items(ps, json, open) + ghost var pr := Elements(ps, json, open); // DISCUSS: Why is this needed? + assert pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps); + Elements(ps, json, open) } } @@ -189,8 +192,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import Values - function JSON(ps: FreshCursor) : (ppr: ParseResult) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + function JSON(ps: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { Core.Structural(ps, Parsers.Parser(Values.Value)) } @@ -216,9 +219,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import Arrays import Constants - function Value(ps: FreshCursor) : (ppr: ParseResult) + function Value(ps: FreshCursor) : (pr: ParseResult) decreases ps.Length(), 1 - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var c := ps.Peek(); if c == '{' as opt_byte then @@ -263,9 +266,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import opened Cursors - function Constant(ps: FreshCursor, expected: bytes) : (ppr: ParseResult) + function Constant(ps: FreshCursor, expected: bytes) : (pr: ParseResult) requires |expected| < TWO_TO_THE_32 - ensures ppr.Success? ==> ppr.value.t.Bytes() == expected + ensures pr.Success? ==> pr.value.t.Bytes() == expected { var ps :- ps.AssertBytes(expected); Success(ps.Split()) @@ -282,8 +285,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Parsers import opened Core - function String(ps: FreshCursor): (ppr: ParseResult) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + function String(ps: FreshCursor): (pr: ParseResult) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var ps :- ps.AssertChar('\"'); var ps :- ps.SkipWhileLexer(StringBody, Partial(StringBodyLexerStart)); @@ -307,8 +310,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { ps.SkipWhile(Digit?).Split() } - function NonEmptyDigits(ps: FreshCursor) : (ppr: ParseResult) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + function NonEmptyDigits(ps: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var pp := Digits(ps); :- Need(!pp.t.Empty?, OtherError(EmptyNumber)); @@ -327,16 +330,16 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { ps.SkipIf(c => c == '-' as byte || c == '+' as byte).Split() } - function TrimmedInt(ps: FreshCursor) : (ppr: ParseResult) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) + function TrimmedInt(ps: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) { var pp := ps.SkipIf(c => c == '0' as byte).Split(); if pp.t.Empty? then NonEmptyDigits(ps) else Success(pp) } - function Exp(ps: FreshCursor) : (ppr: ParseResult>) - ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + function Exp(ps: FreshCursor) : (pr: ParseResult>) + ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) { var SP(e, ps) := ps.SkipIf(c => c == 'e' as byte || c == 'E' as byte).Split(); @@ -349,8 +352,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(NonEmpty(JExp(e, sign, num)), ps)) } - function Frac(ps: FreshCursor) : (ppr: ParseResult>) - ensures ppr.Success? ==> ppr.value.ps.SplitFrom?(ps) + function Frac(ps: FreshCursor) : (pr: ParseResult>) + ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) { var SP(period, ps) := ps.SkipIf(c => c == '.' as byte).Split(); @@ -361,9 +364,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(NonEmpty(JFrac(period, num)), ps)) } - function Number(ps: FreshCursor) : (ppr: ParseResult) - ensures ppr.Success? ==> ppr.value.ps.StrictlySplitFrom?(ps) - ensures ppr.Success? ==> ppr.value.t.Number? + function Number(ps: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + ensures pr.Success? ==> pr.value.t.Number? { var SP(minus, ps) := Minus(ps); var SP(num, ps) :- TrimmedInt(ps); @@ -377,14 +380,14 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Strings import opened Wrappers - type TItem = Value + type TElement = Value const OPEN := '[' as byte const CLOSE := ']' as byte const SEPARATOR: byte := ',' as byte - function Item(ps: FreshCursor, json: SubParser) - : (ppr: ParseResult) + function Element(ps: FreshCursor, json: SubParser) + : (pr: ParseResult) { assert ps.StrictlySplitFrom?(json.ps); assert json.Valid?(); @@ -402,21 +405,21 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Wrappers import Strings - type TItem = jkv + type TElement = jkv const OPEN := '{' as byte const CLOSE := '}' as byte const SEPARATOR: byte := ',' as byte function Colon(ps: FreshCursor) - : (ppr: ParseResult) + : (pr: ParseResult) { var ps :- ps.AssertChar(':'); Success(ps.Split()) } - function Item(ps: FreshCursor, json: SubParser) - : (ppr: ParseResult) + function Element(ps: FreshCursor, json: SubParser) + : (pr: ParseResult) { var SP(k, ps) :- Strings.String(ps); var SP(colon, ps) :- Core.Structural(ps, Parsers.Parser(Colon)); diff --git a/src/JSON/JSON.Grammar.dfy b/src/JSON/JSON.Grammar.dfy new file mode 100644 index 00000000..4adbd114 --- /dev/null +++ b/src/JSON/JSON.Grammar.dfy @@ -0,0 +1,78 @@ +include "../BoundedInts.dfy" +include "Views.dfy" + +module {:options "-functionSyntax:4"} JSON.Grammar { + import opened BoundedInts + import opened Views.Core + + type jchar = v: View | v.Length() == 1 witness View.OfBytes(['b' as byte]) + type jperiod = v: View | v.Char?('.') witness View.OfBytes(['.' as byte]) + type je = v: View | v.Char?('e') || v.Char?('E') witness View.OfBytes(['e' as byte]) + type jcolon = v: View | v.Char?(':') witness View.OfBytes([':' as byte]) + type jcomma = v: View | v.Char?(',') witness View.OfBytes([',' as byte]) + type jlbrace = v: View | v.Char?('{') witness View.OfBytes(['{' as byte]) + type jrbrace = v: View | v.Char?('}') witness View.OfBytes(['}' as byte]) + type jlbracket = v: View | v.Char?('[') witness View.OfBytes(['[' as byte]) + type jrbracket = v: View | v.Char?(']') witness View.OfBytes([']' as byte]) + type jminus = v: View | v.Char?('-') || v.Empty? witness View.OfBytes([]) + type jsign = v: View | v.Char?('-') || v.Char?('+') || v.Empty? witness View.OfBytes([]) + + predicate Blank?(b: byte) { b == 0x20 || b == 0x09 || b == 0x0A || b == 0x0D } + ghost predicate Blanks?(v: View) { forall b | b in v.Bytes() :: Blank?(b) } + type jblanks = v: View | Blanks?(v) witness View.OfBytes([]) + + datatype Structural<+T> = + Structural(before: jblanks, t: T, after: jblanks) + + datatype Maybe<+T> = Empty() | NonEmpty(t: T) + + datatype Prefixed<+S, +T> = + Prefixed(prefix: Maybe>, t: T) + + type PrefixedSequence<+S, +D> = s: seq> | NoLeadingPrefix(s) + ghost predicate NoLeadingPrefix(s: seq>) { + forall idx | 0 <= idx < |s| :: s[idx].prefix.Empty? <==> idx == 0 + } + + datatype Bracketed<+L, +S, +D, +R> = + Bracketed(l: Structural, data: PrefixedSequence, r: Structural) + + const NULL: bytes := ['n' as byte, 'u' as byte, 'l' as byte, 'l' as byte] + const TRUE: bytes := ['t' as byte, 'r' as byte, 'u' as byte, 'e' as byte] + const FALSE: bytes := ['f' as byte, 'a' as byte, 'l' as byte, 's' as byte, 'e' as byte] + + ghost predicate Null?(v: View) { v.Bytes() == NULL } + ghost predicate Bool?(v: View) { v.Bytes() in {TRUE, FALSE} } + predicate Digit?(b: byte) { '0' as byte <= b <= '9' as byte } + ghost predicate Digits?(v: View) { forall b | b in v.Bytes() :: Digit?(b) } + ghost predicate Num?(v: View) { Digits?(v) && !v.Empty? } + ghost predicate Int?(v: View) { v.Char?('0') || (Num?(v) && v.At(0) != '0' as byte) } + + type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce correct escaping + type jnull = v: View | Null?(v) witness View.OfBytes(NULL) + type jbool = v: View | Bool?(v) witness View.OfBytes(TRUE) + type jdigits = v: View | Digits?(v) witness View.OfBytes([]) + type jnum = v: View | Num?(v) witness View.OfBytes(['0' as byte]) + type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) + datatype jkv = KV(k: jstring, colon: Structural, v: Value) + + type jobject = Bracketed + type jarray = Bracketed + type jmembers = PrefixedSequence + type jmember = Prefixed + type jitems = PrefixedSequence + type jitem = Prefixed + + datatype jfrac = JFrac(period: jperiod, num: jnum) + datatype jexp = JExp(e: je, sign: jsign, num: jnum) + + datatype Value = + | Null(n: jnull) + | Bool(b: jbool) + | String(str: jstring) + | Number(minus: jminus, num: jnum, frac: Maybe, exp: Maybe) + | Object(obj: jobject) + | Array(arr: jarray) + + type JSON = Structural +} diff --git a/src/JSON/Serializer.dfy b/src/JSON/JSON.Serializer.dfy similarity index 55% rename from src/JSON/Serializer.dfy rename to src/JSON/JSON.Serializer.dfy index 7dd54d7d..54b1ba14 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/JSON.Serializer.dfy @@ -1,13 +1,13 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" -include "Grammar.dfy" +include "JSON.Spec.dfy" +include "JSON.SpecProperties.dfy" include "Views.Writers.dfy" -include "Stacks.dfy" module {:options "-functionSyntax:4"} JSON.Serializer { import opened BoundedInts import opened Wrappers + import Spec + import SpecProperties import opened Grammar import opened Views.Writers import opened Vs = Views.Core // DISCUSS: Module naming convention? @@ -15,7 +15,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { datatype Error = OutOfMemory method Serialize(js: JSON) returns (bsr: Result, Error>) - ensures bsr.Success? ==> bsr.value[..] == Bytes(js) + ensures bsr.Success? ==> bsr.value[..] == Spec.JSON(js) { var writer := JSON(js); :- Need(writer.Unsaturated?, OutOfMemory); @@ -24,24 +24,17 @@ module {:options "-functionSyntax:4"} JSON.Serializer { } function {:opaque} JSON(js: JSON, writer: Writer := Writer.Empty) : (wr: Writer) - ensures wr.Bytes() == writer.Bytes() + Bytes(js) + ensures wr.Bytes() == writer.Bytes() + Spec.JSON(js) { - // DISCUSS: This doesn't work: - // writer - // .Append(js.before) - // .Then(wr => Value(js.t, wr)) - // .Append(js.after) - // … but this does: - var wr := writer; - var wr := wr.Append(js.before); - var wr := Value(js.t, wr); - var wr := wr.Append(js.after); - wr + writer + .Append(js.before) + .Then(wr => Value(js.t, wr)) + .Append(js.after) } function {:opaque} Value(v: Value, writer: Writer) : (wr: Writer) decreases v, 4 - ensures wr.Bytes() == writer.Bytes() + v.Bytes() + ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { match v case Null(n) => @@ -61,7 +54,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { function {:opaque} Number(v: Value, writer: Writer) : (wr: Writer) requires v.Number? decreases v, 0 - ensures wr.Bytes() == writer.Bytes() + v.Bytes() + ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { var writer := writer.Append(v.minus).Append(v.num); var writer := if v.frac.NonEmpty? then @@ -74,96 +67,54 @@ module {:options "-functionSyntax:4"} JSON.Serializer { } // DISCUSS: Can't be opaque, due to the lambda - function Structural(st: Structural, writer: Writer) : (wr: Writer) - // FIXME ensures writer.Length() + |st.Bytes((v: View) => v.Bytes())| < TWO_TO_THE_32 ==> rwr.Success? - ensures wr.Bytes() == writer.Bytes() + st.Bytes((v: View) => v.Bytes()) + function StructuralView(st: Structural, writer: Writer) : (wr: Writer) + ensures wr.Bytes() == writer.Bytes() + Spec.Structural(st, Spec.View) { writer.Append(st.before).Append(st.t).Append(st.after) } - lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS - ensures forall - pl0: L -> bytes, pd0: Prefixed --> bytes, pr0: R -> bytes, - pl1: L -> bytes, pd1: Prefixed --> bytes, pr1: R -> bytes - | && (forall d | d in bracketed.data :: pd0.requires(d)) - && (forall d | d in bracketed.data :: pd1.requires(d)) - && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) - && (forall l :: pl0(l) == pl1(l)) - && (forall r :: pr0(r) == pr1(r)) - :: bracketed.Bytes(pl0, pd0, pr0) == bracketed.Bytes(pl1, pd1, pr1) - { - forall pl0: L -> bytes, pd0: Prefixed --> bytes, pr0: R -> bytes, - pl1: L -> bytes, pd1: Prefixed --> bytes, pr1: R -> bytes - | && (forall d | d in bracketed.data :: pd0.requires(d)) - && (forall d | d in bracketed.data :: pd1.requires(d)) - && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) - && (forall l :: pl0(l) == pl1(l)) - && (forall r :: pr0(r) == pr1(r)) - { - calc { - bracketed.Bytes(pl0, pd0, pr0); - { ConcatBytes_Morphism(bracketed.data, pd0, pd1); } - bracketed.Bytes(pl1, pd1, pr1); - } - } - } - - lemma {:induction ts} ConcatBytes_Morphism(ts: seq, pt0: T --> bytes, pt1: T --> bytes) - requires forall d | d in ts :: pt0.requires(d) - requires forall d | d in ts :: pt1.requires(d) - requires forall d | d in ts :: pt0(d) == pt1(d) - ensures ConcatBytes(ts, pt0) == ConcatBytes(ts, pt1) - {} - - lemma {:induction ts0} ConcatBytes_Linear(ts0: seq, ts1: seq, pt: T --> bytes) - requires forall d | d in ts0 :: pt.requires(d) - requires forall d | d in ts1 :: pt.requires(d) - ensures ConcatBytes(ts0 + ts1, pt) == ConcatBytes(ts0, pt) + ConcatBytes(ts1, pt) - { - if |ts0| == 0 { - assert [] + ts1 == ts1; - } else { - assert ts0 + ts1 == [ts0[0]] + (ts0[1..] + ts1); - } - } - lemma {:axiom} Assume(b: bool) ensures b + // FIXME refactor below to merge + function {:opaque} Object(v: Value, obj: jobject, writer: Writer) : (wr: Writer) requires v.Object? && obj == v.obj decreases v, 3 - ensures wr.Bytes() == writer.Bytes() + v.Bytes() + ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { - var writer := Structural(obj.l, writer); + var writer := StructuralView(obj.l, writer); var writer := Members(v, obj, writer); - var writer := Structural(obj.r, writer); - - Bracketed_Morphism(obj); // DISCUSS - assert v.Bytes() == obj.Bytes(((l: jlbrace) => l.Bytes()), Grammar.Member, ((r: jrbrace) => r.Bytes())); + var writer := StructuralView(obj.r, writer); + // We call ``ConcatBytes`` with ``Spec.Member``, whereas the spec calls it + // with ``(d: jmember) requires d in obj.data => Spec.Member(d)``. That's + // why we need an explicit cast, which is performed by the lemma below. + SpecProperties.Bracketed_Morphism(obj); + assert Spec.Value(v) == Spec.Bracketed(obj, Spec.Member); writer } function {:opaque} Array(v: Value, arr: jarray, writer: Writer) : (wr: Writer) requires v.Array? && arr == v.arr decreases v, 3 - ensures wr.Bytes() == writer.Bytes() + v.Bytes() + ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { - var writer := Structural(arr.l, writer); + var writer := StructuralView(arr.l, writer); var writer := Items(v, arr, writer); - var writer := Structural(arr.r, writer); - - Bracketed_Morphism(arr); // DISCUSS - assert v.Bytes() == arr.Bytes(((l: jlbracket) => l.Bytes()), Grammar.Item, ((r: jrbracket) => r.Bytes())); + var writer := StructuralView(arr.r, writer); + // We call ``ConcatBytes`` with ``Spec.Item``, whereas the spec calls it + // with ``(d: jitem) requires d in arr.data => Spec.Item(d)``. That's + // why we need an explicit cast, which is performed by the lemma below. + SpecProperties.Bracketed_Morphism(arr); // DISCUSS + assert Spec.Value(v) == Spec.Bracketed(arr, Spec.Item); writer } function {:opaque} Members(ghost v: Value, obj: jobject, writer: Writer) : (wr: Writer) requires obj < v decreases v, 2 - ensures wr.Bytes() == writer.Bytes() + ConcatBytes(obj.data, Grammar.Member) - ensures wr == MembersSpec(v, obj.data, writer) + ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(obj.data, Spec.Member) { MembersSpec(v, obj.data, writer) } by method { @@ -171,22 +122,49 @@ module {:options "-functionSyntax:4"} JSON.Serializer { Assume(false); // FIXME } + function {:opaque} Items(ghost v: Value, arr: jarray, writer: Writer) : (wr: Writer) + requires arr < v + decreases v, 2 + ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(arr.data, Spec.Item) + { + ItemsSpec(v, arr.data, writer) + } by method { + wr := ItemsImpl(v, arr, writer); + Assume(false); // FIXME + } + ghost function MembersSpec(v: Value, members: seq, writer: Writer) : (wr: Writer) requires forall j | 0 <= j < |members| :: members[j] < v decreases v, 1, members - ensures wr.Bytes() == writer.Bytes() + ConcatBytes(members, Grammar.Member) + ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(members, Spec.Member) { // TR elimination doesn't work for mutually recursive methods, so this // function is only used as a spec for Members. if members == [] then writer else var writer := MembersSpec(v, members[..|members|-1], writer); assert members == members[..|members|-1] + [members[|members|-1]]; - ConcatBytes_Linear(members[..|members|-1], [members[|members|-1]], Grammar.Member); + SpecProperties.ConcatBytes_Linear(members[..|members|-1], [members[|members|-1]], Spec.Member); Member(v, members[|members|-1], writer) } // No by method block here, because the loop invariant in the method version // needs to call MembersSpec and the termination checker gets confused by // that. Instead, see Members above. // DISCUSS + ghost function ItemsSpec(v: Value, items: seq, writer: Writer) : (wr: Writer) + requires forall j | 0 <= j < |items| :: items[j] < v + decreases v, 1, items + ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(items, Spec.Item) + { // TR elimination doesn't work for mutually recursive methods, so this + // function is only used as a spec for Items. + if items == [] then writer + else + var writer := ItemsSpec(v, items[..|items|-1], writer); + assert items == items[..|items|-1] + [items[|items|-1]]; + SpecProperties.ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], Spec.Item); + Item(v, items[|items|-1], writer) + } // No by method block here, because the loop invariant in the method version + // needs to call ItemsSpec and the termination checker gets confused by + // that. Instead, see Items above. // DISCUSS + method MembersImpl(ghost v: Value, obj: jobject, writer: Writer) returns (wr: Writer) requires obj < v decreases v, 1 @@ -204,44 +182,6 @@ module {:options "-functionSyntax:4"} JSON.Serializer { assert members[..|members|] == members; } - function {:opaque} Member(ghost v: Value, m: jmember, writer: Writer) : (wr: Writer) - requires m < v - decreases v, 0 - ensures wr.Bytes() == writer.Bytes() + Grammar.Member(m) - { - var writer := if m.prefix.NonEmpty? then Structural(m.prefix.t, writer) else writer; - var writer := writer.Append(m.t.k); - var writer := Structural(m.t.colon, writer); - Value(m.t.v, writer) - } - function {:opaque} Items(ghost v: Value, arr: jarray, writer: Writer) : (wr: Writer) - requires arr < v - decreases v, 2 - ensures wr.Bytes() == writer.Bytes() + ConcatBytes(arr.data, Grammar.Item) - ensures wr == ItemsSpec(v, arr.data, writer) - { - ItemsSpec(v, arr.data, writer) - } by method { - wr := ItemsImpl(v, arr, writer); - Assume(false); // FIXME - } - - ghost function ItemsSpec(v: Value, items: seq, writer: Writer) : (wr: Writer) - requires forall j | 0 <= j < |items| :: items[j] < v - decreases v, 1, items - ensures wr.Bytes() == writer.Bytes() + ConcatBytes(items, Grammar.Item) - { // TR elimination doesn't work for mutually recursive methods, so this - // function is only used as a spec for Items. - if items == [] then writer - else - var writer := ItemsSpec(v, items[..|items|-1], writer); - assert items == items[..|items|-1] + [items[|items|-1]]; - ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], Grammar.Item); - Item(v, items[|items|-1], writer) - } // No by method block here, because the loop invariant in the method version - // needs to call ItemsSpec and the termination checker gets confused by - // that. Instead, see Items above. // DISCUSS - method ItemsImpl(ghost v: Value, arr: jarray, writer: Writer) returns (wr: Writer) requires arr < v decreases v, 1 @@ -259,14 +199,23 @@ module {:options "-functionSyntax:4"} JSON.Serializer { assert items[..|items|] == items; } - function {:opaque} Item(ghost v: Value, m: jitem, writer: Writer) : (wr: Writer) + function {:opaque} Member(ghost v: Value, m: jmember, writer: Writer) : (wr: Writer) requires m < v decreases v, 0 - ensures wr.Bytes() == writer.Bytes() + Grammar.Item(m) + ensures wr.Bytes() == writer.Bytes() + Spec.Member(m) { - var writer := if m.prefix.NonEmpty? then Structural(m.prefix.t, writer) else writer; - Value(m.t, writer) + var writer := if m.prefix.Empty? then writer else StructuralView(m.prefix.t, writer); + var writer := writer.Append(m.t.k); + var writer := StructuralView(m.t.colon, writer); + Value(m.t.v, writer) } - + function {:opaque} Item(ghost v: Value, m: jitem, writer: Writer) : (wr: Writer) + requires m < v + decreases v, 0 + ensures wr.Bytes() == writer.Bytes() + Spec.Item(m) + { + var wr := if m.prefix.Empty? then writer else StructuralView(m.prefix.t, writer); + Value(m.t, wr) + } } diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/JSON.Spec.dfy new file mode 100644 index 00000000..9650cd19 --- /dev/null +++ b/src/JSON/JSON.Spec.dfy @@ -0,0 +1,85 @@ +include "JSON.Grammar.dfy" + +module {:options "-functionSyntax:4"} JSON.Spec { + import opened BoundedInts + + import Vs = Views.Core + import opened Grammar + + function View(v: Vs.View) : bytes { + v.Bytes() + } + + function Structural(self: Structural, pt: T -> bytes): bytes { + View(self.before) + pt(self.t) + View(self.after) + } + + function StructuralView(self: Structural): bytes { + Structural(self, View) + } + + function Maybe(self: Maybe, pt: T -> bytes): (bs: bytes) + ensures self.Empty? ==> bs == [] + ensures self.NonEmpty? ==> bs == pt(self.t) + { + if self.Empty? then [] else pt(self.t) + } + + function ConcatBytes(ts: seq, pt: T --> bytes) : bytes + requires forall d | d in ts :: pt.requires(d) + { + if |ts| == 0 then [] + else pt(ts[0]) + ConcatBytes(ts[1..], pt) + } + + function Bracketed(self: Bracketed, pdatum: Prefixed --> bytes): bytes + requires forall d | d in self.data :: pdatum.requires(d) + { + StructuralView(self.l) + + ConcatBytes(self.data, pdatum) + + StructuralView(self.r) + } + + function KV(self: jkv): bytes { + View(self.k) + StructuralView(self.colon) + Value(self.v) + } + + function Frac(self: jfrac): bytes { + View(self.period) + View(self.num) + } + + function Exp(self: jexp): bytes { + View(self.e) + View(self.sign) + View(self.num) + } + + function Member(self: jmember) : bytes { + // BUG(https://github.com/dafny-lang/dafny/issues/2179) + Maybe>(self.prefix, StructuralView) + KV(self.t) + } + + function Item(self: jitem) : bytes { + // BUG(https://github.com/dafny-lang/dafny/issues/2179) + Maybe>(self.prefix, StructuralView) + Value(self.t) + } + + function Value(self: Value): bytes { + match self { + case Null(n) => + View(n) + case Bool(b) => + View(b) + case String(str) => + View(str) + case Number(minus, num, frac, exp) => + View(minus) + View(num) + Maybe(frac, Frac) + Maybe(exp, Exp) + case Object(obj) => + Bracketed(obj, (d: jmember) requires d in obj.data => Member(d)) + case Array(arr) => + Bracketed(arr, (d: jitem) requires d in arr.data => Item(d)) + } + } + + function JSON(js: JSON) : bytes { + Structural(js, Value) + } +} diff --git a/src/JSON/JSON.SpecProperties.dfy b/src/JSON/JSON.SpecProperties.dfy new file mode 100644 index 00000000..3247acff --- /dev/null +++ b/src/JSON/JSON.SpecProperties.dfy @@ -0,0 +1,48 @@ +include "JSON.Spec.dfy" + +module {:options "-functionSyntax:4"} JSON.SpecProperties { + import opened BoundedInts + + import Vs = Views.Core + import opened Grammar + import Spec + + lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS + ensures forall pd0: Prefixed --> bytes, pd1: Prefixed --> bytes + | && (forall d | d in bracketed.data :: pd0.requires(d)) + && (forall d | d in bracketed.data :: pd1.requires(d)) + && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + :: Spec.Bracketed(bracketed, pd0) == Spec.Bracketed(bracketed, pd1) + { + forall pd0: Prefixed --> bytes, pd1: Prefixed --> bytes + | && (forall d | d in bracketed.data :: pd0.requires(d)) + && (forall d | d in bracketed.data :: pd1.requires(d)) + && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + { + calc { + Spec.Bracketed(bracketed, pd0); + { ConcatBytes_Morphism(bracketed.data, pd0, pd1); } + Spec.Bracketed(bracketed, pd1); + } + } + } + + lemma {:induction ts} ConcatBytes_Morphism(ts: seq, pt0: T --> bytes, pt1: T --> bytes) + requires forall d | d in ts :: pt0.requires(d) + requires forall d | d in ts :: pt1.requires(d) + requires forall d | d in ts :: pt0(d) == pt1(d) + ensures Spec.ConcatBytes(ts, pt0) == Spec.ConcatBytes(ts, pt1) + {} + + lemma {:induction ts0} ConcatBytes_Linear(ts0: seq, ts1: seq, pt: T --> bytes) + requires forall d | d in ts0 :: pt.requires(d) + requires forall d | d in ts1 :: pt.requires(d) + ensures Spec.ConcatBytes(ts0 + ts1, pt) == Spec.ConcatBytes(ts0, pt) + Spec.ConcatBytes(ts1, pt) + { + if |ts0| == 0 { + assert [] + ts1 == ts1; + } else { + assert ts0 + ts1 == [ts0[0]] + (ts0[1..] + ts1); + } + } +} diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index acb1828d..07975a86 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,5 +1,5 @@ -include "Grammar.dfy" -include "Deserializer.dfy" +include "JSON.Serializer.dfy" +include "JSON.Deserializer.dfy" include "../Collections/Sequences/Seq.dfy" import opened BoundedInts @@ -7,7 +7,8 @@ import opened BoundedInts import opened Vs = Views.Core import opened JSON.Grammar -import opened JSON.Deserializer +import JSON.Serializer +import JSON.Deserializer function method bytes_of_ascii(s: string) : bytes { Seq.Map((c: char) requires c in s => if c as int < 256 then c as byte else 0 as byte, s) @@ -47,9 +48,16 @@ method Main() { print "Parse error: " + msg.ToString((e: Deserializer.Core.JSONError) => e.ToString()) + "\n"; expect false; case Success(js) => - var bytes' := Grammar.Bytes(js); - print "=> " + ascii_of_bytes(bytes') + "\n"; - expect bytes' == bytes; + // var bytes' := Grammar.Bytes(js); + var wr := Serializer.JSON(js); + print "Count: ", wr.chain.Count(), "\n"; + var rbytes' := Serializer.Serialize(js); + match rbytes' { + case Failure(msg) => expect false; + case Success(bytes') => + print "=> " + ascii_of_bytes(bytes'[..]) + "\n"; + expect bytes'[..] == bytes; + } } print "\n"; } From c5394f777cb9fcbea61332239324db66ea79c547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Mon, 30 May 2022 15:03:08 -0700 Subject: [PATCH 04/84] json: Clean up grammar and put separators after items, not before --- src/JSON/Cursors.dfy | 19 +- src/JSON/JSON.Deserializer.dfy | 409 +++++++++++++++++-------------- src/JSON/JSON.Grammar.dfy | 32 +-- src/JSON/JSON.Serializer.dfy | 174 +++++++------ src/JSON/JSON.Spec.dfy | 49 ++-- src/JSON/JSON.SpecProperties.dfy | 6 +- src/JSON/Parsers.dfy | 29 ++- src/JSON/Views.dfy | 3 + 8 files changed, 407 insertions(+), 314 deletions(-) diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy index ca4260f4..73802931 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Cursors.dfy @@ -10,7 +10,19 @@ module {:options "-functionSyntax:4"} Cursors { import opened Vs = Views.Core import opened Lx = Lexers.Core - datatype Split<+T> = SP(t: T, ps: FreshCursor) + datatype Split<+T> = SP(t: T, cs: FreshCursor) { + ghost predicate BytesSplitFrom?(cs0: Cursor, spec: T -> bytes) { + cs0.Bytes() == spec(t) + cs.Bytes() + } + + ghost predicate SplitFrom?(cs0: Cursor, spec: T -> bytes) { + cs.SplitFrom?(cs0) && BytesSplitFrom?(cs0, spec) + } + + ghost predicate StrictlySplitFrom?(cs0: Cursor, spec: T -> bytes) { + cs.StrictlySplitFrom?(cs0) && BytesSplitFrom?(cs0, spec) + } + } // LATER: Make this a newtype and put members here instead of requiring `Valid?` everywhere type Cursor = ps: Cursor_ | ps.Valid? @@ -119,7 +131,10 @@ module {:options "-functionSyntax:4"} Cursors { this.(beg := point) } - function Split() : Split requires Valid? { + function Split() : (sp: Split) requires Valid? + ensures sp.SplitFrom?(this, (v: View) => v.Bytes()) + ensures beg != point ==> sp.StrictlySplitFrom?(this, (v: View) => v.Bytes()) + { SP(this.Prefix(), this.Suffix()) } diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy index 4fd23605..b586faf8 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/JSON.Deserializer.dfy @@ -1,4 +1,5 @@ include "JSON.Grammar.dfy" +include "JSON.Spec.dfy" include "Parsers.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { @@ -6,58 +7,61 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened BoundedInts import opened Wrappers + import Spec + import Vs = Views.Core import opened Cursors import opened Parsers import opened Grammar datatype JSONError = - | LeadingSeparator | UnterminatedSequence | EmptyNumber | ExpectingEOF { function ToString() : string { match this - case LeadingSeparator => "Separator not allowed before first item" - case UnterminatedSequence => "Unterminated sequence." + case UnterminatedSequence => "Unterminated sequence" case EmptyNumber => "Number must contain at least one digit" case ExpectingEOF => "Expecting EOF" } } type ParseError = CursorError type ParseResult<+T> = SplitResult - type Parser<+T> = Parsers.Parser - type SubParser<+T> = Parsers.SubParser + type Parser = Parsers.Parser + type SubParser = Parsers.SubParser + + // BUG(https://github.com/dafny-lang/dafny/issues/2179) + const SpecView := (v: Vs.View) => Spec.View(v); // FIXME make more things opaque - function {:opaque} Get(ps: FreshCursor, err: JSONError): (pr: ParseResult) - ensures pr.Success? ==> pr.value.t.Length() == 1 - ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) // FIXME splitfrom should be on pp + function {:opaque} Get(cs: FreshCursor, err: JSONError): (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { - var ps :- ps.Get(err); - Success(ps.Split()) + var cs :- cs.Get(err); + Success(cs.Split()) } - function {:opaque} WS(ps: FreshCursor): (pp: Split) - ensures ps.Bytes() == pp.t.Bytes() + pp.ps.Bytes() - ensures pp.ps.SplitFrom?(ps) + function {:opaque} WS(cs: FreshCursor): (sp: Split) + ensures sp.SplitFrom?(cs, SpecView) { - ps.SkipWhile(Blank?).Split() + cs.SkipWhile(Blank?).Split() } - function Structural(ps: FreshCursor, parser: Parser) + function Structural(cs: FreshCursor, parser: Parser) : (pr: ParseResult>) - requires forall ps :: parser.fn.requires(ps) - // PROOF: pass spec in here ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + requires forall cs :: parser.fn.requires(cs) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) { - var SP(before, ps) := WS(ps); - var SP(val, ps) :- parser.fn(ps); - var SP(after, ps) := WS(ps); - Success(SP(Grammar.Structural(before, val, after), ps)) + var SP(before, cs) := WS(cs); + var SP(val, cs) :- parser.fn(cs); + var SP(after, cs) := WS(cs); + Success(SP(Grammar.Structural(before, val, after), cs)) } + + type ValueParser = sp: SubParser | + forall t :: sp.spec(t) == Spec.Value(t) + witness * } abstract module SequenceParams { @@ -71,13 +75,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const CLOSE: byte const SEPARATOR: byte - type TElement // LATER: With traits, make sure that TElement is convertible to bytes - function Element(ps: FreshCursor, json: SubParser) + type TElement + ghost function ElementSpec(t: TElement) : bytes + function Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) - requires ps.StrictlySplitFrom?(json.ps) - decreases ps.Length() - // PROOF pass spec here - ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) + requires cs.StrictlySplitFrom?(json.cs) + decreases cs.Length() + ensures pr.Success? ==> pr.value.SplitFrom?(cs, ElementSpec) } abstract module Sequences { @@ -94,93 +98,85 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { type jopen = v: View | v.Byte?(OPEN) witness View.OfBytes([OPEN]) type jclose = v: View | v.Byte?(CLOSE) witness View.OfBytes([CLOSE]) type jsep = v: View | v.Byte?(SEPARATOR) witness View.OfBytes([SEPARATOR]) - type TSeq = Bracketed + type TSeq = Bracketed - function {:tailrecursion} SeparatorPrefixedElements( - ps: FreshCursor, json: SubParser, - open: Structural, - items: PrefixedSequence - ): (pr: ParseResult) - requires ps.StrictlySplitFrom?(json.ps) - requires NoLeadingPrefix(items) - decreases ps.Length() - // PROOF ensures pr.Success? ==> ps.Bytes() == pr.value.t.Bytes() + pr.value.ps.Bytes() - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function Open(cs: FreshCursor) + : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, _ => [OPEN]) { - var SP(sep, ps) :- Core.Structural(ps, Parsers.Parser(ps => Get(ps, UnterminatedSequence))); - var s0 := sep.t.At(0); - if s0 == CLOSE then - Success(SP(Grammar.Bracketed(open, items, sep), ps)) - else if s0 == SEPARATOR then - :- Need(|items| > 0, OtherError(LeadingSeparator)); - var SP(item, ps) :- Element(ps, json); - var items := items + [Prefixed(NonEmpty(sep), item)]; - ghost var pr' := SeparatorPrefixedElements(ps, json, open, items); // DISCUSS: Why is this needed? - assert pr'.Success? ==> pr'.value.ps.StrictlySplitFrom?(ps); - SeparatorPrefixedElements(ps, json, open, items) - else - Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0 as opt_byte)) + var cs :- cs.AssertByte(OPEN); + Success(cs.Split()) } - function Open(ps: FreshCursor) - : (pr: ParseResult) + function Close(cs: FreshCursor) + : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, _ => [CLOSE]) { - var ps :- ps.AssertByte(OPEN); - Success(ps.Split()) + var cs :- cs.AssertByte(CLOSE); + Success(cs.Split()) } - function Close(ps: FreshCursor) - : (pr: ParseResult) + function {:tailrecursion} Elements( + cs: FreshCursor, json: ValueParser, + open: Structural, + items: seq> := [] + ) + : (pr: ParseResult) + requires cs.StrictlySplitFrom?(json.cs) + requires forall x | x in items :: x.suffix.NonEmpty? + decreases cs.Length() + ensures pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs) + // PROOF { - var ps :- ps.AssertByte(CLOSE); - Success(ps.Split()) + var SP(item, cs) :- Element(cs, json); + var SP(sep, cs) :- Core.Structural(cs, + Parsers.Parser(cs => Get(cs, UnterminatedSequence), SpecView)); + var s0 := sep.t.At(0); + if s0 == CLOSE then + var items := items + [Suffixed(item, Empty())]; + Success(SP(Grammar.Bracketed(open, items, sep), cs)) + else if s0 == SEPARATOR then + var SP(item, cs) :- Element(cs, json); + var items := items + [Suffixed(item, NonEmpty(sep))]; + ghost var pr' := Elements(cs, json, open, items); // DISCUSS: Why is this needed? + assert pr'.Success? ==> pr'.value.cs.StrictlySplitFrom?(cs); + Elements(cs, json, open, items) + else + Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0 as opt_byte)) } - function Elements(ps: FreshCursor, json: SubParser, open: Structural) + function Bracketed(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) - requires ps.StrictlySplitFrom?(json.ps) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + requires cs.SplitFrom?(json.cs) + ensures pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs) { - if ps.Peek() == CLOSE as opt_byte then - var SP(close, ps) :- Core.Structural(ps, Parsers.Parser(Close)); - Success(SP(Grammar.Bracketed(open, [], close), ps)) + var SP(open, cs) :- Core.Structural(cs, Parsers.Parser(Open, SpecView)); + if cs.Peek() == CLOSE as opt_byte then + var SP(close, cs) :- Core.Structural(cs, Parsers.Parser(Close, SpecView)); + Success(SP(Grammar.Bracketed(open, [], close), cs)) else - var SP(item, ps) :- Element(ps, json); - var items := [Prefixed(Empty(), item)]; - ghost var pr' := SeparatorPrefixedElements(ps, json, open, items); // DISCUSS: Why is this needed? - assert pr'.Success? ==> pr'.value.ps.StrictlySplitFrom?(ps); - SeparatorPrefixedElements(ps, json, open, items) + ghost var pr := Elements(cs, json, open); // DISCUSS: Why is this needed? + assert pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs); + Elements(cs, json, open) } lemma Valid(x: TSeq) ensures x.l.t.Byte?(OPEN) ensures x.r.t.Byte?(CLOSE) - ensures NoLeadingPrefix(x.data) + ensures NoTrailingSuffix(x.data) ensures forall pf | pf in x.data :: - pf.prefix.NonEmpty? ==> pf.prefix.t.t.Byte?(SEPARATOR) + pf.suffix.NonEmpty? ==> pf.suffix.t.t.Byte?(SEPARATOR) { // DISCUSS: Why is this lemma needed? Why does it require a body? var xlt: jopen := x.l.t; var xrt: jclose := x.r.t; forall pf | pf in x.data - ensures pf.prefix.NonEmpty? ==> pf.prefix.t.t.Byte?(SEPARATOR) + ensures pf.suffix.NonEmpty? ==> pf.suffix.t.t.Byte?(SEPARATOR) { - if pf.prefix.NonEmpty? { - var xtt := pf.prefix.t.t; + if pf.suffix.NonEmpty? { + var xtt := pf.suffix.t.t; } } } - - function Bracketed(ps: FreshCursor, json: SubParser) - : (pr: ParseResult) - requires ps.SplitFrom?(json.ps) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) - // ensures pr.Success? ==> Valid?(pr.value.t) - { - var SP(open, ps) :- Core.Structural(ps, Parsers.Parser(Open)); - ghost var pr := Elements(ps, json, open); // DISCUSS: Why is this needed? - assert pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps); - Elements(ps, json, open) - } } module Top { @@ -192,15 +188,15 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import Values - function JSON(ps: FreshCursor) : (pr: ParseResult) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function JSON(cs: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.JSON) { - Core.Structural(ps, Parsers.Parser(Values.Value)) + Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)) } function Text(v: View) : (pr: Result) { - var SP(text, ps) :- JSON(Cursor.OfView(v)); - :- Need(ps.EOF?, OtherError(ExpectingEOF)); + var SP(text, cs) :- JSON(Cursor.OfView(v)); + :- Need(cs.EOF?, OtherError(ExpectingEOF)); Success(text) } } @@ -219,42 +215,43 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import Arrays import Constants - function Value(ps: FreshCursor) : (pr: ParseResult) - decreases ps.Length(), 1 - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) + decreases cs.Length(), 1 + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Value) { - var c := ps.Peek(); - if c == '{' as opt_byte then - var SP(obj, ps) :- Objects.Bracketed(ps, ValueParser(ps)); + var c := cs.Peek(); + if c == '{' as opt_byte then assume false; + var SP(obj, cs) :- Objects.Bracketed(cs, ValueParser(cs)); Objects.Valid(obj); - Success(SP(Grammar.Object(obj), ps)) - else if c == '[' as opt_byte then - var SP(arr, ps) :- Arrays.Bracketed(ps, ValueParser(ps)); + Success(SP(Grammar.Object(obj), cs)) + else if c == '[' as opt_byte then assume false; + var SP(arr, cs) :- Arrays.Bracketed(cs, ValueParser(cs)); Arrays.Valid(arr); - Success(SP(Grammar.Array(arr), ps)) + Success(SP(Grammar.Array(arr), cs)) else if c == '\"' as opt_byte then - var SP(str, ps) :- Strings.String(ps); - Success(SP(Grammar.String(str), ps)) + var SP(str, cs) :- Strings.String(cs); + Success(SP(Grammar.String(str), cs)) else if c == 't' as opt_byte then - var SP(cst, ps) :- Constants.Constant(ps, TRUE); - Success(SP(Grammar.Bool(cst), ps)) + var SP(cst, cs) :- Constants.Constant(cs, TRUE); + Success(SP(Grammar.Bool(cst), cs)) else if c == 'f' as opt_byte then - var SP(cst, ps) :- Constants.Constant(ps, FALSE); - Success(SP(Grammar.Bool(cst), ps)) + var SP(cst, cs) :- Constants.Constant(cs, FALSE); + Success(SP(Grammar.Bool(cst), cs)) else if c == 'n' as opt_byte then - var SP(cst, ps) :- Constants.Constant(ps, NULL); - Success(SP(Grammar.Null(cst), ps)) + var SP(cst, cs) :- Constants.Constant(cs, NULL); + Success(SP(Grammar.Null(cst), cs)) else - Numbers.Number(ps) + var SP(num, cs) :- Numbers.Number(cs); + Success(SP(Grammar.Number(num), cs)) } - function ValueParser(ps: FreshCursor) : (p: SubParser) - decreases ps.Length(), 0 - ensures ps.SplitFrom?(p.ps) + function ValueParser(cs: FreshCursor) : (p: ValueParser) + decreases cs.Length(), 0 + ensures cs.SplitFrom?(p.cs) { - var pre := (ps': FreshCursor) => ps'.Length() < ps.Length(); + var pre := (ps': FreshCursor) => ps'.Length() < cs.Length(); var fn := (ps': FreshCursor) requires pre(ps') => Value(ps'); - Parsers.SubParser(ps, pre, fn) + Parsers.SubParser(cs, pre, fn, Spec.Value) } } @@ -266,12 +263,12 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import opened Cursors - function Constant(ps: FreshCursor, expected: bytes) : (pr: ParseResult) + function Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) requires |expected| < TWO_TO_THE_32 - ensures pr.Success? ==> pr.value.t.Bytes() == expected + ensures pr.Success? ==> pr.value.SplitFrom?(cs, _ => expected) { - var ps :- ps.AssertBytes(expected); - Success(ps.Split()) + var cs :- cs.AssertBytes(expected); + Success(cs.Split()) } } @@ -285,13 +282,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Parsers import opened Core - function String(ps: FreshCursor): (pr: ParseResult) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function String(cs: FreshCursor): (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.String) { - var ps :- ps.AssertChar('\"'); - var ps :- ps.SkipWhileLexer(StringBody, Partial(StringBodyLexerStart)); - var ps :- ps.AssertChar('\"'); - Success(ps.Split()) + var cs :- cs.AssertChar('\"'); + var cs :- cs.SkipWhileLexer(StringBody, Partial(StringBodyLexerStart)); + var cs :- cs.AssertChar('\"'); + Success(cs.Split()) } } @@ -303,76 +300,97 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Cursors import opened Core - function Digits(ps: FreshCursor) - : (pp: Split) - ensures pp.ps.SplitFrom?(ps) + function Digits(cs: FreshCursor) + : (sp: Split) + ensures sp.cs.SplitFrom?(cs) { - ps.SkipWhile(Digit?).Split() + cs.SkipWhile(Digit?).Split() } - function NonEmptyDigits(ps: FreshCursor) : (pr: ParseResult) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function NonEmptyDigits(cs: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { - var pp := Digits(ps); - :- Need(!pp.t.Empty?, OtherError(EmptyNumber)); - Success(pp) + var sp := Digits(cs); + :- Need(!sp.t.Empty?, OtherError(EmptyNumber)); + Success(sp) } - function Minus(ps: FreshCursor) : (pp: Split) - ensures pp.ps.SplitFrom?(ps) + function OptionalMinus(cs: FreshCursor) : (sp: Split) + ensures sp.SplitFrom?(cs, SpecView) { - ps.SkipIf(c => c == '-' as byte).Split() + cs.SkipIf(c => c == '-' as byte).Split() } - function Sign(ps: FreshCursor) : (pp: Split) - ensures pp.ps.SplitFrom?(ps) + function OptionalSign(cs: FreshCursor) : (sp: Split) + ensures sp.SplitFrom?(cs, SpecView) { - ps.SkipIf(c => c == '-' as byte || c == '+' as byte).Split() + cs.SkipIf(c => c == '-' as byte || c == '+' as byte).Split() } - function TrimmedInt(ps: FreshCursor) : (pr: ParseResult) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) + function TrimmedInt(cs: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { - var pp := ps.SkipIf(c => c == '0' as byte).Split(); - if pp.t.Empty? then NonEmptyDigits(ps) - else Success(pp) + var sp := cs.SkipIf(c => c == '0' as byte).Split(); + if sp.t.Empty? then NonEmptyDigits(cs) + else Success(sp) } - function Exp(ps: FreshCursor) : (pr: ParseResult>) - ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) + function Exp(cs: FreshCursor) : (pr: ParseResult>) + ensures pr.Success? ==> pr.value.SplitFrom?(cs, exp => Spec.Maybe(exp, Spec.Exp)) { - var SP(e, ps) := - ps.SkipIf(c => c == 'e' as byte || c == 'E' as byte).Split(); + var SP(e, cs) := + cs.SkipIf(c => c == 'e' as byte || c == 'E' as byte).Split(); if e.Empty? then - Success(SP(Empty(), ps)) + Success(SP(Empty(), cs)) else assert e.Char?('e') || e.Char?('E'); - var SP(sign, ps) := Sign(ps); - var SP(num, ps) :- NonEmptyDigits(ps); - Success(SP(NonEmpty(JExp(e, sign, num)), ps)) + var SP(sign, cs) := OptionalSign(cs); + var SP(num, cs) :- NonEmptyDigits(cs); + Success(SP(NonEmpty(JExp(e, sign, num)), cs)) } - function Frac(ps: FreshCursor) : (pr: ParseResult>) - ensures pr.Success? ==> pr.value.ps.SplitFrom?(ps) + function Frac(cs: FreshCursor) : (pr: ParseResult>) + ensures pr.Success? ==> pr.value.SplitFrom?(cs, frac => Spec.Maybe(frac, Spec.Frac)) { - var SP(period, ps) := - ps.SkipIf(c => c == '.' as byte).Split(); + var SP(period, cs) := + cs.SkipIf(c => c == '.' as byte).Split(); if period.Empty? then - Success(SP(Empty(), ps)) + Success(SP(Empty(), cs)) else - var SP(num, ps) :- NonEmptyDigits(ps); - Success(SP(NonEmpty(JFrac(period, num)), ps)) + var SP(num, cs) :- NonEmptyDigits(cs); + Success(SP(NonEmpty(JFrac(period, num)), cs)) + } + + function NumberFromParts(ghost cs: Cursor, + minus: Split, num: Split, + frac: Split>, exp: Split>) + : (sp: Split) + requires minus.SplitFrom?(cs, SpecView) + requires num.StrictlySplitFrom?(minus.cs, SpecView) + requires frac.SplitFrom?(num.cs, frac => Spec.Maybe(frac, Spec.Frac)) + requires exp.SplitFrom?(frac.cs, exp => Spec.Maybe(exp, Spec.Exp)) + ensures sp.StrictlySplitFrom?(cs, Spec.Number) + { + var sp := SP(Grammar.JNumber(minus.t, num.t, frac.t, exp.t), exp.cs); + calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: + cs.Bytes(); + Spec.View(minus.t) + minus.cs.Bytes(); + Spec.View(minus.t) + Spec.View(num.t) + num.cs.Bytes(); + Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac) + frac.cs.Bytes(); + Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac) + Spec.Maybe(exp.t, Spec.Exp) + exp.cs.Bytes(); + Spec.Number(sp.t) + exp.cs.Bytes(); + } + sp } - function Number(ps: FreshCursor) : (pr: ParseResult) - ensures pr.Success? ==> pr.value.ps.StrictlySplitFrom?(ps) - ensures pr.Success? ==> pr.value.t.Number? + function Number(cs: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Number) { - var SP(minus, ps) := Minus(ps); - var SP(num, ps) :- TrimmedInt(ps); - var SP(frac, ps) :- Frac(ps); - var SP(exp, ps) :- Exp(ps); - Success(SP(Grammar.Number(minus, num, frac, exp), ps)) + var minus := OptionalMinus(cs); + var num :- TrimmedInt(minus.cs); + var frac :- Frac(num.cs); + var exp :- Exp(frac.cs); + Success(NumberFromParts(cs, minus, num, frac, exp)) } } @@ -386,14 +404,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const CLOSE := ']' as byte const SEPARATOR: byte := ',' as byte - function Element(ps: FreshCursor, json: SubParser) + function ElementSpec(t: TElement) : bytes { + Spec.Value(t) + } + function Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { - assert ps.StrictlySplitFrom?(json.ps); - assert json.Valid?(); - assert json.Valid?(); - assert json.fn.requires(ps); - json.fn(ps) + json.fn(cs) } } @@ -411,20 +428,42 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const CLOSE := '}' as byte const SEPARATOR: byte := ',' as byte - function Colon(ps: FreshCursor) + function Colon(cs: FreshCursor) : (pr: ParseResult) { - var ps :- ps.AssertChar(':'); - Success(ps.Split()) + var cs :- cs.AssertChar(':'); + Success(cs.Split()) + } + + function KVFromParts(ghost cs: Cursor, k: Split, + colon: Split>, v: Split) + : (sp: Split) + requires k.StrictlySplitFrom?(cs, Spec.String) + requires colon.StrictlySplitFrom?(k.cs, c => Spec.Structural(c, SpecView)) + requires v.StrictlySplitFrom?(colon.cs, Spec.Value) + ensures sp.StrictlySplitFrom?(cs, Spec.KV) + { + var sp := SP(Grammar.KV(k.t, colon.t, v.t), v.cs); + calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: + cs.Bytes(); + Spec.String(k.t) + k.cs.Bytes(); + Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + colon.cs.Bytes(); + Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + Spec.Value(v.t) + v.cs.Bytes(); + Spec.KV(sp.t) + v.cs.Bytes(); + } + sp } - function Element(ps: FreshCursor, json: SubParser) + function ElementSpec(t: TElement) : bytes { + Spec.KV(t) + } + function Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { - var SP(k, ps) :- Strings.String(ps); - var SP(colon, ps) :- Core.Structural(ps, Parsers.Parser(Colon)); - var SP(v, ps) :- json.fn(ps); - Success(SP(KV(k, colon, v), ps)) + var k :- Strings.String(cs); + var colon :- Core.Structural(k.cs, Parsers.Parser(Colon, SpecView)); + var v :- json.fn(colon.cs); + Success(KVFromParts(cs, k, colon, v)) } } diff --git a/src/JSON/JSON.Grammar.dfy b/src/JSON/JSON.Grammar.dfy index 4adbd114..00bc80b8 100644 --- a/src/JSON/JSON.Grammar.dfy +++ b/src/JSON/JSON.Grammar.dfy @@ -26,16 +26,16 @@ module {:options "-functionSyntax:4"} JSON.Grammar { datatype Maybe<+T> = Empty() | NonEmpty(t: T) - datatype Prefixed<+S, +T> = - Prefixed(prefix: Maybe>, t: T) + datatype Suffixed<+T, +S> = + Suffixed(t: T, suffix: Maybe>) - type PrefixedSequence<+S, +D> = s: seq> | NoLeadingPrefix(s) - ghost predicate NoLeadingPrefix(s: seq>) { - forall idx | 0 <= idx < |s| :: s[idx].prefix.Empty? <==> idx == 0 + type SuffixedSequence<+D, +S> = s: seq> | NoTrailingSuffix(s) + ghost predicate NoTrailingSuffix(s: seq>) { + forall idx | 0 <= idx < |s| :: s[idx].suffix.Empty? <==> idx == |s| - 1 } - datatype Bracketed<+L, +S, +D, +R> = - Bracketed(l: Structural, data: PrefixedSequence, r: Structural) + datatype Bracketed<+L, +D, +S, +R> = + Bracketed(l: Structural, data: SuffixedSequence, r: Structural) const NULL: bytes := ['n' as byte, 'u' as byte, 'l' as byte, 'l' as byte] const TRUE: bytes := ['t' as byte, 'r' as byte, 'u' as byte, 'e' as byte] @@ -48,7 +48,7 @@ module {:options "-functionSyntax:4"} JSON.Grammar { ghost predicate Num?(v: View) { Digits?(v) && !v.Empty? } ghost predicate Int?(v: View) { v.Char?('0') || (Num?(v) && v.At(0) != '0' as byte) } - type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce correct escaping + type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce quoting and escaping type jnull = v: View | Null?(v) witness View.OfBytes(NULL) type jbool = v: View | Bool?(v) witness View.OfBytes(TRUE) type jdigits = v: View | Digits?(v) witness View.OfBytes([]) @@ -56,21 +56,23 @@ module {:options "-functionSyntax:4"} JSON.Grammar { type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) datatype jkv = KV(k: jstring, colon: Structural, v: Value) - type jobject = Bracketed - type jarray = Bracketed - type jmembers = PrefixedSequence - type jmember = Prefixed - type jitems = PrefixedSequence - type jitem = Prefixed + // TODO enforce no leading space before closing bracket to disambiguate WS { WS WS } WS + type jobject = Bracketed + type jarray = Bracketed + type jmembers = SuffixedSequence + type jmember = Suffixed + type jitems = SuffixedSequence + type jitem = Suffixed datatype jfrac = JFrac(period: jperiod, num: jnum) datatype jexp = JExp(e: je, sign: jsign, num: jnum) + datatype jnumber = JNumber(minus: jminus, num: jnum, frac: Maybe, exp: Maybe) datatype Value = | Null(n: jnull) | Bool(b: jbool) | String(str: jstring) - | Number(minus: jminus, num: jnum, frac: Maybe, exp: Maybe) + | Number(num: jnumber) | Object(obj: jobject) | Array(arr: jarray) diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.Serializer.dfy index 54b1ba14..b36f6976 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/JSON.Serializer.dfy @@ -37,31 +37,31 @@ module {:options "-functionSyntax:4"} JSON.Serializer { ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { match v - case Null(n) => - writer.Append(n) - case Bool(b) => - writer.Append(b) - case String(str) => - writer.Append(str) - case Number(minus, num, frac, exp) => - Number(v, writer) - case Object(obj) => - Object(v, obj, writer) - case Array(arr) => - Array(v, arr, writer) + case Null(n) => writer.Append(n) + case Bool(b) => writer.Append(b) + case String(str) => String(str, writer) + case Number(num) => Number(num, writer) + case Object(obj) => Object(obj, writer) + case Array(arr) => Array(arr, writer) } - function {:opaque} Number(v: Value, writer: Writer) : (wr: Writer) - requires v.Number? - decreases v, 0 - ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) + function {:opaque} String(str: jstring, writer: Writer) : (wr: Writer) + decreases str, 0 + ensures wr.Bytes() == writer.Bytes() + Spec.String(str) + { + writer.Append(str) + } + + function {:opaque} Number(num: jnumber, writer: Writer) : (wr: Writer) + decreases num, 0 + ensures wr.Bytes() == writer.Bytes() + Spec.Number(num) { - var writer := writer.Append(v.minus).Append(v.num); - var writer := if v.frac.NonEmpty? then - writer.Append(v.frac.t.period).Append(v.frac.t.num) + var writer := writer.Append(num.minus).Append(num.num); + var writer := if num.frac.NonEmpty? then + writer.Append(num.frac.t.period).Append(num.frac.t.num) else writer; - var writer := if v.exp.NonEmpty? then - writer.Append(v.exp.t.e).Append(v.exp.t.sign).Append(v.exp.t.num) + var writer := if num.exp.NonEmpty? then + writer.Append(num.exp.t.e).Append(num.exp.t.sign).Append(num.exp.t.num) else writer; writer } @@ -77,145 +77,163 @@ module {:options "-functionSyntax:4"} JSON.Serializer { // FIXME refactor below to merge - function {:opaque} Object(v: Value, obj: jobject, writer: Writer) : (wr: Writer) - requires v.Object? && obj == v.obj - decreases v, 3 - ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) + function {:opaque} Object(obj: jobject, writer: Writer) : (wr: Writer) + decreases obj, 3 + ensures wr.Bytes() == writer.Bytes() + Spec.Object(obj) { var writer := StructuralView(obj.l, writer); - var writer := Members(v, obj, writer); + var writer := Members(obj, writer); var writer := StructuralView(obj.r, writer); // We call ``ConcatBytes`` with ``Spec.Member``, whereas the spec calls it // with ``(d: jmember) requires d in obj.data => Spec.Member(d)``. That's // why we need an explicit cast, which is performed by the lemma below. SpecProperties.Bracketed_Morphism(obj); - assert Spec.Value(v) == Spec.Bracketed(obj, Spec.Member); + assert Spec.Object(obj) == Spec.Bracketed(obj, Spec.Member); writer } - function {:opaque} Array(v: Value, arr: jarray, writer: Writer) : (wr: Writer) - requires v.Array? && arr == v.arr - decreases v, 3 - ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) + function {:opaque} Array(arr: jarray, writer: Writer) : (wr: Writer) + decreases arr, 3 + ensures wr.Bytes() == writer.Bytes() + Spec.Array(arr) { var writer := StructuralView(arr.l, writer); - var writer := Items(v, arr, writer); + var writer := Items(arr, writer); var writer := StructuralView(arr.r, writer); // We call ``ConcatBytes`` with ``Spec.Item``, whereas the spec calls it // with ``(d: jitem) requires d in arr.data => Spec.Item(d)``. That's // why we need an explicit cast, which is performed by the lemma below. SpecProperties.Bracketed_Morphism(arr); // DISCUSS - assert Spec.Value(v) == Spec.Bracketed(arr, Spec.Item); + assert Spec.Array(arr) == Spec.Bracketed(arr, Spec.Item); writer } - function {:opaque} Members(ghost v: Value, obj: jobject, writer: Writer) : (wr: Writer) - requires obj < v - decreases v, 2 + function {:opaque} Members(obj: jobject, writer: Writer) : (wr: Writer) + decreases obj, 2 ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(obj.data, Spec.Member) { - MembersSpec(v, obj.data, writer) + MembersSpec(obj, obj.data, writer) } by method { - wr := MembersImpl(v, obj, writer); - Assume(false); // FIXME + wr := MembersImpl(obj, writer); + Assume(false); // BUG(https://github.com/dafny-lang/dafny/issues/2180) } - function {:opaque} Items(ghost v: Value, arr: jarray, writer: Writer) : (wr: Writer) - requires arr < v - decreases v, 2 + function {:opaque} Items(arr: jarray, writer: Writer) : (wr: Writer) + decreases arr, 2 ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(arr.data, Spec.Item) { - ItemsSpec(v, arr.data, writer) + ItemsSpec(arr, arr.data, writer) } by method { - wr := ItemsImpl(v, arr, writer); - Assume(false); // FIXME + wr := ItemsImpl(arr, writer); + Assume(false); // BUG(https://github.com/dafny-lang/dafny/issues/2180) } - ghost function MembersSpec(v: Value, members: seq, writer: Writer) : (wr: Writer) - requires forall j | 0 <= j < |members| :: members[j] < v - decreases v, 1, members + ghost function MembersSpec(obj: jobject, members: seq, writer: Writer) : (wr: Writer) + requires forall j | 0 <= j < |members| :: members[j] < obj + decreases obj, 1, members ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(members, Spec.Member) { // TR elimination doesn't work for mutually recursive methods, so this // function is only used as a spec for Members. if members == [] then writer else - var writer := MembersSpec(v, members[..|members|-1], writer); + var writer := MembersSpec(obj, members[..|members|-1], writer); assert members == members[..|members|-1] + [members[|members|-1]]; SpecProperties.ConcatBytes_Linear(members[..|members|-1], [members[|members|-1]], Spec.Member); - Member(v, members[|members|-1], writer) + Member(obj, members[|members|-1], writer) } // No by method block here, because the loop invariant in the method version // needs to call MembersSpec and the termination checker gets confused by // that. Instead, see Members above. // DISCUSS - ghost function ItemsSpec(v: Value, items: seq, writer: Writer) : (wr: Writer) - requires forall j | 0 <= j < |items| :: items[j] < v + // DISCUSS: Is there a way to avoid passing the ghost `v` around while + // maintaining the termination argument? Maybe the lambda for elements will be enough? + + ghost function SequenceSpec(v: Value, items: seq, + spec: T -> bytes, impl: (ghost Value, T, Writer) --> Writer, + writer: Writer) + : (wr: Writer) + requires forall item, wr | item in items :: impl.requires(v, item, wr) + requires forall item, wr | item in items :: impl(v, item, wr).Bytes() == wr.Bytes() + spec(item) decreases v, 1, items + ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(items, spec) + { // TR elimination doesn't work for mutually recursive methods, so this + // function is only used as a spec for Items. + if items == [] then writer + else + var writer := SequenceSpec(v, items[..|items|-1], spec, impl, writer); + assert items == items[..|items|-1] + [items[|items|-1]]; + SpecProperties.ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], spec); + impl(v, items[|items|-1], writer) + } // No by method block here, because the loop invariant in the method version + // needs to call `SequenceSpec` and the termination checker gets confused by + // that. Instead, see `Sequence`Items above. // DISCUSS + + + ghost function ItemsSpec(arr: jarray, items: seq, writer: Writer) : (wr: Writer) + requires forall j | 0 <= j < |items| :: items[j] < arr + decreases arr, 1, items ensures wr.Bytes() == writer.Bytes() + Spec.ConcatBytes(items, Spec.Item) { // TR elimination doesn't work for mutually recursive methods, so this // function is only used as a spec for Items. if items == [] then writer else - var writer := ItemsSpec(v, items[..|items|-1], writer); + var writer := ItemsSpec(arr, items[..|items|-1], writer); assert items == items[..|items|-1] + [items[|items|-1]]; SpecProperties.ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], Spec.Item); - Item(v, items[|items|-1], writer) + Item(arr, items[|items|-1], writer) } // No by method block here, because the loop invariant in the method version // needs to call ItemsSpec and the termination checker gets confused by // that. Instead, see Items above. // DISCUSS - method MembersImpl(ghost v: Value, obj: jobject, writer: Writer) returns (wr: Writer) - requires obj < v - decreases v, 1 - ensures wr == MembersSpec(v, obj.data, writer); + method MembersImpl(obj: jobject, writer: Writer) returns (wr: Writer) + decreases obj, 1 + ensures wr == MembersSpec(obj, obj.data, writer); { wr := writer; var members := obj.data; - assert wr == MembersSpec(v, members[..0], writer); + assert wr == MembersSpec(obj, members[..0], writer); for i := 0 to |members| // FIXME uint32 - invariant wr == MembersSpec(v, members[..i], writer) + invariant wr == MembersSpec(obj, members[..i], writer) { assert members[..i+1][..i] == members[..i]; - wr := Member(v, members[i], wr); + wr := Member(obj, members[i], wr); } assert members[..|members|] == members; } - method ItemsImpl(ghost v: Value, arr: jarray, writer: Writer) returns (wr: Writer) - requires arr < v - decreases v, 1 - ensures wr == ItemsSpec(v, arr.data, writer); + method ItemsImpl(arr: jarray, writer: Writer) returns (wr: Writer) + decreases arr, 1 + ensures wr == ItemsSpec(arr, arr.data, writer); { wr := writer; var items := arr.data; - assert wr == ItemsSpec(v, items[..0], writer); + assert wr == ItemsSpec(arr, items[..0], writer); for i := 0 to |items| // FIXME uint32 - invariant wr == ItemsSpec(v, items[..i], writer) + invariant wr == ItemsSpec(arr, items[..i], writer) { assert items[..i+1][..i] == items[..i]; - wr := Item(v, items[i], wr); + wr := Item(arr, items[i], wr); } assert items[..|items|] == items; } - function {:opaque} Member(ghost v: Value, m: jmember, writer: Writer) : (wr: Writer) - requires m < v - decreases v, 0 + function {:opaque} Member(ghost obj: jobject, m: jmember, writer: Writer) : (wr: Writer) + requires m < obj + decreases obj, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Member(m) { - var writer := if m.prefix.Empty? then writer else StructuralView(m.prefix.t, writer); var writer := writer.Append(m.t.k); var writer := StructuralView(m.t.colon, writer); - Value(m.t.v, writer) + var writer := Value(m.t.v, writer); + if m.suffix.Empty? then writer else StructuralView(m.suffix.t, writer) } - function {:opaque} Item(ghost v: Value, m: jitem, writer: Writer) : (wr: Writer) - requires m < v - decreases v, 0 + function {:opaque} Item(ghost arr: jarray, m: jitem, writer: Writer) : (wr: Writer) + requires m < arr + decreases arr, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Item(m) { - var wr := if m.prefix.Empty? then writer else StructuralView(m.prefix.t, writer); - Value(m.t, wr) + var writer := Value(m.t, writer); + if m.suffix.Empty? then writer else StructuralView(m.suffix.t, writer) } } diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/JSON.Spec.dfy index 9650cd19..6e2f2ead 100644 --- a/src/JSON/JSON.Spec.dfy +++ b/src/JSON/JSON.Spec.dfy @@ -32,7 +32,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { else pt(ts[0]) + ConcatBytes(ts[1..], pt) } - function Bracketed(self: Bracketed, pdatum: Prefixed --> bytes): bytes + function Bracketed(self: Bracketed, pdatum: Suffixed --> bytes): bytes requires forall d | d in self.data :: pdatum.requires(d) { StructuralView(self.l) + @@ -52,30 +52,43 @@ module {:options "-functionSyntax:4"} JSON.Spec { View(self.e) + View(self.sign) + View(self.num) } - function Member(self: jmember) : bytes { + function Number(self: jnumber): bytes { + View(self.minus) + View(self.num) + Maybe(self.frac, Frac) + Maybe(self.exp, Exp) + } + + function String(self: jstring): bytes { + View(self) + } + + function CommaSuffix(c: Maybe>): bytes { // BUG(https://github.com/dafny-lang/dafny/issues/2179) - Maybe>(self.prefix, StructuralView) + KV(self.t) + Maybe>(c, StructuralView) + } + + function Member(self: jmember) : bytes { + KV(self.t) + CommaSuffix(self.suffix) } function Item(self: jitem) : bytes { - // BUG(https://github.com/dafny-lang/dafny/issues/2179) - Maybe>(self.prefix, StructuralView) + Value(self.t) + Value(self.t) + CommaSuffix(self.suffix) + } + + function Object(obj: jobject) : bytes { + Bracketed(obj, (d: jmember) requires d in obj.data => Member(d)) + } + + function Array(arr: jarray) : bytes { + Bracketed(arr, (d: jitem) requires d in arr.data => Item(d)) } - function Value(self: Value): bytes { + function Value(self: Value) : bytes { match self { - case Null(n) => - View(n) - case Bool(b) => - View(b) - case String(str) => - View(str) - case Number(minus, num, frac, exp) => - View(minus) + View(num) + Maybe(frac, Frac) + Maybe(exp, Exp) - case Object(obj) => - Bracketed(obj, (d: jmember) requires d in obj.data => Member(d)) - case Array(arr) => - Bracketed(arr, (d: jitem) requires d in arr.data => Item(d)) + case Null(n) => View(n) + case Bool(b) => View(b) + case String(str) => String(str) + case Number(num) => Number(num) + case Object(obj) => Object(obj) + case Array(arr) => Array(arr) } } diff --git a/src/JSON/JSON.SpecProperties.dfy b/src/JSON/JSON.SpecProperties.dfy index 3247acff..5e12e496 100644 --- a/src/JSON/JSON.SpecProperties.dfy +++ b/src/JSON/JSON.SpecProperties.dfy @@ -7,14 +7,14 @@ module {:options "-functionSyntax:4"} JSON.SpecProperties { import opened Grammar import Spec - lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS - ensures forall pd0: Prefixed --> bytes, pd1: Prefixed --> bytes + lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS + ensures forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d in bracketed.data :: pd0.requires(d)) && (forall d | d in bracketed.data :: pd1.requires(d)) && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) :: Spec.Bracketed(bracketed, pd0) == Spec.Bracketed(bracketed, pd1) { - forall pd0: Prefixed --> bytes, pd1: Prefixed --> bytes + forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d in bracketed.data :: pd0.requires(d)) && (forall d | d in bracketed.data :: pd1.requires(d)) && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) diff --git a/src/JSON/Parsers.dfy b/src/JSON/Parsers.dfy index 28730380..ec18561d 100644 --- a/src/JSON/Parsers.dfy +++ b/src/JSON/Parsers.dfy @@ -11,19 +11,20 @@ module {:options "-functionSyntax:4"} Parsers { type SplitResult<+T, +R> = CursorResult, R> - type Parser<+T, +R> = p: Parser_ | p.Valid?() + type Parser = p: Parser_ | p.Valid?() // BUG(https://github.com/dafny-lang/dafny/issues/2103) witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) - datatype Parser_<+T, +R> = Parser(fn: FreshCursor -> SplitResult) { + datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, + ghost spec: T -> bytes) { ghost predicate Valid?() { - forall ps': FreshCursor :: fn(ps').Success? ==> fn(ps').value.ps.StrictlySplitFrom?(ps') + forall cs': FreshCursor :: fn(cs').Success? ==> fn(cs').value.StrictlySplitFrom?(cs', spec) } } function {:opaque} ParserWitness(): (p: Parser_) ensures p.Valid?() { - Parser(_ => Failure(EOF)) + Parser(_ => Failure(EOF), _ => []) } // BUG(): It would be much nicer if `SubParser` was a special case of @@ -33,25 +34,27 @@ module {:options "-functionSyntax:4"} Parsers { // making it unprovable in the `SubParser` case (`fn` for subparsers is // typically a lambda, and the `requires` of lambdas are essentially // uninformative/opaque). - datatype SubParser_<+T, +R> = SubParser( - ghost ps: Cursor, + datatype SubParser_ = SubParser( + ghost cs: Cursor, ghost pre: FreshCursor -> bool, - fn: FreshCursor --> SplitResult) + fn: FreshCursor --> SplitResult, + ghost spec: T -> bytes) { ghost predicate Valid?() { - && (forall ps': FreshCursor | pre(ps') :: fn.requires(ps')) - && (forall ps': FreshCursor | ps'.StrictlySplitFrom?(ps) :: pre(ps')) - && (forall ps': FreshCursor | pre(ps') :: fn(ps').Success? ==> fn(ps').value.ps.StrictlySplitFrom?(ps')) + && (forall cs': FreshCursor | pre(cs') :: fn.requires(cs')) + && (forall cs': FreshCursor | cs'.StrictlySplitFrom?(cs) :: pre(cs')) + && (forall cs': FreshCursor | pre(cs') :: fn(cs').Success? ==> fn(cs').value.StrictlySplitFrom?(cs', spec)) } } - type SubParser<+T, +R> = p: SubParser_ | p.Valid?() + type SubParser = p: SubParser_ | p.Valid?() witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) function {:opaque} SubParserWitness(): (subp: SubParser_) ensures subp.Valid?() { SubParser(Cursor([], 0, 0, 0), - (ps: FreshCursor) => false, - (ps: FreshCursor) => Failure(EOF)) + (cs: FreshCursor) => false, + (cs: FreshCursor) => Failure(EOF), + _ => []) } } diff --git a/src/JSON/Views.dfy b/src/JSON/Views.dfy index e03f9b0c..5172ac3d 100644 --- a/src/JSON/Views.dfy +++ b/src/JSON/Views.dfy @@ -8,6 +8,9 @@ module {:options "-functionSyntax:4"} Views.Core { ghost const Valid?: bool := 0 <= beg as int <= end as int <= |s| < TWO_TO_THE_32; + static const Empty: View := + View([], 0, 0) + const Empty? := beg == end From 72708eb353ac40fcdcc58800de5700a6e0a83521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Mon, 30 May 2022 18:56:18 -0700 Subject: [PATCH 05/84] json: Complete soundness proof --- src/JSON/Cursors.dfy | 10 ++ src/JSON/JSON.Deserializer.dfy | 277 ++++++++++++++++++++++--------- src/JSON/JSON.Spec.dfy | 6 +- src/JSON/JSON.SpecProperties.dfy | 12 +- src/JSON/Parsers.dfy | 14 +- src/JSON/Views.dfy | 8 + 6 files changed, 231 insertions(+), 96 deletions(-) diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy index 73802931..7d8ac01a 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Cursors.dfy @@ -247,6 +247,16 @@ module {:options "-functionSyntax:4"} Cursors { AssertByte(c0 as byte) } + function SkipByte(): (ps: Cursor) + requires Valid? + decreases SuffixLength() + ensures ps.AdvancedFrom?(this) + ensures !EOF? ==> ps.StrictlyAdvancedFrom?(this) + { + if EOF? then this + else Skip(1) + } + function SkipIf(p: byte -> bool): (ps: Cursor) requires Valid? decreases SuffixLength() diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy index b586faf8..b0d4b5df 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/JSON.Deserializer.dfy @@ -1,5 +1,6 @@ include "JSON.Grammar.dfy" include "JSON.Spec.dfy" +include "JSON.SpecProperties.dfy" include "Parsers.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { @@ -33,8 +34,6 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { // BUG(https://github.com/dafny-lang/dafny/issues/2179) const SpecView := (v: Vs.View) => Spec.View(v); - // FIXME make more things opaque - function {:opaque} Get(cs: FreshCursor, err: JSONError): (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { @@ -48,7 +47,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { cs.SkipWhile(Blank?).Split() } - function Structural(cs: FreshCursor, parser: Parser) + function {:opaque} Structural(cs: FreshCursor, parser: Parser) : (pr: ParseResult>) requires forall cs :: parser.fn.requires(cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) @@ -59,6 +58,18 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(Grammar.Structural(before, val, after), cs)) } + type jopt = v: Vs.View | v.Length() <= 1 witness Vs.View.OfBytes([]) + + function {:opaque} TryStructural(cs: FreshCursor) + : (sp: Split>) + ensures sp.SplitFrom?(cs, st => Spec.Structural(st, SpecView)) + { + var SP(before, cs) := WS(cs); + var SP(val, cs) := cs.SkipByte().Split(); + var SP(after, cs) := WS(cs); + SP(Grammar.Structural(before, val, after), cs) + } + type ValueParser = sp: SubParser | forall t :: sp.spec(t) == Spec.Value(t) witness * @@ -73,15 +84,16 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const OPEN: byte const CLOSE: byte - const SEPARATOR: byte type TElement + ghost function ElementSpec(t: TElement) : bytes + function Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) requires cs.StrictlySplitFrom?(json.cs) decreases cs.Length() - ensures pr.Success? ==> pr.value.SplitFrom?(cs, ElementSpec) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, ElementSpec) } abstract module Sequences { @@ -89,18 +101,33 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened BoundedInts import opened Params: SequenceParams + import SpecProperties import opened Vs = Views.Core import opened Grammar import opened Cursors import Parsers import opened Core - type jopen = v: View | v.Byte?(OPEN) witness View.OfBytes([OPEN]) - type jclose = v: View | v.Byte?(CLOSE) witness View.OfBytes([CLOSE]) - type jsep = v: View | v.Byte?(SEPARATOR) witness View.OfBytes([SEPARATOR]) - type TSeq = Bracketed + const SEPARATOR: byte := ',' as byte + + type jopen = v: Vs.View | v.Byte?(OPEN) witness Vs.View.OfBytes([OPEN]) + type jclose = v: Vs.View | v.Byte?(CLOSE) witness Vs.View.OfBytes([CLOSE]) + type TBracketed = Bracketed + type TSuffixedElement = Suffixed - function Open(cs: FreshCursor) + ghost function SuffixedElementSpec(e: TSuffixedElement) : bytes { + ElementSpec(e.t) + Spec.CommaSuffix(e.suffix) + } + + ghost function BracketedSpec(ts: TBracketed): bytes { + Spec.Bracketed(ts, SuffixedElementSpec) + } + + ghost function SuffixedElementsSpec(ts: seq): bytes { + Spec.ConcatBytes(ts, SuffixedElementSpec) + } + + function {:opaque} Open(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, _ => [OPEN]) { @@ -108,7 +135,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(cs.Split()) } - function Close(cs: FreshCursor) + function {:opaque} Close(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, _ => [CLOSE]) { @@ -116,51 +143,118 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(cs.Split()) } - function {:tailrecursion} Elements( - cs: FreshCursor, json: ValueParser, - open: Structural, - items: seq> := [] - ) - : (pr: ParseResult) - requires cs.StrictlySplitFrom?(json.cs) - requires forall x | x in items :: x.suffix.NonEmpty? - decreases cs.Length() - ensures pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs) - // PROOF - { - var SP(item, cs) :- Element(cs, json); - var SP(sep, cs) :- Core.Structural(cs, - Parsers.Parser(cs => Get(cs, UnterminatedSequence), SpecView)); - var s0 := sep.t.At(0); - if s0 == CLOSE then - var items := items + [Suffixed(item, Empty())]; - Success(SP(Grammar.Bracketed(open, items, sep), cs)) - else if s0 == SEPARATOR then - var SP(item, cs) :- Element(cs, json); - var items := items + [Suffixed(item, NonEmpty(sep))]; - ghost var pr' := Elements(cs, json, open, items); // DISCUSS: Why is this needed? - assert pr'.Success? ==> pr'.value.cs.StrictlySplitFrom?(cs); - Elements(cs, json, open, items) + function {:opaque} BracketedFromParts(ghost cs: Cursor, + open: Split>, + elems: Split>, + close: Split>) + : (sp: Split) + requires Grammar.NoTrailingSuffix(elems.t) + requires open.StrictlySplitFrom?(cs, c => Spec.Structural(c, SpecView)) + requires elems.SplitFrom?(open.cs, SuffixedElementsSpec) + requires close.StrictlySplitFrom?(elems.cs, c => Spec.Structural(c, SpecView)) + ensures sp.StrictlySplitFrom?(cs, BracketedSpec) + { + var sp := SP(Grammar.Bracketed(open.t, elems.t, close.t), close.cs); + calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: + cs.Bytes(); + Spec.Structural(open.t, SpecView) + open.cs.Bytes(); + Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); + Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + Spec.Structural(close.t, SpecView) + close.cs.Bytes(); + Spec.Bracketed(sp.t, SuffixedElementSpec) + close.cs.Bytes(); + } + sp + } + + function {:opaque} AppendWithSuffix(ghost cs0: FreshCursor, + ghost json: ValueParser, + elems: Split>, + elem: Split, + sep: Split>) + : (elems': Split>) + requires elems.cs.StrictlySplitFrom?(json.cs) + requires elems.SplitFrom?(cs0, SuffixedElementsSpec) + requires elem.StrictlySplitFrom?(elems.cs, ElementSpec) + requires sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)) + requires forall e | e in elems.t :: e.suffix.NonEmpty? + ensures elems'.StrictlySplitFrom?(cs0, SuffixedElementsSpec) + ensures forall e | e in elems'.t :: e.suffix.NonEmpty? + ensures elems'.cs.Length() < elems.cs.Length() + ensures elems'.cs.StrictlySplitFrom?(json.cs) + { + var suffixed := Suffixed(elem.t, NonEmpty(sep.t)); + var elems' := SP(elems.t + [suffixed], sep.cs); // DISCUSS: Moving this down doubles the verification time + SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + elems' + } + + function {:opaque} AppendLast(ghost cs0: FreshCursor, + ghost json: ValueParser, + elems: Split>, + elem: Split, + sep: Split>) + : (elems': Split>) + requires elems.cs.StrictlySplitFrom?(json.cs) + requires elems.SplitFrom?(cs0, SuffixedElementsSpec) + requires elem.StrictlySplitFrom?(elems.cs, ElementSpec) + requires sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)) + requires forall e | e in elems.t :: e.suffix.NonEmpty? + ensures elems'.StrictlySplitFrom?(cs0, SuffixedElementsSpec) + ensures NoTrailingSuffix(elems'.t) + ensures elems'.cs.Length() < elems.cs.Length() + ensures elems'.cs.StrictlySplitFrom?(json.cs) + ensures sep.StrictlySplitFrom?(elems'.cs, c => Spec.Structural(c, SpecView)) + { + var suffixed := Suffixed(elem.t, Empty()); + var elems' := SP(elems.t + [suffixed], elem.cs); + SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + elems' + } + + // The implementation and proof of this function is more painful than + // expected due to the tail recursion. + function {:opaque} {:tailrecursion} Elements( + ghost cs0: FreshCursor, + json: ValueParser, + open: Split>, + elems: Split> + ) // DISCUSS: Why is this function reverified once per instantiation of the module? + : (pr: ParseResult) + requires open.StrictlySplitFrom?(cs0, c => Spec.Structural(c, SpecView)) + requires elems.cs.StrictlySplitFrom?(json.cs) + requires elems.SplitFrom?(open.cs, SuffixedElementsSpec) + requires forall e | e in elems.t :: e.suffix.NonEmpty? + decreases elems.cs.Length() + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs0, BracketedSpec) + { + var elem :- Element(elems.cs, json); + var sep := Core.TryStructural(elem.cs); + var s0 := sep.t.t.Peek(); + if s0 == SEPARATOR as opt_byte then + var elems := AppendWithSuffix(open.cs, json, elems, elem, sep); + Elements(cs0, json, open, elems) + else if s0 == CLOSE as opt_byte then + var elems := AppendLast(open.cs, json, elems, elem, sep); + Success(BracketedFromParts(cs0, open, elems, sep)) else - Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0 as opt_byte)) + Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0)) } - function Bracketed(cs: FreshCursor, json: ValueParser) - : (pr: ParseResult) + function {:opaque} Bracketed(cs: FreshCursor, json: ValueParser) + : (pr: ParseResult) requires cs.SplitFrom?(json.cs) - ensures pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, BracketedSpec) { - var SP(open, cs) :- Core.Structural(cs, Parsers.Parser(Open, SpecView)); + var open :- Core.Structural(cs, Parsers.Parser(Open, SpecView)); + assert open.cs.StrictlySplitFrom?(json.cs); + var elems := SP([], open.cs); if cs.Peek() == CLOSE as opt_byte then - var SP(close, cs) :- Core.Structural(cs, Parsers.Parser(Close, SpecView)); - Success(SP(Grammar.Bracketed(open, [], close), cs)) + var close :- Core.Structural(open.cs, Parsers.Parser(Close, SpecView)); + Success(BracketedFromParts(cs, open, elems, close)) else - ghost var pr := Elements(cs, json, open); // DISCUSS: Why is this needed? - assert pr.Success? ==> pr.value.cs.StrictlySplitFrom?(cs); - Elements(cs, json, open) + Elements(cs, json, open, elems) } - lemma Valid(x: TSeq) + lemma Valid(x: TBracketed) ensures x.l.t.Byte?(OPEN) ensures x.r.t.Byte?(CLOSE) ensures NoTrailingSuffix(x.data) @@ -188,13 +282,15 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import Values - function JSON(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} JSON(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.JSON) { Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)) } - function Text(v: View) : (pr: Result) { + function {:opaque} Text(v: View) : (pr: Result) + ensures pr.Success? ==> v.Bytes() == Spec.JSON(pr.value) + { var SP(text, cs) :- JSON(Cursor.OfView(v)); :- Need(cs.EOF?, OtherError(ExpectingEOF)); Success(text) @@ -215,18 +311,18 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import Arrays import Constants + import SpecProperties + function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) decreases cs.Length(), 1 ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Value) { var c := cs.Peek(); - if c == '{' as opt_byte then assume false; - var SP(obj, cs) :- Objects.Bracketed(cs, ValueParser(cs)); - Objects.Valid(obj); + if c == '{' as opt_byte then + var SP(obj, cs) :- Objects.Object(cs, ValueParser(cs)); Success(SP(Grammar.Object(obj), cs)) - else if c == '[' as opt_byte then assume false; - var SP(arr, cs) :- Arrays.Bracketed(cs, ValueParser(cs)); - Arrays.Valid(arr); + else if c == '[' as opt_byte then + var SP(arr, cs) :- Arrays.Array(cs, ValueParser(cs)); Success(SP(Grammar.Array(arr), cs)) else if c == '\"' as opt_byte then var SP(str, cs) :- Strings.String(cs); @@ -245,7 +341,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(Grammar.Number(num), cs)) } - function ValueParser(cs: FreshCursor) : (p: ValueParser) + function {:opaque} ValueParser(cs: FreshCursor) : (p: ValueParser) decreases cs.Length(), 0 ensures cs.SplitFrom?(p.cs) { @@ -263,8 +359,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Core import opened Cursors - function Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) + function {:opaque} Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) requires |expected| < TWO_TO_THE_32 + ensures pr.Success? ==> pr.value.t.Bytes() == expected ensures pr.Success? ==> pr.value.SplitFrom?(cs, _ => expected) { var cs :- cs.AssertBytes(expected); @@ -282,7 +379,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Parsers import opened Core - function String(cs: FreshCursor): (pr: ParseResult) + function {:opaque} String(cs: FreshCursor): (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.String) { var cs :- cs.AssertChar('\"'); @@ -300,14 +397,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Cursors import opened Core - function Digits(cs: FreshCursor) - : (sp: Split) - ensures sp.cs.SplitFrom?(cs) + function {:opaque} Digits(cs: FreshCursor) : (sp: Split) + ensures sp.SplitFrom?(cs, SpecView) { cs.SkipWhile(Digit?).Split() } - function NonEmptyDigits(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} NonEmptyDigits(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { var sp := Digits(cs); @@ -315,19 +411,19 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(sp) } - function OptionalMinus(cs: FreshCursor) : (sp: Split) + function {:opaque} OptionalMinus(cs: FreshCursor) : (sp: Split) ensures sp.SplitFrom?(cs, SpecView) { cs.SkipIf(c => c == '-' as byte).Split() } - function OptionalSign(cs: FreshCursor) : (sp: Split) + function {:opaque} OptionalSign(cs: FreshCursor) : (sp: Split) ensures sp.SplitFrom?(cs, SpecView) { cs.SkipIf(c => c == '-' as byte || c == '+' as byte).Split() } - function TrimmedInt(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} TrimmedInt(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { var sp := cs.SkipIf(c => c == '0' as byte).Split(); @@ -335,7 +431,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { else Success(sp) } - function Exp(cs: FreshCursor) : (pr: ParseResult>) + function {:opaque} Exp(cs: FreshCursor) : (pr: ParseResult>) ensures pr.Success? ==> pr.value.SplitFrom?(cs, exp => Spec.Maybe(exp, Spec.Exp)) { var SP(e, cs) := @@ -349,7 +445,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(NonEmpty(JExp(e, sign, num)), cs)) } - function Frac(cs: FreshCursor) : (pr: ParseResult>) + function {:opaque} Frac(cs: FreshCursor) : (pr: ParseResult>) ensures pr.Success? ==> pr.value.SplitFrom?(cs, frac => Spec.Maybe(frac, Spec.Frac)) { var SP(period, cs) := @@ -361,9 +457,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(SP(NonEmpty(JFrac(period, num)), cs)) } - function NumberFromParts(ghost cs: Cursor, - minus: Split, num: Split, - frac: Split>, exp: Split>) + function {:opaque} NumberFromParts( + ghost cs: Cursor, + minus: Split, num: Split, + frac: Split>, exp: Split> + ) : (sp: Split) requires minus.SplitFrom?(cs, SpecView) requires num.StrictlySplitFrom?(minus.cs, SpecView) @@ -383,7 +481,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { sp } - function Number(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} Number(cs: FreshCursor) : (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Number) { var minus := OptionalMinus(cs); @@ -402,13 +500,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const OPEN := '[' as byte const CLOSE := ']' as byte - const SEPARATOR: byte := ',' as byte function ElementSpec(t: TElement) : bytes { Spec.Value(t) } - function Element(cs: FreshCursor, json: ValueParser) - : (pr: ParseResult) + function {:opaque} Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { json.fn(cs) } @@ -416,6 +512,17 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { module Arrays refines Sequences { import opened Params = ArrayParams + + function {:opaque} Array(cs: FreshCursor, json: ValueParser) + : (pr: ParseResult) + requires cs.SplitFrom?(json.cs) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Array) + { + var sp :- Bracketed(cs, json); + SpecProperties.Bracketed_Morphism(sp.t); + assert Spec.Bracketed(sp.t, SuffixedElementSpec) == Spec.Array(sp.t); + Success(sp) + } } module ObjectParams refines SequenceParams { @@ -426,17 +533,16 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const OPEN := '{' as byte const CLOSE := '}' as byte - const SEPARATOR: byte := ',' as byte - function Colon(cs: FreshCursor) - : (pr: ParseResult) + function Colon(cs: FreshCursor) : (pr: ParseResult) // DISCUSS: Why can't I make this opaque? + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { var cs :- cs.AssertChar(':'); Success(cs.Split()) } - function KVFromParts(ghost cs: Cursor, k: Split, - colon: Split>, v: Split) + function {:opaque} KVFromParts(ghost cs: Cursor, k: Split, + colon: Split>, v: Split) : (sp: Split) requires k.StrictlySplitFrom?(cs, Spec.String) requires colon.StrictlySplitFrom?(k.cs, c => Spec.Structural(c, SpecView)) @@ -457,7 +563,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { function ElementSpec(t: TElement) : bytes { Spec.KV(t) } - function Element(cs: FreshCursor, json: ValueParser) + function {:opaque} Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { var k :- Strings.String(cs); @@ -469,5 +575,16 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { module Objects refines Sequences { import opened Params = ObjectParams + + function {:opaque} Object(cs: FreshCursor, json: ValueParser) + : (pr: ParseResult) + requires cs.SplitFrom?(json.cs) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Object) + { + var sp :- Bracketed(cs, json); + SpecProperties.Bracketed_Morphism(sp.t); + assert Spec.Bracketed(sp.t, SuffixedElementSpec) == Spec.Object(sp.t); // DISCUSS + Success(sp) + } } } diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/JSON.Spec.dfy index 6e2f2ead..1c5b7e5f 100644 --- a/src/JSON/JSON.Spec.dfy +++ b/src/JSON/JSON.Spec.dfy @@ -33,7 +33,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { } function Bracketed(self: Bracketed, pdatum: Suffixed --> bytes): bytes - requires forall d | d in self.data :: pdatum.requires(d) + requires forall d | d < self :: pdatum.requires(d) { StructuralView(self.l) + ConcatBytes(self.data, pdatum) + @@ -74,11 +74,11 @@ module {:options "-functionSyntax:4"} JSON.Spec { } function Object(obj: jobject) : bytes { - Bracketed(obj, (d: jmember) requires d in obj.data => Member(d)) + Bracketed(obj, (d: jmember) requires d < obj => Member(d)) } function Array(arr: jarray) : bytes { - Bracketed(arr, (d: jitem) requires d in arr.data => Item(d)) + Bracketed(arr, (d: jitem) requires d < arr => Item(d)) } function Value(self: Value) : bytes { diff --git a/src/JSON/JSON.SpecProperties.dfy b/src/JSON/JSON.SpecProperties.dfy index 5e12e496..1be6f700 100644 --- a/src/JSON/JSON.SpecProperties.dfy +++ b/src/JSON/JSON.SpecProperties.dfy @@ -9,15 +9,15 @@ module {:options "-functionSyntax:4"} JSON.SpecProperties { lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS ensures forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes - | && (forall d | d in bracketed.data :: pd0.requires(d)) - && (forall d | d in bracketed.data :: pd1.requires(d)) - && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + | && (forall d | d < bracketed :: pd0.requires(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) :: Spec.Bracketed(bracketed, pd0) == Spec.Bracketed(bracketed, pd1) { forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes - | && (forall d | d in bracketed.data :: pd0.requires(d)) - && (forall d | d in bracketed.data :: pd1.requires(d)) - && (forall d | d in bracketed.data :: pd0(d) == pd1(d)) + | && (forall d | d < bracketed :: pd0.requires(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) { calc { Spec.Bracketed(bracketed, pd0); diff --git a/src/JSON/Parsers.dfy b/src/JSON/Parsers.dfy index ec18561d..8c92a5cc 100644 --- a/src/JSON/Parsers.dfy +++ b/src/JSON/Parsers.dfy @@ -27,13 +27,13 @@ module {:options "-functionSyntax:4"} Parsers { Parser(_ => Failure(EOF), _ => []) } - // BUG(): It would be much nicer if `SubParser` was a special case of - // `Parser`, but that would require making `fn` in parser a partial - // function `-->`. The problem with that is that we would then have to - // restrict the `Valid?` clause of `Parser` on `fn.requires()`, thus - // making it unprovable in the `SubParser` case (`fn` for subparsers is - // typically a lambda, and the `requires` of lambdas are essentially - // uninformative/opaque). + // BUG(https://github.com/dafny-lang/dafny/issues/2137): It would be much + // nicer if `SubParser` was a special case of `Parser`, but that would require + // making `fn` in parser a partial function `-->`. The problem with that is + // that we would then have to restrict the `Valid?` clause of `Parser` on + // `fn.requires()`, thus making it unprovable in the `SubParser` case (`fn` + // for subparsers is typically a lambda, and the `requires` of lambdas are + // essentially uninformative/opaque). datatype SubParser_ = SubParser( ghost cs: Cursor, ghost pre: FreshCursor -> bool, diff --git a/src/JSON/Views.dfy b/src/JSON/Views.dfy index 5172ac3d..bcbc0f3d 100644 --- a/src/JSON/Views.dfy +++ b/src/JSON/Views.dfy @@ -53,6 +53,14 @@ module {:options "-functionSyntax:4"} Views.Core { s[beg + idx] } + function Peek(): (r: opt_byte) + requires Valid? + ensures r < 0 <==> Empty? + { + if Empty? then -1 + else At(0) as opt_byte + } + method Blit(bs: array, start: uint32 := 0) requires Valid? requires start as int + Length() as int <= bs.Length From 2a0c713a17ebe0d6172b9d32e11348626887271c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Tue, 31 May 2022 08:43:49 -0700 Subject: [PATCH 06/84] json: Add a clean low-level API --- src/JSON/Cursors.dfy | 8 +++- src/JSON/JSON.ZeroCopy.API.dfy | 44 +++++++++++++++++++ ...zer.dfy => JSON.ZeroCopy.Deserializer.dfy} | 32 +++++++++++--- ...lizer.dfy => JSON.ZeroCopy.Serializer.dfy} | 29 ++++++++++-- src/JSON/Tests.dfy | 15 +++---- src/JSON/Views.Writers.dfy | 13 +++++- 6 files changed, 119 insertions(+), 22 deletions(-) create mode 100644 src/JSON/JSON.ZeroCopy.API.dfy rename src/JSON/{JSON.Deserializer.dfy => JSON.ZeroCopy.Deserializer.dfy} (95%) rename src/JSON/{JSON.Serializer.dfy => JSON.ZeroCopy.Serializer.dfy} (90%) diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy index 7d8ac01a..afb53d19 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Cursors.dfy @@ -61,10 +61,16 @@ module {:options "-functionSyntax:4"} Cursors { const EOF? := point == end - static function OfView(v: View) : Cursor { + static function OfView(v: View) : FreshCursor { Cursor(v.s, v.beg, v.beg, v.end) } + static function OfBytes(bs: bytes) : FreshCursor + requires |bs| < TWO_TO_THE_32 + { + Cursor(bs, 0, |bs| as uint32, |bs| as uint32) + } + function Bytes() : bytes requires Valid? { diff --git a/src/JSON/JSON.ZeroCopy.API.dfy b/src/JSON/JSON.ZeroCopy.API.dfy new file mode 100644 index 00000000..9618c086 --- /dev/null +++ b/src/JSON/JSON.ZeroCopy.API.dfy @@ -0,0 +1,44 @@ +include "JSON.Grammar.dfy" +include "JSON.Spec.dfy" +include "JSON.ZeroCopy.Serializer.dfy" +include "JSON.ZeroCopy.Deserializer.dfy" + +module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { + import opened BoundedInts + import opened Wrappers + import Vs = Views.Core + + import opened Grammar + import Spec + import Serializer + import Deserializer + + function {:opaque} Serialize(js: JSON) : (bs: seq) + ensures bs == Spec.JSON(js) + { + Serializer.Text(js).Bytes() + } + + method SerializeAlloc(js: JSON) returns (bs: Result, Serializer.Error>) + ensures bs.Success? ==> fresh(bs.value) + ensures bs.Success? ==> bs.value[..] == Spec.JSON(js) + { + bs := Serializer.Serialize(js); + } + + method SerializeBlit(js: JSON, bs: array) returns (len: Result) + modifies bs + ensures len.Success? ==> len.value as int <= bs.Length + ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) + ensures len.Success? ==> bs[len.value..] == old(bs[len.value..]) + ensures len.Failure? ==> unchanged(bs) + { + len := Serializer.SerializeTo(js, bs); + } + + function {:opaque} Deserialize(bs: seq) : (js: Result) + ensures js.Success? ==> bs == Spec.JSON(js.value) + { + Deserializer.API.OfBytes(bs) + } +} diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.ZeroCopy.Deserializer.dfy similarity index 95% rename from src/JSON/JSON.Deserializer.dfy rename to src/JSON/JSON.ZeroCopy.Deserializer.dfy index b0d4b5df..1645055f 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Deserializer.dfy @@ -3,7 +3,7 @@ include "JSON.Spec.dfy" include "JSON.SpecProperties.dfy" include "Parsers.dfy" -module {:options "-functionSyntax:4"} JSON.Deserializer { +module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { module Core { import opened BoundedInts import opened Wrappers @@ -18,15 +18,17 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { | UnterminatedSequence | EmptyNumber | ExpectingEOF + | IntOverflow { function ToString() : string { match this case UnterminatedSequence => "Unterminated sequence" case EmptyNumber => "Number must contain at least one digit" case ExpectingEOF => "Expecting EOF" + case IntOverflow => "Input length does not fit in a 32-bit counter" } } - type ParseError = CursorError + type Error = CursorError type ParseResult<+T> = SplitResult type Parser = Parsers.Parser type SubParser = Parsers.SubParser @@ -74,6 +76,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { forall t :: sp.spec(t) == Spec.Value(t) witness * } + type Error = Core.Error abstract module SequenceParams { import opened BoundedInts @@ -247,7 +250,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { var open :- Core.Structural(cs, Parsers.Parser(Open, SpecView)); assert open.cs.StrictlySplitFrom?(json.cs); var elems := SP([], open.cs); - if cs.Peek() == CLOSE as opt_byte then + if open.cs.Peek() == CLOSE as opt_byte then var close :- Core.Structural(open.cs, Parsers.Parser(Close, SpecView)); Success(BracketedFromParts(cs, open, elems, close)) else @@ -273,7 +276,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { } } - module Top { + module API { + import opened BoundedInts import opened Wrappers import opened Vs = Views.Core @@ -288,13 +292,20 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)) } - function {:opaque} Text(v: View) : (pr: Result) - ensures pr.Success? ==> v.Bytes() == Spec.JSON(pr.value) + function {:opaque} Text(v: View) : (jsr: Result) + ensures jsr.Success? ==> v.Bytes() == Spec.JSON(jsr.value) { var SP(text, cs) :- JSON(Cursor.OfView(v)); :- Need(cs.EOF?, OtherError(ExpectingEOF)); Success(text) } + + function {:opaque} OfBytes(bs: bytes) : (jsr: Result) + ensures jsr.Success? ==> bs == Spec.JSON(jsr.value) + { + :- Need(|bs| < TWO_TO_THE_32, OtherError(IntOverflow)); + Text(Vs.View.OfBytes(bs)) + } } module Values { @@ -411,6 +422,13 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(sp) } + function {:opaque} NonZeroInt(cs: FreshCursor) : (pr: ParseResult) + requires cs.Peek() != '0' as opt_byte + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) + { + NonEmptyDigits(cs) + } + function {:opaque} OptionalMinus(cs: FreshCursor) : (sp: Split) ensures sp.SplitFrom?(cs, SpecView) { @@ -427,7 +445,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { var sp := cs.SkipIf(c => c == '0' as byte).Split(); - if sp.t.Empty? then NonEmptyDigits(cs) + if sp.t.Empty? then NonZeroInt(sp.cs) else Success(sp) } diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.ZeroCopy.Serializer.dfy similarity index 90% rename from src/JSON/JSON.Serializer.dfy rename to src/JSON/JSON.ZeroCopy.Serializer.dfy index b36f6976..8d87c288 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Serializer.dfy @@ -2,7 +2,7 @@ include "JSON.Spec.dfy" include "JSON.SpecProperties.dfy" include "Views.Writers.dfy" -module {:options "-functionSyntax:4"} JSON.Serializer { +module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import opened BoundedInts import opened Wrappers @@ -14,15 +14,36 @@ module {:options "-functionSyntax:4"} JSON.Serializer { datatype Error = OutOfMemory - method Serialize(js: JSON) returns (bsr: Result, Error>) - ensures bsr.Success? ==> bsr.value[..] == Spec.JSON(js) + method Serialize(js: JSON) returns (rbs: Result, Error>) + ensures rbs.Success? ==> fresh(rbs.value) + ensures rbs.Success? ==> rbs.value[..] == Spec.JSON(js) { - var writer := JSON(js); + var writer := Text(js); :- Need(writer.Unsaturated?, OutOfMemory); var bs := writer.ToArray(); return Success(bs); } + method SerializeTo(js: JSON, bs: array) returns (len: Result) + modifies bs + ensures len.Success? ==> len.value as int <= bs.Length + ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) + ensures len.Success? ==> bs[len.value..] == old(bs[len.value..]) + ensures len.Failure? ==> unchanged(bs) + { + var writer := Text(js); + :- Need(writer.Unsaturated?, OutOfMemory); + :- Need(writer.length as int <= bs.Length, OutOfMemory); + writer.Blit(bs); + return Success(writer.length); + } + + function {:opaque} Text(js: JSON) : (wr: Writer) + ensures wr.Bytes() == Spec.JSON(js) + { + JSON(js) + } + function {:opaque} JSON(js: JSON, writer: Writer := Writer.Empty) : (wr: Writer) ensures wr.Bytes() == writer.Bytes() + Spec.JSON(js) { diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 07975a86..3323e82c 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,14 +1,12 @@ -include "JSON.Serializer.dfy" -include "JSON.Deserializer.dfy" +include "JSON.ZeroCopy.API.dfy" include "../Collections/Sequences/Seq.dfy" import opened BoundedInts -import opened Vs = Views.Core - import opened JSON.Grammar -import JSON.Serializer -import JSON.Deserializer +import JSON.ZeroCopy.Serializer +import JSON.ZeroCopy.Deserializer +import JSON.ZeroCopy.API function method bytes_of_ascii(s: string) : bytes { Seq.Map((c: char) requires c in s => if c as int < 256 then c as byte else 0 as byte, s) @@ -43,15 +41,14 @@ method Main() { print input, "\n"; var bytes := bytes_of_ascii(input); - match Deserializer.Top.Text(View.OfBytes(bytes)) { + match API.Deserialize(bytes) { case Failure(msg) => print "Parse error: " + msg.ToString((e: Deserializer.Core.JSONError) => e.ToString()) + "\n"; expect false; case Success(js) => - // var bytes' := Grammar.Bytes(js); var wr := Serializer.JSON(js); print "Count: ", wr.chain.Count(), "\n"; - var rbytes' := Serializer.Serialize(js); + var rbytes' := API.SerializeAlloc(js); match rbytes' { case Failure(msg) => expect false; case Success(bytes') => diff --git a/src/JSON/Views.Writers.dfy b/src/JSON/Views.Writers.dfy index f5aa6378..4e92f66c 100644 --- a/src/JSON/Views.Writers.dfy +++ b/src/JSON/Views.Writers.dfy @@ -100,6 +100,17 @@ module {:options "-functionSyntax:4"} Views.Writers { fn(this) } + method {:tailrecursion} Blit(bs: array) + requires Valid? + requires Unsaturated? + requires Length() <= bs.Length + modifies bs + ensures bs[..length] == Bytes() + ensures bs[length..] == old(bs[length..]) + { + chain.Blit(bs, length); + } + method ToArray() returns (bs: array) requires Valid? requires Unsaturated? @@ -107,7 +118,7 @@ module {:options "-functionSyntax:4"} Views.Writers { ensures bs[..] == Bytes() { bs := new byte[length]; - chain.Blit(bs, length); + Blit(bs); } } } From 593b32ef787b285e61c12d9abf4075e32953c4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Wed, 1 Jun 2022 13:04:00 -0700 Subject: [PATCH 07/84] json: Speed things up using `by method` bodies in a few places --- src/JSON/Cursors.dfy | 33 ++++++++++++++++++++-- src/JSON/JSON.ZeroCopy.Deserializer.dfy | 37 ++++++++++++++++++++++++- src/JSON/Lexers.dfy | 28 ++++++++----------- 3 files changed, 79 insertions(+), 19 deletions(-) diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy index afb53d19..05e6b80f 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Cursors.dfy @@ -281,6 +281,16 @@ module {:options "-functionSyntax:4"} Cursors { { if EOF? || !p(SuffixAt(0)) then this else Skip(1).SkipWhile(p) + } by method { + var point' := this.point; + var end := this.end; + while point' < end && p(this.s[point']) + invariant this.(point := point').Valid? + invariant this.(point := point').SkipWhile(p) == this.SkipWhile(p) + { + point' := point' + 1; + } + return Cursor(this.s, this.beg, point', this.end); } function SkipWhileLexer(step: Lexer, st: LexerState) @@ -292,9 +302,28 @@ module {:options "-functionSyntax:4"} Cursors { match step(st, Peek()) case Accept => Success(this) case Reject(err) => Failure(OtherError(err)) - case partial => + case Partial(st) => if EOF? then Failure(EOF) - else Skip(1).SkipWhileLexer(step, partial) + else Skip(1).SkipWhileLexer(step, st) + } by method { + var point' := point; + var end := this.end; + var st' := st; + while true + invariant this.(point := point').Valid? + invariant this.(point := point').SkipWhileLexer(step, st') == this.SkipWhileLexer(step, st) + decreases this.(point := point').SuffixLength() + { + var eof := point' == end; + var minusone: opt_byte := -1; // BUG(https://github.com/dafny-lang/dafny/issues/2191) + var c := if eof then minusone else this.s[point'] as opt_byte; + match step(st', c) + case Accept => return Success(Cursor(this.s, this.beg, point', this.end)); + case Reject(err) => return Failure(OtherError(err)); + case Partial(st'') => + if eof { return Failure(EOF); } + else { st' := st''; point' := point' + 1; } + } } } } diff --git a/src/JSON/JSON.ZeroCopy.Deserializer.dfy b/src/JSON/JSON.ZeroCopy.Deserializer.dfy index 1645055f..04861317 100644 --- a/src/JSON/JSON.ZeroCopy.Deserializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Deserializer.dfy @@ -47,6 +47,17 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures sp.SplitFrom?(cs, SpecView) { cs.SkipWhile(Blank?).Split() + } by method { + reveal WS(); + var point' := cs.point; + var end := cs.end; + while point' < end && Blank?(cs.s[point']) + invariant cs.(point := point').Valid? + invariant cs.(point := point').SkipWhile(Blank?) == cs.SkipWhile(Blank?) + { + point' := point' + 1; + } + return Cursor(cs.s, cs.beg, point', cs.end).Split(); } function {:opaque} Structural(cs: FreshCursor, parser: Parser) @@ -382,6 +393,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { module Strings { import opened Wrappers + import opened BoundedInts import opened Grammar import opened Cursors @@ -390,11 +402,34 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Parsers import opened Core + function {:opaque} StringBody(cs: Cursor): (pr: CursorResult) + ensures pr.Success? ==> pr.value.AdvancedFrom?(cs) + { + cs.SkipWhileLexer(Strings.StringBody, StringBodyLexerStart) + } by method { + reveal StringBody(); + var escaped := false; + for point' := cs.point to cs.end + invariant cs.(point := point').Valid? + invariant cs.(point := point').SkipWhileLexer(Strings.StringBody, escaped) == StringBody(cs) + { + var byte := cs.s[point']; + if byte == '\"' as byte && !escaped { + return Success(Cursor(cs.s, cs.beg, point', cs.end)); + } else if byte == '\\' as byte { + escaped := !escaped; + } else { + escaped := false; + } + } + return Failure(EOF); + } + function {:opaque} String(cs: FreshCursor): (pr: ParseResult) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.String) { var cs :- cs.AssertChar('\"'); - var cs :- cs.SkipWhileLexer(StringBody, Partial(StringBodyLexerStart)); + var cs :- StringBody(cs); var cs :- cs.AssertChar('\"'); Success(cs.Split()) } diff --git a/src/JSON/Lexers.dfy b/src/JSON/Lexers.dfy index 4faf4cc7..aee4788f 100644 --- a/src/JSON/Lexers.dfy +++ b/src/JSON/Lexers.dfy @@ -6,9 +6,9 @@ module {:options "-functionSyntax:4"} Lexers { import opened Wrappers import opened BoundedInts - datatype LexerState<+T, +R> = Accept | Reject(err: R) | Partial(st: T) + datatype LexerResult<+T, +R> = Accept | Reject(err: R) | Partial(st: T) - type Lexer = (LexerState, opt_byte) -> LexerState + type Lexer = (T, opt_byte) -> LexerResult } module Strings { @@ -18,34 +18,30 @@ module {:options "-functionSyntax:4"} Lexers { type StringBodyLexerState = /* escaped: */ bool const StringBodyLexerStart: StringBodyLexerState := false; - function StringBody(st: LexerState, byte: opt_byte) - : LexerState + function StringBody(escaped: StringBodyLexerState, byte: opt_byte) + : LexerResult { - match st - case Partial(escaped) => - if byte == '\\' as opt_byte then Partial(!escaped) - else if byte == '\"' as opt_byte && !escaped then Accept - else Partial(false) - case _ => st + if byte == '\\' as opt_byte then Partial(!escaped) + else if byte == '\"' as opt_byte && !escaped then Accept + else Partial(false) } datatype StringLexerState = Start | Body(escaped: bool) | End const StringLexerStart: StringLexerState := Start; - function String(st: LexerState, byte: opt_byte) - : LexerState + function String(st: StringLexerState, byte: opt_byte) + : LexerResult { match st - case Partial(Start()) => + case Start() => if byte == '\"' as opt_byte then Partial(Body(false)) else Reject("String must start with double quote") - case Partial(End()) => + case End() => Accept - case Partial(Body(escaped)) => + case Body(escaped) => if byte == '\\' as opt_byte then Partial(Body(!escaped)) else if byte == '\"' as opt_byte && !escaped then Partial(End) else Partial(Body(false)) - case _ => st } } } From 4c50ce6bb6c381dc8f21394dddba6f3c318efaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Wed, 1 Jun 2022 13:05:51 -0700 Subject: [PATCH 08/84] json: Small cleanup in types --- src/JSON/Cursors.dfy | 16 ++++++++-------- src/JSON/Parsers.dfy | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/JSON/Cursors.dfy b/src/JSON/Cursors.dfy index 05e6b80f..c775e367 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Cursors.dfy @@ -49,7 +49,7 @@ module {:options "-functionSyntax:4"} Cursors { case OtherError(err) => pr(err) } } - type CursorResult<+T, +R> = Result> + type CursorResult<+R> = Result> datatype Cursor_ = Cursor(s: bytes, beg: uint32, point: uint32, end: uint32) { ghost const Valid?: bool := @@ -68,7 +68,7 @@ module {:options "-functionSyntax:4"} Cursors { static function OfBytes(bs: bytes) : FreshCursor requires |bs| < TWO_TO_THE_32 { - Cursor(bs, 0, |bs| as uint32, |bs| as uint32) + Cursor(bs, 0, 0, |bs| as uint32) } function Bytes() : bytes @@ -210,7 +210,7 @@ module {:options "-functionSyntax:4"} Cursors { this.(point := point - n) } - function Get(err: R): (ppr: CursorResult) + function Get(err: R): (ppr: CursorResult) requires Valid? ensures ppr.Success? ==> ppr.value.StrictlyAdvancedFrom?(this) { @@ -218,7 +218,7 @@ module {:options "-functionSyntax:4"} Cursors { else Success(Skip(1)) } - function AssertByte(b: byte): (pr: CursorResult) + function AssertByte(b: byte): (pr: CursorResult) requires Valid? ensures pr.Success? ==> !EOF? ensures pr.Success? ==> s[point] == b @@ -229,7 +229,7 @@ module {:options "-functionSyntax:4"} Cursors { else Failure(ExpectingByte(b, nxt)) } - function {:tailrecursion} AssertBytes(bs: bytes, offset: uint32 := 0): (pr: CursorResult) + function {:tailrecursion} AssertBytes(bs: bytes, offset: uint32 := 0): (pr: CursorResult) requires Valid? requires |bs| < TWO_TO_THE_32 requires offset <= |bs| as uint32 @@ -245,7 +245,7 @@ module {:options "-functionSyntax:4"} Cursors { ps.AssertBytes(bs, offset + 1) } - function AssertChar(c0: char): (pr: CursorResult) + function AssertChar(c0: char): (pr: CursorResult) requires Valid? requires c0 as int < 256 ensures pr.Success? ==> pr.value.StrictlyAdvancedFrom?(this) @@ -293,8 +293,8 @@ module {:options "-functionSyntax:4"} Cursors { return Cursor(this.s, this.beg, point', this.end); } - function SkipWhileLexer(step: Lexer, st: LexerState) - : (pr: CursorResult) + function SkipWhileLexer(step: Lexer, st: A) + : (pr: CursorResult) requires Valid? decreases SuffixLength() ensures pr.Success? ==> pr.value.AdvancedFrom?(this) diff --git a/src/JSON/Parsers.dfy b/src/JSON/Parsers.dfy index 8c92a5cc..7cd8de62 100644 --- a/src/JSON/Parsers.dfy +++ b/src/JSON/Parsers.dfy @@ -9,7 +9,7 @@ module {:options "-functionSyntax:4"} Parsers { import opened Views.Core import opened Cursors - type SplitResult<+T, +R> = CursorResult, R> + type SplitResult<+T, +R> = Result, CursorError> type Parser = p: Parser_ | p.Valid?() // BUG(https://github.com/dafny-lang/dafny/issues/2103) From 01983346cc4e6f465770d29b5a6aba4d4c0263bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Wed, 31 Aug 2022 10:43:43 -0700 Subject: [PATCH 09/84] json(wip): Start working on high-level API --- src/JSON/JSON.AST.dfy | 11 ++ src/JSON/JSON.Grammar.dfy | 37 ++++-- src/JSON/JSON.Serializer.dfy | 120 ++++++++++++++++++ src/JSON/JSON.Spec.HighLevel.dfy | 69 ++++++++++ .../{JSON.Spec.dfy => JSON.Spec.LowLevel.dfy} | 2 +- ...s.dfy => JSON.Spec.LowLevelProperties.dfy} | 4 +- src/JSON/JSON.ZeroCopy.API.dfy | 2 +- src/JSON/JSON.ZeroCopy.Deserializer.dfy | 4 +- src/JSON/JSON.ZeroCopy.Serializer.dfy | 4 +- src/JSON/Str.dfy | 88 +++++++++++++ src/JSON/UtfUtils.dfy | 73 +++++++++++ src/JSON/{Stacks.dfy => Vectors.dfy} | 76 +++++++++-- src/JSON/Views.dfy | 31 ++++- src/Math.dfy | 5 + 14 files changed, 495 insertions(+), 31 deletions(-) create mode 100644 src/JSON/JSON.AST.dfy create mode 100644 src/JSON/JSON.Serializer.dfy create mode 100644 src/JSON/JSON.Spec.HighLevel.dfy rename src/JSON/{JSON.Spec.dfy => JSON.Spec.LowLevel.dfy} (97%) rename src/JSON/{JSON.SpecProperties.dfy => JSON.Spec.LowLevelProperties.dfy} (94%) create mode 100644 src/JSON/Str.dfy create mode 100644 src/JSON/UtfUtils.dfy rename src/JSON/{Stacks.dfy => Vectors.dfy} (60%) diff --git a/src/JSON/JSON.AST.dfy b/src/JSON/JSON.AST.dfy new file mode 100644 index 00000000..3293bc10 --- /dev/null +++ b/src/JSON/JSON.AST.dfy @@ -0,0 +1,11 @@ +module {:options "-functionSyntax:4"} JSON.AST { + datatype KV = KV(k: string, v: Value) + datatype Decimal = Decimal(n: int, e10: nat) // (n) * 10^(e10) + datatype Value = + | Null + | Bool(b: bool) + | String(str: string) + | Number(num: Decimal) + | Object(obj: seq) // Not a map to preserve order + | Array(arr: seq) +} diff --git a/src/JSON/JSON.Grammar.dfy b/src/JSON/JSON.Grammar.dfy index 00bc80b8..dc003bc6 100644 --- a/src/JSON/JSON.Grammar.dfy +++ b/src/JSON/JSON.Grammar.dfy @@ -1,3 +1,9 @@ +/// ======================== +/// Low-level JSON grammar +/// ======================== +/// +/// See ``JSON.AST`` for the high-level interface. + include "../BoundedInts.dfy" include "Views.dfy" @@ -5,17 +11,28 @@ module {:options "-functionSyntax:4"} JSON.Grammar { import opened BoundedInts import opened Views.Core + const EMPTY := View.OfBytes([]) + const PERIOD := View.OfBytes(['.' as byte]) + const E := View.OfBytes(['e' as byte]) + const COLON := View.OfBytes([':' as byte]) + const COMMA := View.OfBytes([',' as byte]) + const LBRACE := View.OfBytes(['{' as byte]) + const RBRACE := View.OfBytes(['}' as byte]) + const LBRACKET := View.OfBytes(['[' as byte]) + const RBRACKET := View.OfBytes([']' as byte]) + const MINUS := View.OfBytes(['-' as byte]) + type jchar = v: View | v.Length() == 1 witness View.OfBytes(['b' as byte]) - type jperiod = v: View | v.Char?('.') witness View.OfBytes(['.' as byte]) - type je = v: View | v.Char?('e') || v.Char?('E') witness View.OfBytes(['e' as byte]) - type jcolon = v: View | v.Char?(':') witness View.OfBytes([':' as byte]) - type jcomma = v: View | v.Char?(',') witness View.OfBytes([',' as byte]) - type jlbrace = v: View | v.Char?('{') witness View.OfBytes(['{' as byte]) - type jrbrace = v: View | v.Char?('}') witness View.OfBytes(['}' as byte]) - type jlbracket = v: View | v.Char?('[') witness View.OfBytes(['[' as byte]) - type jrbracket = v: View | v.Char?(']') witness View.OfBytes([']' as byte]) - type jminus = v: View | v.Char?('-') || v.Empty? witness View.OfBytes([]) - type jsign = v: View | v.Char?('-') || v.Char?('+') || v.Empty? witness View.OfBytes([]) + type jperiod = v: View | v.Char?('.') witness PERIOD + type je = v: View | v.Char?('e') || v.Char?('E') witness E + type jcolon = v: View | v.Char?(':') witness COLON + type jcomma = v: View | v.Char?(',') witness COMMA + type jlbrace = v: View | v.Char?('{') witness LBRACE + type jrbrace = v: View | v.Char?('}') witness RBRACE + type jlbracket = v: View | v.Char?('[') witness LBRACKET + type jrbracket = v: View | v.Char?(']') witness RBRACKET + type jminus = v: View | v.Char?('-') || v.Empty? witness MINUS + type jsign = v: View | v.Char?('-') || v.Char?('+') || v.Empty? witness EMPTY predicate Blank?(b: byte) { b == 0x20 || b == 0x09 || b == 0x0A || b == 0x0D } ghost predicate Blanks?(v: View) { forall b | b in v.Bytes() :: Blank?(b) } diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.Serializer.dfy new file mode 100644 index 00000000..8ce12532 --- /dev/null +++ b/src/JSON/JSON.Serializer.dfy @@ -0,0 +1,120 @@ +/// ============================================= +/// Serialization from JSON.AST to JSON.Grammar +/// ============================================= +/// +/// For the spec, see ``JSON.Spec.HighLevel.dfy``. + +include "../Collections/Sequences/Seq.dfy" +include "../BoundedInts.dfy" +include "../Math.dfy" + +include "Views.dfy" +include "Vectors.dfy" +include "UtfUtils.dfy" +include "JSON.AST.dfy" +include "JSON.Grammar.dfy" +include "JSON.Spec.HighLevel.dfy" +include "JSON.Spec.LowLevel.dfy" + +module {:options "-functionSyntax:4"} JSON.Serializer { + import Seq + import Math + import opened BoundedInts + import opened Str + import UtfUtils + + import AST + import Spec.HighLevel + import opened Vectors + import opened Grammar + import opened Views.Core + + type bytes = seq + type string32 = s: string | |s| < TWO_TO_THE_32 + + function Bool(b: bool): jbool { + View.OfBytes(if b then TRUE else FALSE) + } + + method Utf8Encode(st: Vector, cp: uint32) + ensures st.Model() + { + + } + + function Transcode16To8Escaped(str: string32, start: uint32 := 0): seq { + UtfUtils.Transcode16To8(HighLevel.Escape(str)) + } by method { + var len := |str| as uint32; + if len == 0 { + return []; + } + var st := new Vectors.Vector(0, len); + var c0: uint16 := 0; + var c1: uint16 := str[0] as uint16; + var idx: uint32 := 0; + while idx < len { + var c0 := c1; + var c1 := str[idx + 1] as uint16; + if c0 < 0xD800 || c0 > 0xDBFF { + Utf8Encode(st, UtfUtils.Utf16Decode1(c0)); + idx := idx +1; + } else { + Utf8Encode(st, UtfUtils.Utf16Decode2(c0, c1)); + idx := idx + 2; + } + } + } + + + + + + function String(str: string): jstring { + Transcode16To8Escaped(str) + } + + function Sign(n: int): jminus { + View.OfBytes(if n < 0 then ['-' as byte] else []) + } + + function Number(dec: AST.Decimal): jnumber { + var minus: jminus := Sign(dec.n); + var num: jnum := View.OfBytes(Str.OfInt(Math.Abs(dec.n))); // FIXME + var frac: Maybe := Empty(); + var exp: jexp := + var e: je := View.OfBytes(['e' as byte]); + var sign: jsign := Sign(dec.e10); + var num: jnum := View.OfBytes(Str.OfInt(Math.Abs(dec.e10))); + JExp(e, sign, num); + JNumber(minus, num, Empty, NonEmpty(exp)) + } + + function KV(kv: AST.KV): Suffixed { + String(kv.k) + Bytes(", ") + Value(kv.v) + } + + function MkStructural(v: View): Structural { + Structural(EMPTY, v, EMPTY) + } + + function Object(obj: seq): jobject { + Bracketed(MkStructural(LBRACE), [], MkStructural(RBRACE)) + // Bytes("{") + Join(Bytes(","), Seq.Map(kv requires kv in obj => KV(kv), obj)) + Bytes("}") + } + + function Array(arr: seq): jarray { + Bracketed(MkStructural(LBRACKET), [], MkStructural(RBRACKET)) + // Bytes("[") + Join(Bytes(","), Seq.Map(v requires v in arr => Value(v), arr)) + Bytes("]") + } + + function Value(js: AST.Value): Value { + match js + case Null => Grammar.Null(View.OfBytes(NULL)) + case Bool(b) => Grammar.Bool(Bool(b)) + case String(str) => Grammar.String(String(str)) + case Number(dec) => Grammar.Number(Number(dec)) + case Object(obj) => Grammar.Object(Object(obj)) + case Array(arr) => Grammar.Array(Array(arr)) + } +} diff --git a/src/JSON/JSON.Spec.HighLevel.dfy b/src/JSON/JSON.Spec.HighLevel.dfy new file mode 100644 index 00000000..d3a1cc2e --- /dev/null +++ b/src/JSON/JSON.Spec.HighLevel.dfy @@ -0,0 +1,69 @@ +/// ============================================= +/// Serialization from AST.JSON to bytes (Spec) +/// ============================================= +/// +/// This is the high-level spec. For the implementation, see +/// ``JSON.Serializer.dfy``. + +include "../BoundedInts.dfy" + +include "JSON.AST.dfy" +include "UtfUtils.dfy" +include "Str.dfy" + +module {:options "-functionSyntax:4"} JSON.Spec.HighLevel { + import opened BoundedInts + + import opened Str + import opened AST + import opened UtfUtils + + type bytes = seq + + function Escape(str: string): string { + if str == [] then [] + else + (if str[0] == '\"' || str[0] == '\\' then ['\\', str[0]] else [str[0]]) + + Escape(str[1..]) + } + + function String(str: string): bytes { + Str.ToBytes("'") + Transcode16To8(Escape(str)) + Str.ToBytes("'") + } + + function Number(dec: Decimal): bytes { + Transcode16To8(Str.OfInt(dec.n)) + Str.ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10)) + } + + function KV(kv: KV): bytes { + String(kv.k) + Str.ToBytes(", ") + Value(kv.v) + } + + function Join(sep: bytes, items: seq): bytes { + if |items| == 0 then [] + else if |items| == 1 then items[0] + else items[0] + sep + Join(sep, items[1..]) + } + + function Object(obj: seq): bytes { + Str.ToBytes("{") + + Join(Str.ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KV(obj[i]))) + + Str.ToBytes("}") + } + + function Array(arr: seq): bytes { + Str.ToBytes("[") + + Join(Str.ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => Value(arr[i]))) + + Str.ToBytes("]") + } + + function Value(js: Value): seq { + match js + case Null => Str.ToBytes("null") + case Bool(b: bool) => if b then Str.ToBytes("true") else Str.ToBytes("false") + case String(str: string) => String(str) + case Number(dec: Decimal) => Number(dec) + case Object(obj: seq) => Object(obj) + case Array(arr: seq) => Array(arr) + } +} diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/JSON.Spec.LowLevel.dfy similarity index 97% rename from src/JSON/JSON.Spec.dfy rename to src/JSON/JSON.Spec.LowLevel.dfy index 1c5b7e5f..3fd57385 100644 --- a/src/JSON/JSON.Spec.dfy +++ b/src/JSON/JSON.Spec.LowLevel.dfy @@ -1,6 +1,6 @@ include "JSON.Grammar.dfy" -module {:options "-functionSyntax:4"} JSON.Spec { +module {:options "-functionSyntax:4"} JSON.LowLevel.Spec { import opened BoundedInts import Vs = Views.Core diff --git a/src/JSON/JSON.SpecProperties.dfy b/src/JSON/JSON.Spec.LowLevelProperties.dfy similarity index 94% rename from src/JSON/JSON.SpecProperties.dfy rename to src/JSON/JSON.Spec.LowLevelProperties.dfy index 1be6f700..1659643b 100644 --- a/src/JSON/JSON.SpecProperties.dfy +++ b/src/JSON/JSON.Spec.LowLevelProperties.dfy @@ -1,6 +1,6 @@ -include "JSON.Spec.dfy" +include "JSON.LowLevel.Spec.dfy" -module {:options "-functionSyntax:4"} JSON.SpecProperties { +module {:options "-functionSyntax:4"} JSON.LowLevel.SpecProperties { import opened BoundedInts import Vs = Views.Core diff --git a/src/JSON/JSON.ZeroCopy.API.dfy b/src/JSON/JSON.ZeroCopy.API.dfy index 9618c086..313eba75 100644 --- a/src/JSON/JSON.ZeroCopy.API.dfy +++ b/src/JSON/JSON.ZeroCopy.API.dfy @@ -1,5 +1,5 @@ include "JSON.Grammar.dfy" -include "JSON.Spec.dfy" +include "JSON.LowLevel.Spec.dfy" include "JSON.ZeroCopy.Serializer.dfy" include "JSON.ZeroCopy.Deserializer.dfy" diff --git a/src/JSON/JSON.ZeroCopy.Deserializer.dfy b/src/JSON/JSON.ZeroCopy.Deserializer.dfy index 04861317..98581ccf 100644 --- a/src/JSON/JSON.ZeroCopy.Deserializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Deserializer.dfy @@ -1,6 +1,6 @@ include "JSON.Grammar.dfy" -include "JSON.Spec.dfy" -include "JSON.SpecProperties.dfy" +include "JSON.LowLevel.Spec.dfy" +include "JSON.LowLevel.SpecProperties.dfy" include "Parsers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { diff --git a/src/JSON/JSON.ZeroCopy.Serializer.dfy b/src/JSON/JSON.ZeroCopy.Serializer.dfy index 8d87c288..3b2d0f24 100644 --- a/src/JSON/JSON.ZeroCopy.Serializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Serializer.dfy @@ -1,5 +1,5 @@ -include "JSON.Spec.dfy" -include "JSON.SpecProperties.dfy" +include "JSON.LowLevel.Spec.dfy" +include "JSON.LowLevel.SpecProperties.dfy" include "Views.Writers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { diff --git a/src/JSON/Str.dfy b/src/JSON/Str.dfy new file mode 100644 index 00000000..0e21c2d2 --- /dev/null +++ b/src/JSON/Str.dfy @@ -0,0 +1,88 @@ +include "../BoundedInts.dfy" + +module {:options "-functionSyntax:4"} Str { + module Private { + function Digits(n: int, base: int): (digits: seq) + requires base > 1 + requires n >= 0 + decreases n + ensures forall d | d in digits :: 0 <= d < base + { + if n == 0 then + [] + else + assert n > 0; + assert base > 1; + assert n < base * n; + assert n / base < n; + Digits(n / base, base) + [n % base] + } + + function OfDigits(digits: seq, chars: seq) : string + requires forall d | d in digits :: 0 <= d < |chars| + { + if digits == [] then "" + else + assert digits[0] in digits; + assert forall d | d in digits[1..] :: d in digits; + [chars[digits[0]]] + OfDigits(digits[1..], chars) + } + } + + function OfInt_any(n: int, chars: seq) : string + requires |chars| > 1 + { + var base := |chars|; + if n == 0 then + "0" + else if n > 0 then + Private.OfDigits(Private.Digits(n, base), chars) + else + "-" + Private.OfDigits(Private.Digits(-n, base), chars) + } + + const HEX_DIGITS := [ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'B', 'C', 'D', 'E', 'F'] + + function OfInt(n: int, base: int := 10) : string + requires 2 <= base <= 16 + { + OfInt_any(n, HEX_DIGITS[..base]) + } + + method Test() { // FIXME {:test}? + expect OfInt(0, 10) == "0"; + expect OfInt(3, 10) == "3"; + expect OfInt(302, 10) == "302"; + expect OfInt(-3, 10) == "-3"; + expect OfInt(-302, 10) == "-302"; + } + + function OfBool(b: bool) : string { + if b then "true" else "false" + } + + function OfChar(c: char) : string { + [c] + } + + function Join(sep: string, strs: seq) : string { + if |strs| == 0 then "" + else if |strs| == 1 then strs[0] + else strs[0] + sep + Join(sep, strs[1..]) + } + + function Concat(strs: seq) : string { + Join("", strs) + } + + import opened BoundedInts + + function ToBytes(s: string) : seq + requires forall c: char | c in s :: c as int < 256 + { + seq(|s|, i requires 0 <= i < |s| => + assert s[i] in s; s[i] as byte) + } +} diff --git a/src/JSON/UtfUtils.dfy b/src/JSON/UtfUtils.dfy new file mode 100644 index 00000000..7abe5b97 --- /dev/null +++ b/src/JSON/UtfUtils.dfy @@ -0,0 +1,73 @@ +include "../BoundedInts.dfy" + +module {:options "-functionSyntax:4"} UtfUtils { + import opened BoundedInts + + function Utf16Decode1(c: uint16): uint32 + requires c < 0xD800 || c > 0xDBFF + { + c as uint32 + } + + function Utf16Decode2(c0: uint16, c1: uint16): uint32 + requires 0xD800 <= c0 as int <= 0xDBFF + { + (0x10000 + | (((c0 as bv32) & 0x03FF) << 10) + | ((c1 as bv32) & 0x03FF)) + as uint32 + } + + newtype opt_uint16 = c: int | -1 <= c as int < TWO_TO_THE_16 + + function Utf16DecodeChars(c0: uint16, c1: opt_uint16): (r: (uint32, uint8)) + ensures r.1 in {1, 2} + ensures c1 == -1 ==> r.1 == 1 + { + if c0 < 0xD800 || c0 > 0xDBFF then + (Utf16Decode1(c0), 1) + else if c1 >= 0 then + (Utf16Decode2(c0, c1 as uint16), 2) + else + (0xFFFD, 1) + } + + function Utf16Decode(str: string): seq { + if str == [] then + [] + else + var c0 := str[0] as uint16; + var c1 := if |str| > 1 then str[1] as opt_uint16 else -1; + var (cp, delta) := Utf16DecodeChars(c0, c1); + [cp] + Utf16Decode(str[delta..]) + } + + function Utf8Encode1(cp: uint32): seq { + var bv := cp as bv32; + if cp < 0x80 then + [cp as uint8] + else if cp < 0x0800 then + [( (bv >> 6) & 0x1F) as uint8, + ( (bv & 0x3F) | 0x80) as uint8] + else if cp < 0x10000 then + [( (bv >> 12) & 0x0F) as uint8, + (((bv >> 6) & 0x3F) | 0x80) as uint8, + ( (bv & 0x3F) | 0x80) as uint8] + else if cp < 0x110000 then + [( (bv >> 18) & 0x07) as uint8, + (((bv >> 12) & 0x3F) | 0x80) as uint8, + (((bv >> 6) & 0x3F) | 0x80) as uint8, + ( (bv & 0x3F) | 0x80) as uint8] + else + [] // Invalid: drop // TODO + } + + function Utf8Encode(codepoints: seq): seq { + if codepoints == [] then [] + else Utf8Encode1(codepoints[0]) + Utf8Encode(codepoints[1..]) + } + + function Transcode16To8(s: string): seq { + Utf8Encode(Utf16Decode(s)) + } +} diff --git a/src/JSON/Stacks.dfy b/src/JSON/Vectors.dfy similarity index 60% rename from src/JSON/Stacks.dfy rename to src/JSON/Vectors.dfy index 0ba59a7f..6fa1f673 100644 --- a/src/JSON/Stacks.dfy +++ b/src/JSON/Vectors.dfy @@ -1,13 +1,13 @@ include "../BoundedInts.dfy" include "../Wrappers.dfy" -module {:options "-functionSyntax:4"} Stacks { +module {:options "-functionSyntax:4"} Vectors { import opened BoundedInts import opened Wrappers datatype Error = OutOfMemory - class Stack { + class Vector { ghost var Repr : seq const a: A @@ -67,7 +67,7 @@ module {:options "-functionSyntax:4"} Stacks { method Realloc(new_capacity: uint32) requires Valid?() requires new_capacity > capacity - modifies this, data + modifies this, `data ensures Valid?() ensures Repr == old(Repr) ensures size == old(size) @@ -79,6 +79,66 @@ module {:options "-functionSyntax:4"} Stacks { Blit(old_data, old_capacity); } + function DefaultNewCapacity(capacity: uint32) : uint32 { + if capacity < MAX_CAPACITY_BEFORE_DOUBLING + then 2 * capacity + else MAX_CAPACITY + } + + method ReallocDefault() returns (o: Outcome) + requires Valid?() + modifies this, `data + ensures Valid?() + ensures Repr == old(Repr) + ensures size == old(size) + ensures old(capacity) == MAX_CAPACITY <==> o.Fail? + ensures o.Fail? ==> + && unchanged(this) + && unchanged(data) + ensures o.Pass? ==> + && fresh(data) + && old(capacity) < MAX_CAPACITY + && capacity == old(if capacity < MAX_CAPACITY_BEFORE_DOUBLING + then 2 * capacity else MAX_CAPACITY) + + { + if capacity == MAX_CAPACITY { + return Fail(OutOfMemory); + } + Realloc(DefaultNewCapacity(capacity)); + return Pass; + } + + method Ensure(reserved: uint32) returns (o: Outcome) + requires Valid?() + modifies this, `data + ensures Valid?() + ensures Repr == old(Repr) + ensures size == old(size) + ensures reserved <= capacity - size ==> + o.Pass? + ensures o.Pass? ==> + old(size as int + reserved as int) <= capacity as int + ensures o.Fail? ==> + reserved > MAX_CAPACITY - size + { + if reserved > MAX_CAPACITY - size { + return Fail(OutOfMemory); + } + if reserved <= capacity - size { + return Pass; + } + var new_capacity := capacity; + while reserved > new_capacity - size + decreases MAX_CAPACITY - new_capacity + invariant new_capacity >= capacity + { + new_capacity := DefaultNewCapacity(new_capacity); + } + Realloc(new_capacity); + return Pass; + } + method PopFast(a: A) requires Valid?() requires size > 0 @@ -117,14 +177,8 @@ module {:options "-functionSyntax:4"} Stacks { && Repr == old(Repr) + [a] { if size == capacity { - if capacity < MAX_CAPACITY_BEFORE_DOUBLING { - Realloc(2 * capacity); - } else { - if capacity == MAX_CAPACITY { - return Fail(OutOfMemory); - } - Realloc(MAX_CAPACITY); - } + var d := ReallocDefault(); + if d.Fail? { return d; } } PushFast(a); return Pass; diff --git a/src/JSON/Views.dfy b/src/JSON/Views.dfy index bcbc0f3d..2ef504ec 100644 --- a/src/JSON/Views.dfy +++ b/src/JSON/Views.dfy @@ -29,6 +29,25 @@ module {:options "-functionSyntax:4"} Views.Core { View(bs, 0 as uint32, |bs| as uint32) } + static function OfString(s: string) : bytes + requires forall c: char | c in s :: c as int < 256 + { + seq(|s|, i requires 0 <= i < |s| => + assert s[i] in s; s[i] as byte) + } + + ghost predicate SliceOf?(v': View) { + v'.s == s && v'.beg <= beg && end <= v'.end + } + + ghost predicate StrictPrefixOf?(v': View) { + v'.s == s && v'.beg == beg && end < v'.end + } + + ghost predicate StrictSuffixOf?(v': View) { + v'.s == s && v'.beg < beg && end == v'.end + } + ghost predicate Byte?(c: byte) requires Valid? { @@ -79,8 +98,16 @@ module {:options "-functionSyntax:4"} Views.Core { } predicate Adjacent(lv: View, rv: View) { - lv.s == rv.s && - lv.end == rv.beg + // Compare endpoints first to short-circuit the potentially-costly string + // comparison + && lv.end == rv.beg + // We would prefer to use reference equality here, but doing so in a sound + // way is tricky (see chapter 9 of ‘Verasco: a Formally Verified C Static + // Analyzer’ by Jacques-Henri Jourdan for details). The runtime optimizes + // the common case of physical equality and otherwise performs a length + // check, so the worst case (checking for adjacency in two slices that have + // equal but not physically-equal contents) is hopefully not too common. + && lv.s == rv.s } function Merge(lv: View, rv: View) : View diff --git a/src/Math.dfy b/src/Math.dfy index b58d098b..17b0f2bc 100644 --- a/src/Math.dfy +++ b/src/Math.dfy @@ -22,4 +22,9 @@ module Math { a } + function method Abs(a: int): (a': int) + ensures a' >= 0 + { + if a >= 0 then a else -a + } } From a827a779b45ca6f21bf490945061f8672c5e0351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 1 Sep 2022 01:08:13 -0700 Subject: [PATCH 10/84] json: Clean up encoder and add decoder --- src/JSON/JSON.API.dfy | 53 ++++++ src/JSON/JSON.AST.dfy | 10 +- src/JSON/JSON.Deserializer.dfy | 106 +++++++++++ src/JSON/JSON.Errors.dfy | 30 ++++ ...ec.LowLevel.dfy => JSON.LowLevel.Spec.dfy} | 0 ...s.dfy => JSON.LowLevel.SpecProperties.dfy} | 0 src/JSON/JSON.Serializer.dfy | 170 ++++++++++++------ ...{JSON.Spec.HighLevel.dfy => JSON.Spec.dfy} | 33 ++-- src/JSON/JSON.ZeroCopy.API.dfy | 9 +- src/JSON/JSON.ZeroCopy.Deserializer.dfy | 22 +-- src/JSON/JSON.ZeroCopy.Serializer.dfy | 12 +- src/JSON/Str.dfy | 139 ++++++++++++-- src/JSON/UtfUtils.dfy | 87 +++++++-- src/JSON/Vectors.dfy | 8 +- src/JSON/Views.dfy | 6 +- src/Math.dfy | 5 + src/Wrappers.dfy | 4 +- 17 files changed, 553 insertions(+), 141 deletions(-) create mode 100644 src/JSON/JSON.API.dfy create mode 100644 src/JSON/JSON.Deserializer.dfy create mode 100644 src/JSON/JSON.Errors.dfy rename src/JSON/{JSON.Spec.LowLevel.dfy => JSON.LowLevel.Spec.dfy} (100%) rename src/JSON/{JSON.Spec.LowLevelProperties.dfy => JSON.LowLevel.SpecProperties.dfy} (100%) rename src/JSON/{JSON.Spec.HighLevel.dfy => JSON.Spec.dfy} (58%) diff --git a/src/JSON/JSON.API.dfy b/src/JSON/JSON.API.dfy new file mode 100644 index 00000000..490cdcd4 --- /dev/null +++ b/src/JSON/JSON.API.dfy @@ -0,0 +1,53 @@ +include "JSON.Errors.dfy" +include "JSON.Grammar.dfy" +include "JSON.Spec.dfy" +include "JSON.Serializer.dfy" +include "JSON.ZeroCopy.Serializer.dfy" + +module {:options "-functionSyntax:4"} JSON.API { + import opened BoundedInts + import opened Wrappers + import Vs = Views.Core + + import opened Errors + import opened AST + import Spec + import S = Serializer + import ZS = ZeroCopy.Serializer + // import Deserializer + + function {:opaque} Serialize(js: JSON) : (bs: SerializationResult>) + // TODO: Carry proofs all the way + // ensures bs.Success? ==> bs.value == Spec.JSON(js) + { + var js :- S.JSON(js); + Success(ZS.Text(js).Bytes()) + } + + method SerializeAlloc(js: JSON) returns (bs: SerializationResult>) + // TODO: Carry proofs all the way + // ensures bs.Success? ==> fresh(bs.value) + // ensures bs.Success? ==> bs.value[..] == Spec.JSON(js) + { + var js :- S.JSON(js); + bs := ZS.Serialize(js); + } + + method SerializeBlit(js: JSON, bs: array) returns (len: SerializationResult) + modifies bs + // TODO: Carry proofs all the way + // ensures len.Success? ==> len.value as int <= bs.Length + // ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) + // ensures len.Success? ==> bs[len.value..] == old(bs[len.value..]) + // ensures len.Failure? ==> unchanged(bs) + { + var js :- S.JSON(js); + len := ZS.SerializeTo(js, bs); + } + + // function {:opaque} Deserialize(bs: seq) : (js: Result) + // ensures js.Success? ==> bs == Spec.JSON(js.value) + // { + // Deserializer.API.OfBytes(bs) + // } +} diff --git a/src/JSON/JSON.AST.dfy b/src/JSON/JSON.AST.dfy index 3293bc10..0664c6a7 100644 --- a/src/JSON/JSON.AST.dfy +++ b/src/JSON/JSON.AST.dfy @@ -1,11 +1,11 @@ module {:options "-functionSyntax:4"} JSON.AST { - datatype KV = KV(k: string, v: Value) - datatype Decimal = Decimal(n: int, e10: nat) // (n) * 10^(e10) - datatype Value = + datatype Decimal = + Decimal(n: int, e10: int) // (n) * 10^(e10) + datatype JSON = | Null | Bool(b: bool) | String(str: string) | Number(num: Decimal) - | Object(obj: seq) // Not a map to preserve order - | Array(arr: seq) + | Object(obj: seq<(string, JSON)>) // Not a map to preserve order + | Array(arr: seq) } diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy new file mode 100644 index 00000000..040dfb2a --- /dev/null +++ b/src/JSON/JSON.Deserializer.dfy @@ -0,0 +1,106 @@ +/// =============================================== +/// Deserialization from JSON.Grammar to JSON.AST +/// =============================================== +/// +/// For the spec, see ``JSON.Spec.dfy``. + +include "../Collections/Sequences/Seq.dfy" +include "../BoundedInts.dfy" +include "../Math.dfy" + +include "Views.dfy" +include "Vectors.dfy" +include "UtfUtils.dfy" +include "JSON.Errors.dfy" +include "JSON.AST.dfy" +include "JSON.Grammar.dfy" +include "JSON.Spec.dfy" + +module {:options "-functionSyntax:4"} JSON.Deserializer { + import Seq + import Math + + import opened Wrappers + import opened BoundedInts + import opened Str + import UtfUtils + + import AST + import Spec + import opened Errors + import opened Vectors + import opened Grammar + import opened Views.Core + + function Bool(js: Grammar.jbool): bool { + assert js.Bytes() in {Grammar.TRUE, Grammar.FALSE}; + js.At(0) == 't' as byte + } + + function Transcode8To16Unescaped(str: seq): DeserializationResult + // TODO Optimize with a function by method + { // FIXME unescape unicode + Str.UnescapeQuotes(UtfUtils.Transcode8To16(str)).MapFailure(_ => EscapeAtEOS) + } + + function String(js: Grammar.jstring): DeserializationResult { + Transcode8To16Unescaped(js.Bytes()) + } + + module ByteStrConversion refines Str.ParametricConversion { + import opened BoundedInts + type Char = byte + } + + const DIGITS := map[ + '0' as uint8 := 0, '1' as uint8 := 1, '2' as uint8 := 2, '3' as uint8 := 3, + '4' as uint8 := 4, '5' as uint8 := 5, '6' as uint8 := 6, '7' as uint8 := 7, + '8' as uint8 := 8, '9' as uint8 := 9 + ] + + const MINUS := '-' as uint8 + + function ToInt(sign: jsign, n: jnum): DeserializationResult { + var n: int := ByteStrConversion.ToNat_any(n.Bytes(), 10, DIGITS); + Success(if sign.Char?('-') then -n else n) + } + + function Number(js: Grammar.jnumber): DeserializationResult { + var JNumber(minus, num, frac, exp) := js; + var n :- + ToInt(minus, num); + var e10 :- match exp + case Empty => Success(0) + case NonEmpty(JExp(_, sign, num)) => ToInt(sign, num); + match frac + case Empty => Success(AST.Decimal(n, e10)) + case NonEmpty(JFrac(_, num)) => + var pow10 := num.Length() as int; + var frac :- ToInt(View.Empty, num); + Success(AST.Decimal(n * Math.IntPow(10, pow10) + frac, e10 - pow10)) + } + + function KV(js: Grammar.jkv): DeserializationResult<(string, AST.JSON)> { + var k :- String(js.k); + var v :- Value(js.v); + Success((k, v)) + } + + function Object(js: Grammar.jobject): DeserializationResult> { + Seq.MapWithResult(d requires d in js.data => KV(d.t), js.data) + } + + function Array(js: Grammar.jarray): DeserializationResult> { + Seq.MapWithResult(d requires d in js.data => Value(d.t), js.data) + } + + function Value(js: Grammar.Value): DeserializationResult { + match js + case Null(_) => Success(AST.Null()) + case Bool(b) => Success(AST.Bool(Bool(b))) + case String(str) => var s :- String(str); Success(AST.String(s)) + case Number(dec) => var n :- Number(dec); Success(AST.Number(n)) + case Object(obj) => var o :- Object(obj); Success(AST.Object(o)) + case Array(arr) => var a :- Array(arr); Success(AST.Array(a)) + } +} diff --git a/src/JSON/JSON.Errors.dfy b/src/JSON/JSON.Errors.dfy new file mode 100644 index 00000000..531b9f91 --- /dev/null +++ b/src/JSON/JSON.Errors.dfy @@ -0,0 +1,30 @@ +include "../Wrappers.dfy" + +module {:options "-functionSyntax:4"} JSON.Errors { + import Wrappers + + datatype DeserializationError = + | UnterminatedSequence + | EscapeAtEOS + | EmptyNumber + | ExpectingEOF + | IntOverflow + { + function ToString() : string { + match this + case EscapeAtEOS => "Escape character at end of string" + case UnterminatedSequence => "Unterminated sequence" + case EmptyNumber => "Number must contain at least one digit" + case ExpectingEOF => "Expecting EOF" + case IntOverflow => "Input length does not fit in a 32-bit counter" + } + } + + datatype SerializationError = + | OutOfMemory + | IntTooLarge(i: int) + | StringTooLong(s: string) + + type SerializationResult<+T> = Wrappers.Result + type DeserializationResult<+T> = Wrappers.Result +} diff --git a/src/JSON/JSON.Spec.LowLevel.dfy b/src/JSON/JSON.LowLevel.Spec.dfy similarity index 100% rename from src/JSON/JSON.Spec.LowLevel.dfy rename to src/JSON/JSON.LowLevel.Spec.dfy diff --git a/src/JSON/JSON.Spec.LowLevelProperties.dfy b/src/JSON/JSON.LowLevel.SpecProperties.dfy similarity index 100% rename from src/JSON/JSON.Spec.LowLevelProperties.dfy rename to src/JSON/JSON.LowLevel.SpecProperties.dfy diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.Serializer.dfy index 8ce12532..1b88417b 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/JSON.Serializer.dfy @@ -2,7 +2,7 @@ /// Serialization from JSON.AST to JSON.Grammar /// ============================================= /// -/// For the spec, see ``JSON.Spec.HighLevel.dfy``. +/// For the spec, see ``JSON.Spec.dfy``. include "../Collections/Sequences/Seq.dfy" include "../BoundedInts.dfy" @@ -11,110 +11,168 @@ include "../Math.dfy" include "Views.dfy" include "Vectors.dfy" include "UtfUtils.dfy" +include "JSON.Errors.dfy" include "JSON.AST.dfy" include "JSON.Grammar.dfy" -include "JSON.Spec.HighLevel.dfy" -include "JSON.Spec.LowLevel.dfy" +include "JSON.Spec.dfy" module {:options "-functionSyntax:4"} JSON.Serializer { import Seq import Math + import opened Wrappers import opened BoundedInts import opened Str import UtfUtils import AST - import Spec.HighLevel + import Spec + import opened Errors import opened Vectors import opened Grammar import opened Views.Core + type Result<+T> = SerializationResult + type bytes = seq + type bytes32 = bs: bytes | |bs| < TWO_TO_THE_32 type string32 = s: string | |s| < TWO_TO_THE_32 function Bool(b: bool): jbool { View.OfBytes(if b then TRUE else FALSE) } - method Utf8Encode(st: Vector, cp: uint32) - ensures st.Model() - { + // method Utf8Encode(st: Vector, cp: uint32) + function Transcode16To8Escaped(str: string32, start: uint32 := 0): bytes { + UtfUtils.Transcode16To8(Str.EscapeQuotes(str)) + } + // by method { + // var len := |str| as uint32; + // if len == 0 { + // return []; + // } + // var st := new Vectors.Vector(0, len); + // var c0: uint16 := 0; + // var c1: uint16 := str[0] as uint16; + // var idx: uint32 := 0; + // while idx < len { + // var c0 := c1; + // var c1 := str[idx + 1] as uint16; + // if c0 < 0xD800 || c0 > 0xDBFF { + // Utf8Encode(st, UtfUtils.Utf16Decode1(c0)); + // idx := idx +1; + // } else { + // Utf8Encode(st, UtfUtils.Utf16Decode2(c0, c1)); + // idx := idx + 2; + // } + // } + // } + + function CheckLength(s: seq, err: SerializationError): Outcome { + Need(|s| < TWO_TO_THE_32, err) } - function Transcode16To8Escaped(str: string32, start: uint32 := 0): seq { - UtfUtils.Transcode16To8(HighLevel.Escape(str)) - } by method { - var len := |str| as uint32; - if len == 0 { - return []; - } - var st := new Vectors.Vector(0, len); - var c0: uint16 := 0; - var c1: uint16 := str[0] as uint16; - var idx: uint32 := 0; - while idx < len { - var c0 := c1; - var c1 := str[idx + 1] as uint16; - if c0 < 0xD800 || c0 > 0xDBFF { - Utf8Encode(st, UtfUtils.Utf16Decode1(c0)); - idx := idx +1; - } else { - Utf8Encode(st, UtfUtils.Utf16Decode2(c0, c1)); - idx := idx + 2; - } - } + function String(str: string): Result { + :- CheckLength(str, StringTooLong(str)); + var bs := Transcode16To8Escaped(str); + :- CheckLength(bs, StringTooLong(str)); + Success(View.OfBytes(bs)) } + function Sign(n: int): jminus { + View.OfBytes(if n < 0 then ['-' as byte] else []) + } + module ByteStrConversion refines Str.ParametricConversion { + import opened BoundedInts + type Char = uint8 + } + const DIGITS := [ + '0' as uint8, '1' as uint8, '2' as uint8, '3' as uint8, + '4' as uint8, '5' as uint8, '6' as uint8, '7' as uint8, + '8' as uint8, '9' as uint8 + ] + const MINUS := '-' as uint8 - function String(str: string): jstring { - Transcode16To8Escaped(str) + function Int'(n: int) : (str: bytes) + ensures forall c | c in str :: c in DIGITS || c == MINUS + { + ByteStrConversion.OfInt_any(n, DIGITS, MINUS) } - function Sign(n: int): jminus { - View.OfBytes(if n < 0 then ['-' as byte] else []) + function Int(n: int) : Result { + var bs := Int'(n); + :- CheckLength(bs, IntTooLarge(n)); + Success(View.OfBytes(bs)) } - function Number(dec: AST.Decimal): jnumber { + function Number(dec: AST.Decimal): Result { var minus: jminus := Sign(dec.n); - var num: jnum := View.OfBytes(Str.OfInt(Math.Abs(dec.n))); // FIXME + var num: jnum :- Int(Math.Abs(dec.n)); var frac: Maybe := Empty(); - var exp: jexp := + var exp: jexp :- var e: je := View.OfBytes(['e' as byte]); var sign: jsign := Sign(dec.e10); - var num: jnum := View.OfBytes(Str.OfInt(Math.Abs(dec.e10))); - JExp(e, sign, num); - JNumber(minus, num, Empty, NonEmpty(exp)) + var num: jnum :- Int(Math.Abs(dec.e10)); + Success(JExp(e, sign, num)); + Success(JNumber(minus, num, Empty, NonEmpty(exp))) } - function KV(kv: AST.KV): Suffixed { - String(kv.k) + Bytes(", ") + Value(kv.v) + function MkStructural(v: T): Structural { + Structural(EMPTY, v, EMPTY) } - function MkStructural(v: View): Structural { - Structural(EMPTY, v, EMPTY) + const COLON: Structural := + MkStructural(Grammar.COLON) + + function KV(kv: (string, AST.JSON)): Result { + var k :- String(kv.0); + var v :- Value(kv.1); + Success(Grammar.KV(k, COLON, v)) + } + + function MkSuffixedSequence(ds: seq, suffix: Structural) + : SuffixedSequence + { + match |ds| + case 0 => [] + case 1 => [Suffixed(ds[0], Empty)] + case _ => [Suffixed(ds[0], NonEmpty(suffix))] + + MkSuffixedSequence(ds[1..], suffix) } - function Object(obj: seq): jobject { - Bracketed(MkStructural(LBRACE), [], MkStructural(RBRACE)) - // Bytes("{") + Join(Bytes(","), Seq.Map(kv requires kv in obj => KV(kv), obj)) + Bytes("}") + const COMMA: Structural := + MkStructural(Grammar.COMMA) + + function Object(obj: seq<(string, AST.JSON)>): Result { + var items :- Seq.MapWithResult(v requires v in obj => KV(v), obj); + Success(Bracketed(MkStructural(LBRACE), + MkSuffixedSequence(items, COMMA), + MkStructural(RBRACE))) } - function Array(arr: seq): jarray { - Bracketed(MkStructural(LBRACKET), [], MkStructural(RBRACKET)) - // Bytes("[") + Join(Bytes(","), Seq.Map(v requires v in arr => Value(v), arr)) + Bytes("]") + + function Array(arr: seq): Result { + var items :- Seq.MapWithResult(v requires v in arr => Value(v), arr); + Success(Bracketed(MkStructural(LBRACKET), + MkSuffixedSequence(items, COMMA), + MkStructural(RBRACKET))) } - function Value(js: AST.Value): Value { + function Value(js: AST.JSON): Result { match js - case Null => Grammar.Null(View.OfBytes(NULL)) - case Bool(b) => Grammar.Bool(Bool(b)) - case String(str) => Grammar.String(String(str)) - case Number(dec) => Grammar.Number(Number(dec)) - case Object(obj) => Grammar.Object(Object(obj)) - case Array(arr) => Grammar.Array(Array(arr)) + case Null => Success(Grammar.Null(View.OfBytes(NULL))) + case Bool(b) => Success(Grammar.Bool(Bool(b))) + case String(str) => var s :- String(str); Success(Grammar.String(s)) + case Number(dec) => var n :- Number(dec); Success(Grammar.Number(n)) + case Object(obj) => var o :- Object(obj); Success(Grammar.Object(o)) + case Array(arr) => var a :- Array(arr); Success(Grammar.Array(a)) + } + + function JSON(js: AST.JSON): Result { + var val :- Value(js); + Success(MkStructural(val)) } } diff --git a/src/JSON/JSON.Spec.HighLevel.dfy b/src/JSON/JSON.Spec.dfy similarity index 58% rename from src/JSON/JSON.Spec.HighLevel.dfy rename to src/JSON/JSON.Spec.dfy index d3a1cc2e..7c75e7ab 100644 --- a/src/JSON/JSON.Spec.HighLevel.dfy +++ b/src/JSON/JSON.Spec.dfy @@ -11,7 +11,7 @@ include "JSON.AST.dfy" include "UtfUtils.dfy" include "Str.dfy" -module {:options "-functionSyntax:4"} JSON.Spec.HighLevel { +module {:options "-functionSyntax:4"} JSON.Spec { import opened BoundedInts import opened Str @@ -20,23 +20,16 @@ module {:options "-functionSyntax:4"} JSON.Spec.HighLevel { type bytes = seq - function Escape(str: string): string { - if str == [] then [] - else - (if str[0] == '\"' || str[0] == '\\' then ['\\', str[0]] else [str[0]]) - + Escape(str[1..]) - } - function String(str: string): bytes { - Str.ToBytes("'") + Transcode16To8(Escape(str)) + Str.ToBytes("'") + Str.ToBytes("'") + Transcode16To8(Str.EscapeQuotes(str)) + Str.ToBytes("'") } function Number(dec: Decimal): bytes { Transcode16To8(Str.OfInt(dec.n)) + Str.ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10)) } - function KV(kv: KV): bytes { - String(kv.k) + Str.ToBytes(", ") + Value(kv.v) + function KV(kv: (string, JSON)): bytes { + String(kv.0) + Str.ToBytes(", ") + JSON(kv.1) } function Join(sep: bytes, items: seq): bytes { @@ -45,25 +38,25 @@ module {:options "-functionSyntax:4"} JSON.Spec.HighLevel { else items[0] + sep + Join(sep, items[1..]) } - function Object(obj: seq): bytes { + function Object(obj: seq<(string, JSON)>): bytes { Str.ToBytes("{") + Join(Str.ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KV(obj[i]))) + Str.ToBytes("}") } - function Array(arr: seq): bytes { + function Array(arr: seq): bytes { Str.ToBytes("[") + - Join(Str.ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => Value(arr[i]))) + + Join(Str.ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => JSON(arr[i]))) + Str.ToBytes("]") } - function Value(js: Value): seq { + function JSON(js: JSON): bytes { match js case Null => Str.ToBytes("null") - case Bool(b: bool) => if b then Str.ToBytes("true") else Str.ToBytes("false") - case String(str: string) => String(str) - case Number(dec: Decimal) => Number(dec) - case Object(obj: seq) => Object(obj) - case Array(arr: seq) => Array(arr) + case Bool(b) => if b then Str.ToBytes("true") else Str.ToBytes("false") + case String(str) => String(str) + case Number(dec) => Number(dec) + case Object(obj) => Object(obj) + case Array(arr) => Array(arr) } } diff --git a/src/JSON/JSON.ZeroCopy.API.dfy b/src/JSON/JSON.ZeroCopy.API.dfy index 313eba75..a775b21a 100644 --- a/src/JSON/JSON.ZeroCopy.API.dfy +++ b/src/JSON/JSON.ZeroCopy.API.dfy @@ -1,3 +1,4 @@ +include "JSON.Errors.dfy" include "JSON.Grammar.dfy" include "JSON.LowLevel.Spec.dfy" include "JSON.ZeroCopy.Serializer.dfy" @@ -9,7 +10,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { import Vs = Views.Core import opened Grammar - import Spec + import LowLevel.Spec import Serializer import Deserializer @@ -19,14 +20,14 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { Serializer.Text(js).Bytes() } - method SerializeAlloc(js: JSON) returns (bs: Result, Serializer.Error>) + method SerializeAlloc(js: JSON) returns (bs: SerializationResult>) ensures bs.Success? ==> fresh(bs.value) ensures bs.Success? ==> bs.value[..] == Spec.JSON(js) { bs := Serializer.Serialize(js); } - method SerializeBlit(js: JSON, bs: array) returns (len: Result) + method SerializeBlit(js: JSON, bs: array) returns (len: SerializationResult) modifies bs ensures len.Success? ==> len.value as int <= bs.Length ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) @@ -36,7 +37,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { len := Serializer.SerializeTo(js, bs); } - function {:opaque} Deserialize(bs: seq) : (js: Result) + function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) ensures js.Success? ==> bs == Spec.JSON(js.value) { Deserializer.API.OfBytes(bs) diff --git a/src/JSON/JSON.ZeroCopy.Deserializer.dfy b/src/JSON/JSON.ZeroCopy.Deserializer.dfy index 98581ccf..7975f98a 100644 --- a/src/JSON/JSON.ZeroCopy.Deserializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Deserializer.dfy @@ -1,3 +1,4 @@ +include "JSON.Errors.dfy" include "JSON.Grammar.dfy" include "JSON.LowLevel.Spec.dfy" include "JSON.LowLevel.SpecProperties.dfy" @@ -8,26 +9,13 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Wrappers - import Spec + import LowLevel.Spec import Vs = Views.Core import opened Cursors import opened Parsers import opened Grammar - datatype JSONError = - | UnterminatedSequence - | EmptyNumber - | ExpectingEOF - | IntOverflow - { - function ToString() : string { - match this - case UnterminatedSequence => "Unterminated sequence" - case EmptyNumber => "Number must contain at least one digit" - case ExpectingEOF => "Expecting EOF" - case IntOverflow => "Input length does not fit in a 32-bit counter" - } - } + type JSONError = Errors.DeserializationError type Error = CursorError type ParseResult<+T> = SplitResult type Parser = Parsers.Parser @@ -115,7 +103,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Params: SequenceParams - import SpecProperties + import LowLevel.SpecProperties import opened Vs = Views.Core import opened Grammar import opened Cursors @@ -333,7 +321,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import Arrays import Constants - import SpecProperties + import LowLevel.SpecProperties function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) decreases cs.Length(), 1 diff --git a/src/JSON/JSON.ZeroCopy.Serializer.dfy b/src/JSON/JSON.ZeroCopy.Serializer.dfy index 3b2d0f24..4ad9f0c0 100644 --- a/src/JSON/JSON.ZeroCopy.Serializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Serializer.dfy @@ -1,3 +1,4 @@ +include "JSON.Errors.dfy" include "JSON.LowLevel.Spec.dfy" include "JSON.LowLevel.SpecProperties.dfy" include "Views.Writers.dfy" @@ -6,15 +7,14 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import opened BoundedInts import opened Wrappers - import Spec - import SpecProperties + import opened Errors + import LowLevel.Spec + import LowLevel.SpecProperties import opened Grammar import opened Views.Writers import opened Vs = Views.Core // DISCUSS: Module naming convention? - datatype Error = OutOfMemory - - method Serialize(js: JSON) returns (rbs: Result, Error>) + method Serialize(js: JSON) returns (rbs: SerializationResult>) ensures rbs.Success? ==> fresh(rbs.value) ensures rbs.Success? ==> rbs.value[..] == Spec.JSON(js) { @@ -24,7 +24,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { return Success(bs); } - method SerializeTo(js: JSON, bs: array) returns (len: Result) + method SerializeTo(js: JSON, bs: array) returns (len: SerializationResult) modifies bs ensures len.Success? ==> len.value as int <= bs.Length ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) diff --git a/src/JSON/Str.dfy b/src/JSON/Str.dfy index 0e21c2d2..efcc5bf5 100644 --- a/src/JSON/Str.dfy +++ b/src/JSON/Str.dfy @@ -1,7 +1,17 @@ include "../BoundedInts.dfy" +include "../Wrappers.dfy" module {:options "-functionSyntax:4"} Str { - module Private { + import opened Wrappers + + abstract module ParametricConversion { + import opened Wrappers + + type Char(==) + type String = seq + + // FIXME the design in LittleEndianNat makes BASE a module-level constant + // instead of a function argument function Digits(n: int, base: int): (digits: seq) requires base > 1 requires n >= 0 @@ -18,37 +28,136 @@ module {:options "-functionSyntax:4"} Str { Digits(n / base, base) + [n % base] } - function OfDigits(digits: seq, chars: seq) : string + function OfDigits(digits: seq, chars: seq) : (str: String) requires forall d | d in digits :: 0 <= d < |chars| + ensures forall c | c in str :: c in chars { - if digits == [] then "" + if digits == [] then [] else assert digits[0] in digits; assert forall d | d in digits[1..] :: d in digits; [chars[digits[0]]] + OfDigits(digits[1..], chars) } + + function OfNat_any(n: nat, chars: seq) : (str: String) + requires |chars| > 1 + ensures forall c | c in str :: c in chars + { + var base := |chars|; + if n == 0 then [chars[0]] + else OfDigits(Digits(n, base), chars) + } + + predicate NumberStr(str: String, minus: Char, is_digit: Char -> bool) { + str != [] ==> + && (str[0] == minus || is_digit(str[0])) + && forall c | c in str[1..] :: is_digit(c) + } + + function OfInt_any(n: int, chars: seq, minus: Char) : (str: String) + requires |chars| > 1 + ensures NumberStr(str, minus, c => c in chars) + { + if n >= 0 then OfNat_any(n, chars) + else [minus] + OfNat_any(-n, chars) + } + + function DigitsTable(digits: seq): map + requires forall i, j | 0 <= i < j < |digits| :: digits[i] != digits[j] + { + map i: nat | 0 <= i < |digits| :: digits[i] := i + } + + function ToNat_any(str: String, base: nat, digits: map) : (n: nat) + requires forall c | c in str :: c in digits + requires base > 0 + { + if str == [] then 0 + else ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]] + } + + function ToInt_any(str: String, minus: Char, base: nat, digits: map) : (s: int) + requires NumberStr(str, minus, c => c in digits) + requires str != [minus] + requires base > 0 + { + if [minus] <= str then -(ToNat_any(str[1..], base, digits) as int) + else + assert str == [] || str == [str[0]] + str[1..]; + ToNat_any(str, base, digits) + } } - function OfInt_any(n: int, chars: seq) : string - requires |chars| > 1 - { - var base := |chars|; - if n == 0 then - "0" - else if n > 0 then - Private.OfDigits(Private.Digits(n, base), chars) - else - "-" + Private.OfDigits(Private.Digits(-n, base), chars) + abstract module ParametricEscaping { + import opened Wrappers + + type Char(==) + type String = seq + + function Escape(str: String, special: set, escape: Char): String { + if str == [] then str + else if str[0] in special then [escape, str[0]] + Escape(str[1..], special, escape) + else [str[0]] + Escape(str[1..], special, escape) + } + + datatype UnescapeError = + EscapeAtEOS + + function Unescape(str: String, escape: Char): Result { + if str == [] then Success(str) + else if str[0] == escape then + if |str| > 1 then var tl :- Unescape(str[2..], escape); Success([str[1]] + tl) + else Failure(EscapeAtEOS) + else var tl :- Unescape(str[1..], escape); Success([str[0]] + tl) + } + + lemma {:induction false} Unescape_Escape(str: String, special: set, escape: Char) + requires escape in special + ensures Unescape(Escape(str, special, escape), escape) == Success(str) + { + if str == [] { + } else { + assert str == [str[0]] + str[1..]; + Unescape_Escape(str[1..], special, escape); + } + } + } + + module CharStrConversion refines ParametricConversion { + type Char = char + } + + module CharStrEscaping refines ParametricEscaping { + type Char = char } const HEX_DIGITS := [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'] + const HEX_TABLE := + CharStrConversion.DigitsTable(HEX_DIGITS); + + function OfInt(n: int, base: int := 10) : (str: string) + requires 2 <= base <= 16 + ensures CharStrConversion.NumberStr(str, '-', c => c in HEX_DIGITS[..base]) + { + CharStrConversion.OfInt_any(n, HEX_DIGITS[..base], '-') + } - function OfInt(n: int, base: int := 10) : string + function ToInt(str: string, base: int := 10) : (n: int) + requires str != "-" requires 2 <= base <= 16 + requires CharStrConversion.NumberStr(str, '-', c => c in HEX_DIGITS[..base]) { - OfInt_any(n, HEX_DIGITS[..base]) + CharStrConversion.ToInt_any(str, '-', base, HEX_TABLE) + } + + function EscapeQuotes(str: string): string { + CharStrEscaping.Escape(str, {'\"', '\''}, '\\') + } + + function UnescapeQuotes(str: string): Result { + CharStrEscaping.Unescape(str, '\\') } method Test() { // FIXME {:test}? diff --git a/src/JSON/UtfUtils.dfy b/src/JSON/UtfUtils.dfy index 7abe5b97..82827bb2 100644 --- a/src/JSON/UtfUtils.dfy +++ b/src/JSON/UtfUtils.dfy @@ -4,7 +4,7 @@ module {:options "-functionSyntax:4"} UtfUtils { import opened BoundedInts function Utf16Decode1(c: uint16): uint32 - requires c < 0xD800 || c > 0xDBFF + requires c < 0xD800 || 0xDBFF < c { c as uint32 } @@ -13,8 +13,8 @@ module {:options "-functionSyntax:4"} UtfUtils { requires 0xD800 <= c0 as int <= 0xDBFF { (0x10000 - | (((c0 as bv32) & 0x03FF) << 10) - | ((c1 as bv32) & 0x03FF)) + + ((((c0 as bv32) & 0x03FF) << 10) + | ((c1 as bv32) & 0x03FF))) as uint32 } @@ -24,12 +24,12 @@ module {:options "-functionSyntax:4"} UtfUtils { ensures r.1 in {1, 2} ensures c1 == -1 ==> r.1 == 1 { - if c0 < 0xD800 || c0 > 0xDBFF then + if c0 < 0xD800 || 0xDBFF < c0 then (Utf16Decode1(c0), 1) else if c1 >= 0 then (Utf16Decode2(c0, c1 as uint16), 2) else - (0xFFFD, 1) + (0xFFFD, 1) // Replacement character } function Utf16Decode(str: string): seq { @@ -42,19 +42,41 @@ module {:options "-functionSyntax:4"} UtfUtils { [cp] + Utf16Decode(str[delta..]) } - function Utf8Encode1(cp: uint32): seq { + function Utf16Encode2(cp: uint32): seq + requires cp < 0x100000 + { + var bv := cp as bv32; + [(0xD800 | (bv >> 10)) as char, + (0xDC00 | (bv & 0x03FF)) as char] + } + + function Utf16Encode1(cp: uint32): seq { + if cp < 0xD800 || 0xDBFF < cp < 0x10000 then + [cp as char] + else if 0x10000 <= cp < 0x110000 then + Utf16Encode2(cp - 0x10000) + else + [] // Invalid: drop // TODO + } + + function Utf16Encode(codepoints: seq): seq { + if codepoints == [] then [] + else Utf16Encode1(codepoints[0]) + Utf16Encode(codepoints[1..]) + } + + function Utf8Encode1(cp: uint32): seq { var bv := cp as bv32; if cp < 0x80 then [cp as uint8] else if cp < 0x0800 then - [( (bv >> 6) & 0x1F) as uint8, + [(((bv >> 6) & 0x1F) | 0xC0) as uint8, ( (bv & 0x3F) | 0x80) as uint8] else if cp < 0x10000 then - [( (bv >> 12) & 0x0F) as uint8, + [(((bv >> 12) & 0x0F) | 0xE0) as uint8, (((bv >> 6) & 0x3F) | 0x80) as uint8, - ( (bv & 0x3F) | 0x80) as uint8] + ( (bv & 0x3F) | 0x80) as uint8] else if cp < 0x110000 then - [( (bv >> 18) & 0x07) as uint8, + [(((bv >> 18) & 0x07) | 0xF0) as uint8, (((bv >> 12) & 0x3F) | 0x80) as uint8, (((bv >> 6) & 0x3F) | 0x80) as uint8, ( (bv & 0x3F) | 0x80) as uint8] @@ -62,6 +84,47 @@ module {:options "-functionSyntax:4"} UtfUtils { [] // Invalid: drop // TODO } + newtype opt_uint8 = c: int | -1 <= c as int < TWO_TO_THE_8 + + function Utf8DecodeChars(c0: uint8, c1: opt_uint8, c2: opt_uint8, c3: opt_uint8): (r: (uint32, uint8)) + ensures r.1 in {1, 2, 3, 4} + ensures c1 == -1 ==> r.1 <= 1 + ensures c2 == -1 ==> r.1 <= 2 + ensures c3 == -1 ==> r.1 <= 3 + { + if (c0 as bv32 & 0x80) == 0 then + (c0 as uint32, 1) + else if (c0 as bv32 & 0xE0) == 0xC0 && c1 > -1 then + (( (((c0 as bv32) & 0x1F) << 6) + | ((c1 as bv32) & 0x3F )) as uint32, + 2) + else if (c0 as bv32 & 0xF0) == 0xE0 && c1 > -1 && c2 > -1 then + (( (((c0 as bv32) & 0x0F) << 12) + | (((c1 as bv32) & 0x3F) << 6) + | ( (c2 as bv32) & 0x3F )) as uint32, + 3) + else if (c0 as bv32 & 0xF8) == 0xF0 && c1 > -1 && c2 > -1 && c3 > -1 then + (( (((c0 as bv32) & 0x07) << 18) + | (((c1 as bv32) & 0x3F) << 12) + | (((c2 as bv32) & 0x3F) << 6) + | ( (c3 as bv32) & 0x3F )) as uint32, + 4) + else + (0xFFFD, 1) // Replacement character + } + + function Utf8Decode(str: seq): seq { + if str == [] then + [] + else + var c0 := str[0] as uint8; + var c1 := if |str| > 1 then str[1] as opt_uint8 else -1; + var c2 := if |str| > 2 then str[2] as opt_uint8 else -1; + var c3 := if |str| > 3 then str[3] as opt_uint8 else -1; + var (cp, delta) := Utf8DecodeChars(c0, c1, c2, c3); + [cp] + Utf8Decode(str[delta..]) + } + function Utf8Encode(codepoints: seq): seq { if codepoints == [] then [] else Utf8Encode1(codepoints[0]) + Utf8Encode(codepoints[1..]) @@ -70,4 +133,8 @@ module {:options "-functionSyntax:4"} UtfUtils { function Transcode16To8(s: string): seq { Utf8Encode(Utf16Decode(s)) } + + function Transcode8To16(s: seq): string { + Utf16Encode(Utf8Decode(s)) + } } diff --git a/src/JSON/Vectors.dfy b/src/JSON/Vectors.dfy index 6fa1f673..8c052fa4 100644 --- a/src/JSON/Vectors.dfy +++ b/src/JSON/Vectors.dfy @@ -5,7 +5,7 @@ module {:options "-functionSyntax:4"} Vectors { import opened BoundedInts import opened Wrappers - datatype Error = OutOfMemory + datatype VectorError = OutOfMemory class Vector { ghost var Repr : seq @@ -85,7 +85,7 @@ module {:options "-functionSyntax:4"} Vectors { else MAX_CAPACITY } - method ReallocDefault() returns (o: Outcome) + method ReallocDefault() returns (o: Outcome) requires Valid?() modifies this, `data ensures Valid?() @@ -109,7 +109,7 @@ module {:options "-functionSyntax:4"} Vectors { return Pass; } - method Ensure(reserved: uint32) returns (o: Outcome) + method Ensure(reserved: uint32) returns (o: Outcome) requires Valid?() modifies this, `data ensures Valid?() @@ -164,7 +164,7 @@ module {:options "-functionSyntax:4"} Vectors { Repr := Repr + [a]; } - method Push(a: A) returns (o: Outcome) + method Push(a: A) returns (o: Outcome) requires Valid?() modifies this, data ensures Valid?() diff --git a/src/JSON/Views.dfy b/src/JSON/Views.dfy index 2ef504ec..e517f859 100644 --- a/src/JSON/Views.dfy +++ b/src/JSON/Views.dfy @@ -48,13 +48,15 @@ module {:options "-functionSyntax:4"} Views.Core { v'.s == s && v'.beg < beg && end == v'.end } - ghost predicate Byte?(c: byte) + predicate Byte?(c: byte) requires Valid? { Bytes() == [c] + } by method { + return Length() == 1 && At(0) == c; } - ghost predicate Char?(c: char) + predicate Char?(c: char) requires Valid? requires c as int < 256 { diff --git a/src/Math.dfy b/src/Math.dfy index 17b0f2bc..6232af5c 100644 --- a/src/Math.dfy +++ b/src/Math.dfy @@ -27,4 +27,9 @@ module Math { { if a >= 0 then a else -a } + + function method {:opaque} IntPow(x: int, n: nat) : int { + if n == 0 then 1 + else x * IntPow(x, n - 1) + } } diff --git a/src/Wrappers.dfy b/src/Wrappers.dfy index ddb9b7f3..31894b08 100644 --- a/src/Wrappers.dfy +++ b/src/Wrappers.dfy @@ -38,14 +38,14 @@ module Wrappers { } datatype Result<+T, +R> = | Success(value: T) | Failure(error: R) { - function method ToOption(): Option + function method ToOption(): Option { match this case Success(s) => Some(s) case Failure(e) => None() } - function method UnwrapOr(default: T): T + function method UnwrapOr(default: T): T { match this case Success(s) => s From 8af0e5427f772a98168a6ad93d88baec96592455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 1 Sep 2022 02:38:07 -0700 Subject: [PATCH 11/84] json: Optimize traversals and add support for unicode escapes --- src/JSON/JSON.Deserializer.dfy | 45 +++++++++++++++++++++++++++++-- src/JSON/JSON.Errors.dfy | 4 ++- src/JSON/JSON.Serializer.dfy | 30 +++++++++++++++------ src/JSON/Str.dfy | 49 +++++++++++++++++++++++++++++----- src/JSON/UtfUtils.dfy | 46 +++++++++++++++++-------------- 5 files changed, 136 insertions(+), 38 deletions(-) diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy index 040dfb2a..8264c309 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/JSON.Deserializer.dfy @@ -37,10 +37,51 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { js.At(0) == 't' as byte } + function Unescape(str: string, start: nat := 0): DeserializationResult + decreases |str| - start + { // Assumes UTF-16 strings + if start >= |str| then Success([]) + else if str[start] == '\\' then + if |str| == start + 1 then + Failure(EscapeAtEOS) + else + var c := str[start + 1]; + if str[start + 1] == 'u' then + if |str| <= start + 5 then + Failure(EscapeAtEOS) + else + var code := str[start + 1..start + 5]; + if exists c | c in str :: c !in Str.HEX_TABLE then + Failure(UnsupportedEscape) + else + var tl :- Unescape(str, start + 5); + reveal Math.IntPow(); + Success([Str.ToNat(code, 16) as char] + tl) + else + var unescaped: uint16 := match c + case '\"' => 0x22 // quotation mark + case '\\' => 0x5C // reverse solidus + case 'b' => 0x62 // backspace + case 'f' => 0x66 // form feed + case 'n' => 0x6E // line feed + case 'r' => 0x72 // carriage return + case 't' => 0x74 // tab + case _ => 0; + if unescaped == 0 then + Failure(UnsupportedEscape) + else + var tl :- Unescape(str, start + 1); + Success([unescaped as char] + tl) + else + var tl :- Unescape(str, start + 1); + Success([str[start]] + tl) + } + + function Transcode8To16Unescaped(str: seq): DeserializationResult // TODO Optimize with a function by method - { // FIXME unescape unicode - Str.UnescapeQuotes(UtfUtils.Transcode8To16(str)).MapFailure(_ => EscapeAtEOS) + { + Unescape(UtfUtils.Transcode8To16(str)).MapFailure(_ => EscapeAtEOS) } function String(js: Grammar.jstring): DeserializationResult { diff --git a/src/JSON/JSON.Errors.dfy b/src/JSON/JSON.Errors.dfy index 531b9f91..2117a984 100644 --- a/src/JSON/JSON.Errors.dfy +++ b/src/JSON/JSON.Errors.dfy @@ -5,6 +5,7 @@ module {:options "-functionSyntax:4"} JSON.Errors { datatype DeserializationError = | UnterminatedSequence + | UnsupportedEscape | EscapeAtEOS | EmptyNumber | ExpectingEOF @@ -12,8 +13,9 @@ module {:options "-functionSyntax:4"} JSON.Errors { { function ToString() : string { match this - case EscapeAtEOS => "Escape character at end of string" case UnterminatedSequence => "Unterminated sequence" + case UnsupportedEscape => "Unsupported escape sequence" + case EscapeAtEOS => "Escape character at end of string" case EmptyNumber => "Number must contain at least one digit" case ExpectingEOF => "Expecting EOF" case IntOverflow => "Input length does not fit in a 32-bit counter" diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.Serializer.dfy index 1b88417b..cea6a74d 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/JSON.Serializer.dfy @@ -41,10 +41,25 @@ module {:options "-functionSyntax:4"} JSON.Serializer { View.OfBytes(if b then TRUE else FALSE) } - // method Utf8Encode(st: Vector, cp: uint32) + function Escape(str: string, start: nat := 0): string + decreases |str| - start + { + if start >= |str| then [] + else + (match str[start] as uint16 + case 0x22 => "\\\"" // quotation mark + case 0x5C => "\\\\" // reverse solidus + case 0x62 => "\\b" // backspace + case 0x66 => "\\f" // form feed + case 0x6E => "\\n" // line feed + case 0x72 => "\\r" // carriage return + case 0x74 => "\\t" // tab + case _ => [str[0]]) + + Escape(str, start + 1) + } function Transcode16To8Escaped(str: string32, start: uint32 := 0): bytes { - UtfUtils.Transcode16To8(Str.EscapeQuotes(str)) + UtfUtils.Transcode16To8(Escape(str)) } // by method { // var len := |str| as uint32; @@ -133,14 +148,13 @@ module {:options "-functionSyntax:4"} JSON.Serializer { Success(Grammar.KV(k, COLON, v)) } - function MkSuffixedSequence(ds: seq, suffix: Structural) + function MkSuffixedSequence(ds: seq, suffix: Structural, start: nat := 0) : SuffixedSequence + decreases |ds| - start { - match |ds| - case 0 => [] - case 1 => [Suffixed(ds[0], Empty)] - case _ => [Suffixed(ds[0], NonEmpty(suffix))] - + MkSuffixedSequence(ds[1..], suffix) + if start >= |ds| then [] + else if start == |ds| - 1 then [Suffixed(ds[start], Empty)] + else [Suffixed(ds[start], NonEmpty(suffix))] + MkSuffixedSequence(ds, suffix, start + 1) } const COMMA: Structural := diff --git a/src/JSON/Str.dfy b/src/JSON/Str.dfy index efcc5bf5..1283bf78 100644 --- a/src/JSON/Str.dfy +++ b/src/JSON/Str.dfy @@ -1,11 +1,14 @@ include "../BoundedInts.dfy" include "../Wrappers.dfy" +include "../Math.dfy" module {:options "-functionSyntax:4"} Str { import opened Wrappers + import Math abstract module ParametricConversion { import opened Wrappers + import Math type Char(==) type String = seq @@ -69,17 +72,36 @@ module {:options "-functionSyntax:4"} Str { } function ToNat_any(str: String, base: nat, digits: map) : (n: nat) - requires forall c | c in str :: c in digits requires base > 0 + requires forall c | c in str :: c in digits { if str == [] then 0 else ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]] } + lemma {:induction false} ToNat_bound(str: String, base: nat, digits: map) + requires base > 0 + requires forall c | c in str :: c in digits + requires forall c | c in str :: digits[c] < base + ensures ToNat_any(str, base, digits) < Math.IntPow(base, |str|) + { + reveal Math.IntPow(); + if str == [] { + } else { + calc <= { + ToNat_any(str, base, digits); + ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]]; + { ToNat_bound(str[..|str| - 1], base, digits); } + (Math.IntPow(base, |str| - 1) - 1) * base + base - 1; + Math.IntPow(base, |str| - 1) * base - 1; + } + } + } + function ToInt_any(str: String, minus: Char, base: nat, digits: map) : (s: int) - requires NumberStr(str, minus, c => c in digits) - requires str != [minus] requires base > 0 + requires str != [minus] + requires NumberStr(str, minus, c => c in digits) { if [minus] <= str then -(ToNat_any(str[1..], base, digits) as int) else @@ -133,9 +155,13 @@ module {:options "-functionSyntax:4"} Str { const HEX_DIGITS := [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F'] - const HEX_TABLE := - CharStrConversion.DigitsTable(HEX_DIGITS); + 'A', 'B', 'C', 'D', 'E', 'F' + ] + const HEX_TABLE := map[ + '0' := 0, '1' := 1, '2' := 2, '3' := 3, '4' := 4, '5' := 5, '6' := 6, '7' := 7, '8' := 8, '9' := 9, + 'a' := 0xA, 'b' := 0xB, 'c' := 0xC, 'd' := 0xD, 'e' := 0xE, 'f' := 0xF, + 'A' := 0xA, 'B' := 0xB, 'C' := 0xC, 'D' := 0xD, 'E' := 0xE, 'F' := 0xF + ] function OfInt(n: int, base: int := 10) : (str: string) requires 2 <= base <= 16 @@ -144,10 +170,19 @@ module {:options "-functionSyntax:4"} Str { CharStrConversion.OfInt_any(n, HEX_DIGITS[..base], '-') } + function ToNat(str: string, base: int := 10) : (n: nat) + requires 2 <= base <= 16 + requires forall c | c in str :: c in HEX_TABLE && HEX_TABLE[c] as int < base + ensures n < Math.IntPow(base, |str|) + { + CharStrConversion.ToNat_bound(str, base, HEX_TABLE); + CharStrConversion.ToNat_any(str, base, HEX_TABLE) + } + function ToInt(str: string, base: int := 10) : (n: int) requires str != "-" requires 2 <= base <= 16 - requires CharStrConversion.NumberStr(str, '-', c => c in HEX_DIGITS[..base]) + requires CharStrConversion.NumberStr(str, '-', (c: char) => c in HEX_TABLE && HEX_TABLE[c] as int < base) { CharStrConversion.ToInt_any(str, '-', base, HEX_TABLE) } diff --git a/src/JSON/UtfUtils.dfy b/src/JSON/UtfUtils.dfy index 82827bb2..1599a4d0 100644 --- a/src/JSON/UtfUtils.dfy +++ b/src/JSON/UtfUtils.dfy @@ -32,14 +32,15 @@ module {:options "-functionSyntax:4"} UtfUtils { (0xFFFD, 1) // Replacement character } - function Utf16Decode(str: string): seq { - if str == [] then - [] + function Utf16Decode(str: string, start: nat := 0): seq + decreases |str| - start + { + if start >= |str| then [] else - var c0 := str[0] as uint16; - var c1 := if |str| > 1 then str[1] as opt_uint16 else -1; + var c0 := str[start] as uint16; + var c1 := if |str| > start + 1 then str[start + 1] as opt_uint16 else -1; var (cp, delta) := Utf16DecodeChars(c0, c1); - [cp] + Utf16Decode(str[delta..]) + [cp] + Utf16Decode(str, start + delta as nat) } function Utf16Encode2(cp: uint32): seq @@ -59,9 +60,11 @@ module {:options "-functionSyntax:4"} UtfUtils { [] // Invalid: drop // TODO } - function Utf16Encode(codepoints: seq): seq { - if codepoints == [] then [] - else Utf16Encode1(codepoints[0]) + Utf16Encode(codepoints[1..]) + function Utf16Encode(codepoints: seq, start: nat := 0): seq + decreases |codepoints| - start + { + if start >= |codepoints| then [] + else Utf16Encode1(codepoints[start]) + Utf16Encode(codepoints, start + 1) } function Utf8Encode1(cp: uint32): seq { @@ -113,21 +116,24 @@ module {:options "-functionSyntax:4"} UtfUtils { (0xFFFD, 1) // Replacement character } - function Utf8Decode(str: seq): seq { - if str == [] then - [] + function Utf8Decode(str: seq, start: nat := 0): seq + decreases |str| - start + { + if start >= |str| then [] else - var c0 := str[0] as uint8; - var c1 := if |str| > 1 then str[1] as opt_uint8 else -1; - var c2 := if |str| > 2 then str[2] as opt_uint8 else -1; - var c3 := if |str| > 3 then str[3] as opt_uint8 else -1; + var c0 := str[start] as uint8; + var c1 := if |str| > start + 1 then str[start + 1] as opt_uint8 else -1; + var c2 := if |str| > start + 2 then str[start + 2] as opt_uint8 else -1; + var c3 := if |str| > start + 3 then str[start + 3] as opt_uint8 else -1; var (cp, delta) := Utf8DecodeChars(c0, c1, c2, c3); - [cp] + Utf8Decode(str[delta..]) + [cp] + Utf8Decode(str, start + delta as nat) } - function Utf8Encode(codepoints: seq): seq { - if codepoints == [] then [] - else Utf8Encode1(codepoints[0]) + Utf8Encode(codepoints[1..]) + function Utf8Encode(codepoints: seq, start: nat := 0): seq + decreases |codepoints| - start + { + if start >= |codepoints| then [] + else Utf8Encode1(codepoints[start]) + Utf8Encode(codepoints, start + 1) } function Transcode16To8(s: string): seq { From 32068c05f1a88f4bbd2330fc427f651e51b08557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 1 Sep 2022 18:04:52 -0700 Subject: [PATCH 12/84] json: Finish unicode encoding and decoding --- src/JSON/JSON.API.dfy | 35 +++---- src/JSON/JSON.Deserializer.dfy | 50 +++++---- src/JSON/JSON.Errors.dfy | 27 ++++- src/JSON/JSON.Grammar.dfy | 5 +- src/JSON/JSON.LowLevel.Spec.dfy | 4 +- src/JSON/JSON.Serializer.dfy | 43 +++----- src/JSON/JSON.Spec.dfy | 39 ++++++- src/JSON/JSON.ZeroCopy.API.dfy | 1 + src/JSON/JSON.ZeroCopy.Deserializer.dfy | 46 +++++--- src/JSON/JSON.ZeroCopy.Serializer.dfy | 7 +- src/JSON/Str.dfy | 33 ++++-- src/JSON/Tests.dfy | Bin 1597 -> 7732 bytes src/Math.dfy | 133 +++++++++++++++++++++++- src/NonlinearArithmetic/Power.dfy | 4 + 14 files changed, 322 insertions(+), 105 deletions(-) diff --git a/src/JSON/JSON.API.dfy b/src/JSON/JSON.API.dfy index 490cdcd4..04fd85f0 100644 --- a/src/JSON/JSON.API.dfy +++ b/src/JSON/JSON.API.dfy @@ -2,9 +2,12 @@ include "JSON.Errors.dfy" include "JSON.Grammar.dfy" include "JSON.Spec.dfy" include "JSON.Serializer.dfy" -include "JSON.ZeroCopy.Serializer.dfy" +include "JSON.Deserializer.dfy" +include "JSON.ZeroCopy.API.dfy" module {:options "-functionSyntax:4"} JSON.API { + // TODO: Propagate proofs + import opened BoundedInts import opened Wrappers import Vs = Views.Core @@ -13,41 +16,31 @@ module {:options "-functionSyntax:4"} JSON.API { import opened AST import Spec import S = Serializer - import ZS = ZeroCopy.Serializer - // import Deserializer + import DS = Deserializer + import ZAPI = ZeroCopy.API function {:opaque} Serialize(js: JSON) : (bs: SerializationResult>) - // TODO: Carry proofs all the way - // ensures bs.Success? ==> bs.value == Spec.JSON(js) { var js :- S.JSON(js); - Success(ZS.Text(js).Bytes()) + Success(ZAPI.Serialize(js)) } method SerializeAlloc(js: JSON) returns (bs: SerializationResult>) - // TODO: Carry proofs all the way - // ensures bs.Success? ==> fresh(bs.value) - // ensures bs.Success? ==> bs.value[..] == Spec.JSON(js) { var js :- S.JSON(js); - bs := ZS.Serialize(js); + bs := ZAPI.SerializeAlloc(js); } method SerializeBlit(js: JSON, bs: array) returns (len: SerializationResult) modifies bs - // TODO: Carry proofs all the way - // ensures len.Success? ==> len.value as int <= bs.Length - // ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) - // ensures len.Success? ==> bs[len.value..] == old(bs[len.value..]) - // ensures len.Failure? ==> unchanged(bs) { var js :- S.JSON(js); - len := ZS.SerializeTo(js, bs); + len := ZAPI.SerializeBlit(js, bs); } - // function {:opaque} Deserialize(bs: seq) : (js: Result) - // ensures js.Success? ==> bs == Spec.JSON(js.value) - // { - // Deserializer.API.OfBytes(bs) - // } + function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) + { + var js :- ZAPI.Deserialize(bs); + DS.JSON(js) + } } diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/JSON.Deserializer.dfy index 8264c309..f0bbab73 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/JSON.Deserializer.dfy @@ -46,46 +46,46 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Failure(EscapeAtEOS) else var c := str[start + 1]; - if str[start + 1] == 'u' then - if |str| <= start + 5 then + if c == 'u' then + if |str| <= start + 6 then Failure(EscapeAtEOS) else - var code := str[start + 1..start + 5]; - if exists c | c in str :: c !in Str.HEX_TABLE then - Failure(UnsupportedEscape) + var code := str[start + 2..start + 6]; + if exists c | c in code :: c !in Str.HEX_TABLE then + Failure(UnsupportedEscape(code)) else - var tl :- Unescape(str, start + 5); - reveal Math.IntPow(); - Success([Str.ToNat(code, 16) as char] + tl) + var tl :- Unescape(str, start + 6); + var hd := Str.ToNat(code, 16); + assert hd < 0x10000 by { reveal Math.IntPow(); } + Success([hd as char] + tl) else var unescaped: uint16 := match c - case '\"' => 0x22 // quotation mark - case '\\' => 0x5C // reverse solidus - case 'b' => 0x62 // backspace - case 'f' => 0x66 // form feed - case 'n' => 0x6E // line feed - case 'r' => 0x72 // carriage return - case 't' => 0x74 // tab - case _ => 0; - if unescaped == 0 then - Failure(UnsupportedEscape) + case '\"' => 0x22 as uint16 // quotation mark + case '\\' => 0x5C as uint16 // reverse solidus + case 'b' => 0x08 as uint16 // backspace + case 'f' => 0x0C as uint16 // form feed + case 'n' => 0x0A as uint16 // line feed + case 'r' => 0x0D as uint16 // carriage return + case 't' => 0x09 as uint16 // tab + case _ => 0 as uint16; + if unescaped == 0 as uint16 then + Failure(UnsupportedEscape(str[start..start+2])) else - var tl :- Unescape(str, start + 1); + var tl :- Unescape(str, start + 2); Success([unescaped as char] + tl) else var tl :- Unescape(str, start + 1); Success([str[start]] + tl) } - function Transcode8To16Unescaped(str: seq): DeserializationResult // TODO Optimize with a function by method { - Unescape(UtfUtils.Transcode8To16(str)).MapFailure(_ => EscapeAtEOS) + Unescape(UtfUtils.Transcode8To16(str)) } function String(js: Grammar.jstring): DeserializationResult { - Transcode8To16Unescaped(js.Bytes()) + Transcode8To16Unescaped(js.contents.Bytes()) } module ByteStrConversion refines Str.ParametricConversion { @@ -117,7 +117,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { case Empty => Success(AST.Decimal(n, e10)) case NonEmpty(JFrac(_, num)) => var pow10 := num.Length() as int; - var frac :- ToInt(View.Empty, num); + var frac :- ToInt(minus, num); Success(AST.Decimal(n * Math.IntPow(10, pow10) + frac, e10 - pow10)) } @@ -144,4 +144,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { case Object(obj) => var o :- Object(obj); Success(AST.Object(o)) case Array(arr) => var a :- Array(arr); Success(AST.Array(a)) } + + function JSON(js: Grammar.JSON): DeserializationResult { + Value(js.t) + } } diff --git a/src/JSON/JSON.Errors.dfy b/src/JSON/JSON.Errors.dfy index 2117a984..9d6cc490 100644 --- a/src/JSON/JSON.Errors.dfy +++ b/src/JSON/JSON.Errors.dfy @@ -1,24 +1,39 @@ include "../Wrappers.dfy" +include "../BoundedInts.dfy" +include "Str.dfy" module {:options "-functionSyntax:4"} JSON.Errors { import Wrappers + import opened BoundedInts + import Str datatype DeserializationError = | UnterminatedSequence - | UnsupportedEscape + | UnsupportedEscape(str: string) | EscapeAtEOS | EmptyNumber | ExpectingEOF | IntOverflow + | ReachedEOF + | ExpectingByte(expected: byte, b: opt_byte) + | ExpectingAnyByte(expected_sq: seq, b: opt_byte) { function ToString() : string { match this case UnterminatedSequence => "Unterminated sequence" - case UnsupportedEscape => "Unsupported escape sequence" + case UnsupportedEscape(str) => "Unsupported escape sequence: " + str case EscapeAtEOS => "Escape character at end of string" case EmptyNumber => "Number must contain at least one digit" case ExpectingEOF => "Expecting EOF" case IntOverflow => "Input length does not fit in a 32-bit counter" + case ReachedEOF => "Reached EOF" + case ExpectingByte(b0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + "Expecting '" + [b0 as char] + "', read " + c + case ExpectingAnyByte(bs0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); + "Expecting one of '" + c0s + "', read " + c } } @@ -26,6 +41,14 @@ module {:options "-functionSyntax:4"} JSON.Errors { | OutOfMemory | IntTooLarge(i: int) | StringTooLong(s: string) + { + function ToString() : string { + match this + case OutOfMemory => "Out of memory" + case IntTooLarge(i: int) => "Integer too large: " + Str.OfInt(i) + case StringTooLong(s: string) => "String too long: " + s + } + } type SerializationResult<+T> = Wrappers.Result type DeserializationResult<+T> = Wrappers.Result diff --git a/src/JSON/JSON.Grammar.dfy b/src/JSON/JSON.Grammar.dfy index dc003bc6..3ebb9a0a 100644 --- a/src/JSON/JSON.Grammar.dfy +++ b/src/JSON/JSON.Grammar.dfy @@ -12,6 +12,7 @@ module {:options "-functionSyntax:4"} JSON.Grammar { import opened Views.Core const EMPTY := View.OfBytes([]) + const DOUBLEQUOTE := View.OfBytes(['\"' as byte]) const PERIOD := View.OfBytes(['.' as byte]) const E := View.OfBytes(['e' as byte]) const COLON := View.OfBytes([':' as byte]) @@ -23,6 +24,7 @@ module {:options "-functionSyntax:4"} JSON.Grammar { const MINUS := View.OfBytes(['-' as byte]) type jchar = v: View | v.Length() == 1 witness View.OfBytes(['b' as byte]) + type jquote = v: View | v.Char?('\"') witness DOUBLEQUOTE type jperiod = v: View | v.Char?('.') witness PERIOD type je = v: View | v.Char?('e') || v.Char?('E') witness E type jcolon = v: View | v.Char?(':') witness COLON @@ -65,12 +67,13 @@ module {:options "-functionSyntax:4"} JSON.Grammar { ghost predicate Num?(v: View) { Digits?(v) && !v.Empty? } ghost predicate Int?(v: View) { v.Char?('0') || (Num?(v) && v.At(0) != '0' as byte) } - type jstring = v: View | true witness View.OfBytes([]) // TODO: Enforce quoting and escaping type jnull = v: View | Null?(v) witness View.OfBytes(NULL) type jbool = v: View | Bool?(v) witness View.OfBytes(TRUE) type jdigits = v: View | Digits?(v) witness View.OfBytes([]) type jnum = v: View | Num?(v) witness View.OfBytes(['0' as byte]) type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) + type jstr = v: View | true witness View.OfBytes([]) // TODO: Enforce quoting and escaping + datatype jstring = JString(lq: jquote, contents: jstr, rq: jquote) datatype jkv = KV(k: jstring, colon: Structural, v: Value) // TODO enforce no leading space before closing bracket to disambiguate WS { WS WS } WS diff --git a/src/JSON/JSON.LowLevel.Spec.dfy b/src/JSON/JSON.LowLevel.Spec.dfy index 3fd57385..8a7a1d75 100644 --- a/src/JSON/JSON.LowLevel.Spec.dfy +++ b/src/JSON/JSON.LowLevel.Spec.dfy @@ -41,7 +41,7 @@ module {:options "-functionSyntax:4"} JSON.LowLevel.Spec { } function KV(self: jkv): bytes { - View(self.k) + StructuralView(self.colon) + Value(self.v) + String(self.k) + StructuralView(self.colon) + Value(self.v) } function Frac(self: jfrac): bytes { @@ -57,7 +57,7 @@ module {:options "-functionSyntax:4"} JSON.LowLevel.Spec { } function String(self: jstring): bytes { - View(self) + View(self.lq) + View(self.contents) + View(self.rq) } function CommaSuffix(c: Maybe>): bytes { diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/JSON.Serializer.dfy index cea6a74d..869fceef 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/JSON.Serializer.dfy @@ -41,26 +41,9 @@ module {:options "-functionSyntax:4"} JSON.Serializer { View.OfBytes(if b then TRUE else FALSE) } - function Escape(str: string, start: nat := 0): string - decreases |str| - start - { - if start >= |str| then [] - else - (match str[start] as uint16 - case 0x22 => "\\\"" // quotation mark - case 0x5C => "\\\\" // reverse solidus - case 0x62 => "\\b" // backspace - case 0x66 => "\\f" // form feed - case 0x6E => "\\n" // line feed - case 0x72 => "\\r" // carriage return - case 0x74 => "\\t" // tab - case _ => [str[0]]) - + Escape(str, start + 1) - } - - function Transcode16To8Escaped(str: string32, start: uint32 := 0): bytes { - UtfUtils.Transcode16To8(Escape(str)) - } + function Transcode16To8Escaped(str: string, start: uint32 := 0): bytes { + UtfUtils.Transcode16To8(Spec.Escape(str)) + } // FIXME speed up using a `by method` // by method { // var len := |str| as uint32; // if len == 0 { @@ -88,10 +71,9 @@ module {:options "-functionSyntax:4"} JSON.Serializer { } function String(str: string): Result { - :- CheckLength(str, StringTooLong(str)); var bs := Transcode16To8Escaped(str); :- CheckLength(bs, StringTooLong(str)); - Success(View.OfBytes(bs)) + Success(Grammar.JString(Grammar.DOUBLEQUOTE, View.OfBytes(bs), Grammar.DOUBLEQUOTE)) } function Sign(n: int): jminus { @@ -126,13 +108,16 @@ module {:options "-functionSyntax:4"} JSON.Serializer { function Number(dec: AST.Decimal): Result { var minus: jminus := Sign(dec.n); var num: jnum :- Int(Math.Abs(dec.n)); - var frac: Maybe := Empty(); - var exp: jexp :- - var e: je := View.OfBytes(['e' as byte]); - var sign: jsign := Sign(dec.e10); - var num: jnum :- Int(Math.Abs(dec.e10)); - Success(JExp(e, sign, num)); - Success(JNumber(minus, num, Empty, NonEmpty(exp))) + var frac: Maybe := Empty(); + var exp: Maybe :- + if dec.e10 == 0 then + Success(Empty()) + else + var e: je := View.OfBytes(['e' as byte]); + var sign: jsign := Sign(dec.e10); + var num: jnum :- Int(Math.Abs(dec.e10)); + Success(NonEmpty(JExp(e, sign, num))); + Success(JNumber(minus, num, Empty, exp)) } function MkStructural(v: T): Structural { diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/JSON.Spec.dfy index 7c75e7ab..2a0c895a 100644 --- a/src/JSON/JSON.Spec.dfy +++ b/src/JSON/JSON.Spec.dfy @@ -20,16 +20,49 @@ module {:options "-functionSyntax:4"} JSON.Spec { type bytes = seq + function EscapeUnicode(c: uint16): string { + var s := Str.OfNat(c as nat, 16); + assert |s| <= 4 by { + assert c as nat <= 0xFFFF; + assert Math.IntLog(16, c as nat) <= Math.IntLog(16, 0xFFFF) by { + Math.IntLog_Increasing(16, c as nat, 0xFFFF); + } + assert Math.IntLog(16, 0xFFFF) == 3 by { reveal Math.IntLog(); } + } + s + seq(4 - |s|, _ => ' ') + } + + function Escape(str: string, start: nat := 0): string + decreases |str| - start + { + if start >= |str| then [] + else + (match str[start] as uint16 + case 0x22 => "\\\"" // quotation mark + case 0x5C => "\\\\" // reverse solidus + case 0x08 => "\\b" // backspace + case 0x0C => "\\f" // form feed + case 0x0A => "\\n" // line feed + case 0x0D => "\\r" // carriage return + case 0x09 => "\\t" // tab + case c => + if c < 0x001F then "\\u" + EscapeUnicode(c) + else [str[start]]) + + Escape(str, start + 1) + } + function String(str: string): bytes { - Str.ToBytes("'") + Transcode16To8(Str.EscapeQuotes(str)) + Str.ToBytes("'") + Str.ToBytes("\"") + Transcode16To8(Escape(str)) + Str.ToBytes("\"") } function Number(dec: Decimal): bytes { - Transcode16To8(Str.OfInt(dec.n)) + Str.ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10)) + Transcode16To8(Str.OfInt(dec.n)) + + (if dec.e10 == 0 then [] + else Str.ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) } function KV(kv: (string, JSON)): bytes { - String(kv.0) + Str.ToBytes(", ") + JSON(kv.1) + String(kv.0) + Str.ToBytes(":") + JSON(kv.1) } function Join(sep: bytes, items: seq): bytes { diff --git a/src/JSON/JSON.ZeroCopy.API.dfy b/src/JSON/JSON.ZeroCopy.API.dfy index a775b21a..87238c9f 100644 --- a/src/JSON/JSON.ZeroCopy.API.dfy +++ b/src/JSON/JSON.ZeroCopy.API.dfy @@ -10,6 +10,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { import Vs = Views.Core import opened Grammar + import opened Errors import LowLevel.Spec import Serializer import Deserializer diff --git a/src/JSON/JSON.ZeroCopy.Deserializer.dfy b/src/JSON/JSON.ZeroCopy.Deserializer.dfy index 7975f98a..f2240b11 100644 --- a/src/JSON/JSON.ZeroCopy.Deserializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Deserializer.dfy @@ -14,6 +14,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Cursors import opened Parsers import opened Grammar + import Errors type JSONError = Errors.DeserializationError type Error = CursorError @@ -281,28 +282,37 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Vs = Views.Core import opened Grammar - import opened Cursors import opened Core + import opened Errors + import Cursors import Values - function {:opaque} JSON(cs: FreshCursor) : (pr: ParseResult) + function LiftCursorError(err: Cursors.CursorError): DeserializationError { + match err + case EOF => ReachedEOF + case ExpectingByte(expected, b) => ExpectingByte(expected, b) + case ExpectingAnyByte(expected_sq, b) => ExpectingAnyByte(expected_sq, b) + case OtherError(err) => err + } + + function {:opaque} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.JSON) { - Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)) + Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)).MapFailure(LiftCursorError) } - function {:opaque} Text(v: View) : (jsr: Result) + function {:opaque} Text(v: View) : (jsr: DeserializationResult) ensures jsr.Success? ==> v.Bytes() == Spec.JSON(jsr.value) { - var SP(text, cs) :- JSON(Cursor.OfView(v)); - :- Need(cs.EOF?, OtherError(ExpectingEOF)); + var SP(text, cs) :- JSON(Cursors.Cursor.OfView(v)); + :- Need(cs.EOF?, Errors.ExpectingEOF); Success(text) } - function {:opaque} OfBytes(bs: bytes) : (jsr: Result) + function {:opaque} OfBytes(bs: bytes) : (jsr: DeserializationResult) ensures jsr.Success? ==> bs == Spec.JSON(jsr.value) { - :- Need(|bs| < TWO_TO_THE_32, OtherError(IntOverflow)); + :- Need(|bs| < TWO_TO_THE_32, Errors.IntOverflow); Text(Vs.View.OfBytes(bs)) } } @@ -369,7 +379,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Core import opened Cursors - function {:opaque} Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) + function {:opaque} Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) requires |expected| < TWO_TO_THE_32 ensures pr.Success? ==> pr.value.t.Bytes() == expected ensures pr.Success? ==> pr.value.SplitFrom?(cs, _ => expected) @@ -413,14 +423,22 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { return Failure(EOF); } - function {:opaque} String(cs: FreshCursor): (pr: ParseResult) - ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.String) + function Quote(cs: FreshCursor) : (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { - var cs :- cs.AssertChar('\"'); - var cs :- StringBody(cs); var cs :- cs.AssertChar('\"'); Success(cs.Split()) } + + function {:opaque} String(cs: FreshCursor): (pr: ParseResult) + ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.String) + { + var SP(lq, cs) :- Quote(cs); + var contents :- StringBody(cs); + var SP(contents, cs) := contents.Split(); + var SP(rq, cs) :- Quote(cs); + Success(SP(Grammar.JString(lq, contents, rq), cs)) + } } module Numbers { @@ -441,7 +459,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, SpecView) { var sp := Digits(cs); - :- Need(!sp.t.Empty?, OtherError(EmptyNumber)); + :- Need(!sp.t.Empty?, OtherError(Errors.EmptyNumber)); Success(sp) } diff --git a/src/JSON/JSON.ZeroCopy.Serializer.dfy b/src/JSON/JSON.ZeroCopy.Serializer.dfy index 4ad9f0c0..e93dc61e 100644 --- a/src/JSON/JSON.ZeroCopy.Serializer.dfy +++ b/src/JSON/JSON.ZeroCopy.Serializer.dfy @@ -70,7 +70,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { decreases str, 0 ensures wr.Bytes() == writer.Bytes() + Spec.String(str) { - writer.Append(str) + writer + .Append(str.lq) + .Append(str.contents) + .Append(str.rq) } function {:opaque} Number(num: jnumber, writer: Writer) : (wr: Writer) @@ -243,7 +246,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { decreases obj, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Member(m) { - var writer := writer.Append(m.t.k); + var writer := String(m.t.k, writer); var writer := StructuralView(m.t.colon, writer); var writer := Value(m.t.v, writer); if m.suffix.Empty? then writer else StructuralView(m.suffix.t, writer) diff --git a/src/JSON/Str.dfy b/src/JSON/Str.dfy index 1283bf78..2ee820af 100644 --- a/src/JSON/Str.dfy +++ b/src/JSON/Str.dfy @@ -15,25 +15,37 @@ module {:options "-functionSyntax:4"} Str { // FIXME the design in LittleEndianNat makes BASE a module-level constant // instead of a function argument - function Digits(n: int, base: int): (digits: seq) + function Digits(n: nat, base: int): (digits: seq) requires base > 1 - requires n >= 0 decreases n + ensures n == 0 ==> |digits| == 0 + ensures n > 0 ==> |digits| == Math.IntLog(base, n) + 1 ensures forall d | d in digits :: 0 <= d < base { if n == 0 then + assert Math.IntPow(base, 0) == 1 by { reveal Math.IntPow(); } [] else - assert n > 0; - assert base > 1; assert n < base * n; assert n / base < n; - Digits(n / base, base) + [n % base] + var digits' := Digits(n / base, base); + var digits := digits' + [n % base]; + assert |digits| == Math.IntLog(base, n) + 1 by { + if n < base { + assert |digits| == 1; + assert Math.IntLog(base, n) == 0 by { reveal Math.IntLog(); } + } else { + assert |digits'| == |digits| - 1; + assert Math.IntLog(base, n) == Math.IntLog(base, n / base) + 1 by { reveal Math.IntLog(); } + } + } + digits } function OfDigits(digits: seq, chars: seq) : (str: String) requires forall d | d in digits :: 0 <= d < |chars| ensures forall c | c in str :: c in chars + ensures |str| == |digits| { if digits == [] then [] else @@ -44,10 +56,11 @@ module {:options "-functionSyntax:4"} Str { function OfNat_any(n: nat, chars: seq) : (str: String) requires |chars| > 1 + ensures |str| == Math.IntLog(|chars|, n) + 1 ensures forall c | c in str :: c in chars { var base := |chars|; - if n == 0 then [chars[0]] + if n == 0 then reveal Math.IntLog(); [chars[0]] else OfDigits(Digits(n, base), chars) } @@ -163,6 +176,14 @@ module {:options "-functionSyntax:4"} Str { 'A' := 0xA, 'B' := 0xB, 'C' := 0xC, 'D' := 0xD, 'E' := 0xE, 'F' := 0xF ] + function OfNat(n: nat, base: int := 10) : (str: string) + requires 2 <= base <= 16 + ensures |str| == Math.IntLog(base, n) + 1 + ensures forall c | c in str :: c in HEX_DIGITS[..base] + { + CharStrConversion.OfNat_any(n, HEX_DIGITS[..base]) + } + function OfInt(n: int, base: int := 10) : (str: string) requires 2 <= base <= 16 ensures CharStrConversion.NumberStr(str, '-', c => c in HEX_DIGITS[..base]) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 3323e82c96ac043282164a936a2d2605d142cc9a..ccfd3717ab5b7b9169a9cf0febffb15cf5c2d308 100644 GIT binary patch literal 7732 zcmeHM+io015Upn<{$aE)wguV54i_tNA~{Zgpg>S;Kw=%CwRhK9@GU#z7_jr7yzm8l z0B^uKRa@KBJu`c85Y9uin(<6`S9MkOsngZtzyFb;9Lv5O%9-?}FPXIDTPb9q_aWMI zX`^Q#zoO@Td9P;tm$EHS`s$3XZjRpM}!LKfS9LjI{wwQ*8kjdXzi>(P~=GB!w=q%JP z#yx{?Cy39Pu0$rmqaB@{F>A~BI{sASJA%xG>dNpmgnU?mZPmqjlJhSfP@ z>2emh|CA|!RU##L>$V)BwU8GYkxc62zpN3w1L-~JDKttecpItg8U7N-nJmjs_{(v1 zdb*G_%s*2NjDH~;llb44l}U_atG3nlea({?UtvaH#d^Dl0NRxzw8^I6=QOKQv|pz+ zTUbN#%FkE@_cnO(A$<3>@kp_aBbPKX@z}hvhgwX+_1LP_VM`=Emv0`Xoi`(bD zv7Xn8%;}kB`;TXop+y*lE!5=Bc#@rH7pvD z9Co}?UMyf0YeVy$_14DNLW>=FKIN8y$}n~wgIjo_rc5pqy_Xef;>h!#bDW~$AfH$? zW2-ut<9ULhSiT2Fb6H#SHY(r2xXN3Gd^t{(jpYNUx)#=1FA<+h$Wl8!Aakq=SKvon z#&b(}igD@M%Y4+54%pAuj@tOtf=`UMw|fk%ua04;*%1;WJ4Ddt2>#Uh9a~m6lOc)o z*;7E)Jk?=Z`=-?*E29D8%F3J>l=g=`EUqTwYA%+tN}d+SGS!N`IwrN(&n?a^vBoDr z__;iR?L_Gzs$u7Jj(3@AO-aOL65G?B)OTgsxW1dlti(^KO_-zqebt%fJ@*065}S(HPai`yRp%jjn4?k+^VP4Zf7KeNrDs!~9j)fFj6FxJ+o%>^pl%?R z+oe&<;G5<=F7F%L;JtnPmcdI?Zd(ppXw<gf;G=A~RRa5_jCQc{<{LJjCB~)yOKCXHe?c)7~C>2l(+^ zXXg*+x~uFFUqIUd<_$a=^SR~u+q6rB?Bu`^V#ayV=(5C zvL(NXk|kMb+c#U*4gq<&axi=L(Ml9p3%@{+adWhF(PuL;HaYk>Loe@PA8`gIBhB77 zeDvacUEbliGw?eJ_nGhTpM@$9M8428PoXEa+2?4yPQ(~;xa#hO ze>3v;xPtl$xkHxVS*^@(L{IufAg)dMwU~@ufb+r^d!F86J}jSIsC8D&dlkJUF8>$D CtcQdE literal 1597 zcmbtUQE$^Q5Pr|EIQeCX$g*~@2?aW7l!1mcHqnU(l4?p)ml`E@u@l&6``@{y=i%`9dGzAslphXXx}Bu;B*U!{!f|V_HV(9clw~Q4VF-_R zQkJKzziU**aVN%*PDn`Maz+FRnIM}g=Q?E}^aa7R<4LI65NxFoBD^d^AQAsZjtPKy z(^sC?&el~Xq#xMRnkx;0x;=ne!Mbh#+WG)~WayR4rY1z|O9V0je*AukR}9t`u*S(6 z+z;;t?T5(-v&50D9YqK%mCS__ubmy@wZ?AJEnVPnRaO@Zpc~%1MSeSVE1LnJ$Z&e! zWuj~}_bY=pWOUZLIsxn^(+J{PD;)!aXMniqR!!`c@_imdFM*E6O7n=W`2|G~A@ChM z2#{b`Os^OAOiRe#F?M{5J$}PC7)MekRmN0G-7^CE=(P01p diff --git a/src/Math.dfy b/src/Math.dfy index 6232af5c..73edcf41 100644 --- a/src/Math.dfy +++ b/src/Math.dfy @@ -28,8 +28,137 @@ module Math { if a >= 0 then a else -a } - function method {:opaque} IntPow(x: int, n: nat) : int { + function method {:opaque} IntPow(base: int, n: nat): int { if n == 0 then 1 - else x * IntPow(x, n - 1) + else base * IntPow(base, n - 1) } + + function method {:opaque} IntLog(base: nat, pow: nat): nat + requires base > 1 + decreases pow + // TODO: pow > 0 ==> IntPow(base, IntLog(base, pow)) <= pow < IntPow(base, IntLog(base, pow) + 1) + { + if pow < base then 0 + else + assert pow < pow * base; assert pow / base < pow; + 1 + IntLog(base, pow / base) + } + + lemma {:induction false} Divide_Increasing(base: nat, pow: nat, pow': nat) + requires base > 1 + requires pow <= pow' + ensures pow / base <= pow' / base + { + var q, q', r, r' := pow / base, pow' / base, pow % base, pow' % base; + assert pow == q * base + r; + assert pow' == q' * base + r'; + assert 0 <= r < base && 0 <= r' < base; + var dp, dq, dr := pow' - pow, q' - q, r' - r; + assert dp == dq * base + dr; + if dq < 0 { + assert dq * base < -base; + assert dr <= base; + assert dp == dq * base + dr < 0; + assert false; + } + } + + lemma {:induction false} IntLog_Increasing(base: nat, pow: nat, pow': nat) + requires base > 1 + requires pow <= pow' + ensures IntLog(base, pow) <= IntLog(base, pow') + decreases pow + { + reveal IntLog(); + if pow' < base { + assert IntLog(base, pow) == 0 == IntLog(base, pow'); + } else if pow < base { + assert IntLog(base, pow) == 0; + } else { + assert pow < pow * base; assert pow / base < pow; + assert pow' < pow' * base; assert pow' / base < pow'; + assert pow / base <= pow' / base by { Divide_Increasing(base, pow, pow'); } + IntLog_Increasing(base, pow / base, pow' / base); + } + } + + // lemma {:induction n} IntPow_Nat(base: nat, n: nat) + // ensures IntPow(base, n) >= 0 + // { + // reveal IntPow(); + // } + + // lemma {:induction n} IntPow_NonZero(base: int, n: nat) + // requires base != 0 + // ensures IntPow(base, n) != 0 + // { + // reveal IntPow(); + // } + + // function method {:opaque} NatPow(base: nat, n: nat): nat { + // IntPow_Nat(base, n); + // IntPow(base, n) + // } + + // lemma {:induction false} NatPow_NonZero(base: nat, n: nat) + // requires base != 0 + // ensures NatPow(base, n) != 0 + // { + // reveal NatPow(); IntPow_NonZero(base, n); + // } + + // function method {:opaque} IntLogSup(base: nat, pow: nat): (log: nat) + // requires base > 1 + // ensures log > 0 + // decreases pow + // // IntPow(base, IntLogSup(base, pow) - 1) - 1 <= pow <= IntPow(base, IntLogSup(base, pow)) + // { + // if pow < base then 1 + // else + // assert pow < pow * base; + // assert pow / base < pow; + // 1 + IntLog(base, pow / base) + // } + + // // lemma {:induction false} IntLogSup_transfer(base: nat, n: nat, n': nat) + // // requires base > 1 + // // requires n > n' + // // ensures + // // (NatPow_NonZero(base, n); NatPow_NonZero(base, n'); + // // IntLogSup(base, NatPow(base, n), NatPow(base, n')) == IntLogSup(base, NatPow(base, n - n'))) + // // { + // // reveal NatPow(); + // // reveal IntPow(); + // // reveal IntLogSup(); + // // if n' == 0 { + // // } else { + // // NatPow_NonZero(base, n); NatPow_NonZero(base, n'); NatPow_NonZero(base, n' - 1); + // // calc { + // // IntLogSup(base, NatPow(base, n), NatPow(base, n')); + // // 1 + IntLogSup(base, NatPow(base, n), NatPow(base, n' - 1) * base) - 1; + // // { assume NatPow(base, n) > NatPow(base, n'); } // FIXME wrong + // // IntLogSup(base, NatPow(base, n), NatPow(base, n' - 1)) - 1; + // // { IntLogSup_transfer(); } + // // IntLogSup(base, NatPow(base, n - n')) + // // } + // // } + // // } + + // lemma {:induction false} IntPow_IntLogSup(base: nat, n: nat) + // requires base > 1 + // ensures IntLogSup(base, NatPow(base, n)) - 1 < n <= IntLogSup(base, NatPow(base, n)) + // { + // reveal IntLogSup(); + // reveal IntPow(); + // reveal NatPow(); + // if n == 0 { + // } else { + // calc { + // IntLogSup(base, NatPow(base, n)); + // IntLogSup(base, NatPow(base, n - 1) * base); + // IntLogSup(base, NatPow(base, n - 1) * base); + // 1 + IntLogSup(base, NatPow(base, n - 1)); + // } + // } + // } } diff --git a/src/NonlinearArithmetic/Power.dfy b/src/NonlinearArithmetic/Power.dfy index 5d86e94c..61f74bec 100644 --- a/src/NonlinearArithmetic/Power.dfy +++ b/src/NonlinearArithmetic/Power.dfy @@ -12,12 +12,14 @@ former takes arguments and may be more stable and less reliant on Z3 heuristics. The latter includes automation and its use requires less effort */ +include "../Math.dfy" include "DivMod.dfy" include "Internals/GeneralInternals.dfy" include "Mul.dfy" include "Internals/MulInternals.dfy" module Power { + import Math import opened DivMod import opened GeneralInternals import opened Mul @@ -25,7 +27,9 @@ module Power { function method {:opaque} Pow(b: int, e: nat): int decreases e + ensures Pow(b, e) == Math.IntPow(b, e) { + reveal Math.IntPow(); if e == 0 then 1 else From 5ffcf3555739312ace07b221d776f68d75611d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 1 Sep 2022 21:46:51 -0700 Subject: [PATCH 13/84] json: Adopt a consistent naming scheme across modules --- src/JSON/{JSON.API.dfy => API.dfy} | 14 +++--- src/JSON/{JSON.AST.dfy => AST.dfy} | 0 ...JSON.Deserializer.dfy => Deserializer.dfy} | 24 +++++----- src/JSON/{JSON.Errors.dfy => Errors.dfy} | 4 +- src/JSON/{JSON.Grammar.dfy => Grammar.dfy} | 4 +- ...ON.LowLevel.Spec.dfy => LowLevel.Spec.dfy} | 4 +- ...erties.dfy => LowLevel.SpecProperties.dfy} | 4 +- .../{JSON.Serializer.dfy => Serializer.dfy} | 28 ++++++------ src/JSON/{JSON.Spec.dfy => Spec.dfy} | 39 +++++++++------- src/JSON/Tests.dfy | Bin 7732 -> 3583 bytes src/JSON/{ => Utils}/Cursors.dfy | 6 +-- src/JSON/{ => Utils}/Lexers.dfy | 6 +-- src/JSON/{ => Utils}/Parsers.dfy | 6 +-- src/JSON/{ => Utils}/Str.dfy | 17 ++----- src/JSON/{UtfUtils.dfy => Utils/Unicode.dfy} | 4 +- src/JSON/{ => Utils}/Vectors.dfy | 6 +-- src/JSON/{ => Utils}/Views.Writers.dfy | 6 +-- src/JSON/{ => Utils}/Views.dfy | 4 +- .../API.dfy} | 12 ++--- .../Deserializer.dfy} | 42 +++++++++--------- .../Serializer.dfy} | 12 ++--- 21 files changed, 120 insertions(+), 122 deletions(-) rename src/JSON/{JSON.API.dfy => API.dfy} (82%) rename src/JSON/{JSON.AST.dfy => AST.dfy} (100%) rename src/JSON/{JSON.Deserializer.dfy => Deserializer.dfy} (93%) rename src/JSON/{JSON.Errors.dfy => Errors.dfy} (97%) rename src/JSON/{JSON.Grammar.dfy => Grammar.dfy} (98%) rename src/JSON/{JSON.LowLevel.Spec.dfy => LowLevel.Spec.dfy} (97%) rename src/JSON/{JSON.LowLevel.SpecProperties.dfy => LowLevel.SpecProperties.dfy} (96%) rename src/JSON/{JSON.Serializer.dfy => Serializer.dfy} (91%) rename src/JSON/{JSON.Spec.dfy => Spec.dfy} (71%) rename src/JSON/{ => Utils}/Cursors.dfy (98%) rename src/JSON/{ => Utils}/Lexers.dfy (91%) rename src/JSON/{ => Utils}/Parsers.dfy (94%) rename src/JSON/{ => Utils}/Str.dfy (95%) rename src/JSON/{UtfUtils.dfy => Utils/Unicode.dfy} (97%) rename src/JSON/{ => Utils}/Vectors.dfy (97%) rename src/JSON/{ => Utils}/Views.Writers.dfy (95%) rename src/JSON/{ => Utils}/Views.dfy (97%) rename src/JSON/{JSON.ZeroCopy.API.dfy => ZeroCopy/API.dfy} (85%) rename src/JSON/{JSON.ZeroCopy.Deserializer.dfy => ZeroCopy/Deserializer.dfy} (97%) rename src/JSON/{JSON.ZeroCopy.Serializer.dfy => ZeroCopy/Serializer.dfy} (97%) diff --git a/src/JSON/JSON.API.dfy b/src/JSON/API.dfy similarity index 82% rename from src/JSON/JSON.API.dfy rename to src/JSON/API.dfy index 04fd85f0..ffd671fa 100644 --- a/src/JSON/JSON.API.dfy +++ b/src/JSON/API.dfy @@ -1,16 +1,16 @@ -include "JSON.Errors.dfy" -include "JSON.Grammar.dfy" -include "JSON.Spec.dfy" -include "JSON.Serializer.dfy" -include "JSON.Deserializer.dfy" -include "JSON.ZeroCopy.API.dfy" +include "Errors.dfy" +include "Grammar.dfy" +include "Spec.dfy" +include "Serializer.dfy" +include "Deserializer.dfy" +include "ZeroCopy/API.dfy" module {:options "-functionSyntax:4"} JSON.API { // TODO: Propagate proofs import opened BoundedInts import opened Wrappers - import Vs = Views.Core + import Vs = Utils.Views.Core import opened Errors import opened AST diff --git a/src/JSON/JSON.AST.dfy b/src/JSON/AST.dfy similarity index 100% rename from src/JSON/JSON.AST.dfy rename to src/JSON/AST.dfy diff --git a/src/JSON/JSON.Deserializer.dfy b/src/JSON/Deserializer.dfy similarity index 93% rename from src/JSON/JSON.Deserializer.dfy rename to src/JSON/Deserializer.dfy index f0bbab73..96d8a1d5 100644 --- a/src/JSON/JSON.Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -8,13 +8,13 @@ include "../Collections/Sequences/Seq.dfy" include "../BoundedInts.dfy" include "../Math.dfy" -include "Views.dfy" -include "Vectors.dfy" -include "UtfUtils.dfy" -include "JSON.Errors.dfy" -include "JSON.AST.dfy" -include "JSON.Grammar.dfy" -include "JSON.Spec.dfy" +include "Utils/Views.dfy" +include "Utils/Vectors.dfy" +include "Utils/Unicode.dfy" +include "Errors.dfy" +include "AST.dfy" +include "Grammar.dfy" +include "Spec.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { import Seq @@ -22,15 +22,15 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Wrappers import opened BoundedInts - import opened Str - import UtfUtils + import opened Utils.Str + import Utils.Unicode import AST import Spec import opened Errors - import opened Vectors + import opened Utils.Vectors import opened Grammar - import opened Views.Core + import opened Utils.Views.Core function Bool(js: Grammar.jbool): bool { assert js.Bytes() in {Grammar.TRUE, Grammar.FALSE}; @@ -81,7 +81,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { function Transcode8To16Unescaped(str: seq): DeserializationResult // TODO Optimize with a function by method { - Unescape(UtfUtils.Transcode8To16(str)) + Unescape(Unicode.Transcode8To16(str)) } function String(js: Grammar.jstring): DeserializationResult { diff --git a/src/JSON/JSON.Errors.dfy b/src/JSON/Errors.dfy similarity index 97% rename from src/JSON/JSON.Errors.dfy rename to src/JSON/Errors.dfy index 9d6cc490..62ee1a0c 100644 --- a/src/JSON/JSON.Errors.dfy +++ b/src/JSON/Errors.dfy @@ -1,11 +1,11 @@ include "../Wrappers.dfy" include "../BoundedInts.dfy" -include "Str.dfy" +include "Utils/Str.dfy" module {:options "-functionSyntax:4"} JSON.Errors { import Wrappers import opened BoundedInts - import Str + import Utils.Str datatype DeserializationError = | UnterminatedSequence diff --git a/src/JSON/JSON.Grammar.dfy b/src/JSON/Grammar.dfy similarity index 98% rename from src/JSON/JSON.Grammar.dfy rename to src/JSON/Grammar.dfy index 3ebb9a0a..b5c326fb 100644 --- a/src/JSON/JSON.Grammar.dfy +++ b/src/JSON/Grammar.dfy @@ -5,11 +5,11 @@ /// See ``JSON.AST`` for the high-level interface. include "../BoundedInts.dfy" -include "Views.dfy" +include "Utils/Views.dfy" module {:options "-functionSyntax:4"} JSON.Grammar { import opened BoundedInts - import opened Views.Core + import opened Utils.Views.Core const EMPTY := View.OfBytes([]) const DOUBLEQUOTE := View.OfBytes(['\"' as byte]) diff --git a/src/JSON/JSON.LowLevel.Spec.dfy b/src/JSON/LowLevel.Spec.dfy similarity index 97% rename from src/JSON/JSON.LowLevel.Spec.dfy rename to src/JSON/LowLevel.Spec.dfy index 8a7a1d75..85a45f79 100644 --- a/src/JSON/JSON.LowLevel.Spec.dfy +++ b/src/JSON/LowLevel.Spec.dfy @@ -1,9 +1,9 @@ -include "JSON.Grammar.dfy" +include "Grammar.dfy" module {:options "-functionSyntax:4"} JSON.LowLevel.Spec { import opened BoundedInts - import Vs = Views.Core + import Vs = Utils.Views.Core import opened Grammar function View(v: Vs.View) : bytes { diff --git a/src/JSON/JSON.LowLevel.SpecProperties.dfy b/src/JSON/LowLevel.SpecProperties.dfy similarity index 96% rename from src/JSON/JSON.LowLevel.SpecProperties.dfy rename to src/JSON/LowLevel.SpecProperties.dfy index 1659643b..5862e591 100644 --- a/src/JSON/JSON.LowLevel.SpecProperties.dfy +++ b/src/JSON/LowLevel.SpecProperties.dfy @@ -1,9 +1,9 @@ -include "JSON.LowLevel.Spec.dfy" +include "LowLevel.Spec.dfy" module {:options "-functionSyntax:4"} JSON.LowLevel.SpecProperties { import opened BoundedInts - import Vs = Views.Core + import Vs = Utils.Views.Core import opened Grammar import Spec diff --git a/src/JSON/JSON.Serializer.dfy b/src/JSON/Serializer.dfy similarity index 91% rename from src/JSON/JSON.Serializer.dfy rename to src/JSON/Serializer.dfy index 869fceef..61b0b0d2 100644 --- a/src/JSON/JSON.Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -8,28 +8,28 @@ include "../Collections/Sequences/Seq.dfy" include "../BoundedInts.dfy" include "../Math.dfy" -include "Views.dfy" -include "Vectors.dfy" -include "UtfUtils.dfy" -include "JSON.Errors.dfy" -include "JSON.AST.dfy" -include "JSON.Grammar.dfy" -include "JSON.Spec.dfy" +include "Utils/Views.dfy" +include "Utils/Vectors.dfy" +include "Utils/Unicode.dfy" +include "Errors.dfy" +include "AST.dfy" +include "Grammar.dfy" +include "Spec.dfy" module {:options "-functionSyntax:4"} JSON.Serializer { import Seq import Math import opened Wrappers import opened BoundedInts - import opened Str - import UtfUtils + import opened Utils.Str + import Utils.Unicode import AST import Spec import opened Errors - import opened Vectors + import opened Utils.Vectors import opened Grammar - import opened Views.Core + import opened Utils.Views.Core type Result<+T> = SerializationResult @@ -42,7 +42,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { } function Transcode16To8Escaped(str: string, start: uint32 := 0): bytes { - UtfUtils.Transcode16To8(Spec.Escape(str)) + Unicode.Transcode16To8(Spec.Escape(str)) } // FIXME speed up using a `by method` // by method { // var len := |str| as uint32; @@ -57,10 +57,10 @@ module {:options "-functionSyntax:4"} JSON.Serializer { // var c0 := c1; // var c1 := str[idx + 1] as uint16; // if c0 < 0xD800 || c0 > 0xDBFF { - // Utf8Encode(st, UtfUtils.Utf16Decode1(c0)); + // Utf8Encode(st, Unicode.Utf16Decode1(c0)); // idx := idx +1; // } else { - // Utf8Encode(st, UtfUtils.Utf16Decode2(c0, c1)); + // Utf8Encode(st, Unicode.Utf16Decode2(c0, c1)); // idx := idx + 2; // } // } diff --git a/src/JSON/JSON.Spec.dfy b/src/JSON/Spec.dfy similarity index 71% rename from src/JSON/JSON.Spec.dfy rename to src/JSON/Spec.dfy index 2a0c895a..9bce4ebe 100644 --- a/src/JSON/JSON.Spec.dfy +++ b/src/JSON/Spec.dfy @@ -7,16 +7,16 @@ include "../BoundedInts.dfy" -include "JSON.AST.dfy" -include "UtfUtils.dfy" -include "Str.dfy" +include "AST.dfy" +include "Utils/Unicode.dfy" +include "Utils/Str.dfy" module {:options "-functionSyntax:4"} JSON.Spec { import opened BoundedInts - import opened Str + import opened Utils.Str import opened AST - import opened UtfUtils + import opened Utils.Unicode type bytes = seq @@ -51,18 +51,25 @@ module {:options "-functionSyntax:4"} JSON.Spec { + Escape(str, start + 1) } + function ToBytes(s: string) : seq + requires forall c: char | c in s :: c as int < 256 + { + seq(|s|, i requires 0 <= i < |s| => + assert s[i] in s; s[i] as byte) + } + function String(str: string): bytes { - Str.ToBytes("\"") + Transcode16To8(Escape(str)) + Str.ToBytes("\"") + ToBytes("\"") + Transcode16To8(Escape(str)) + ToBytes("\"") } function Number(dec: Decimal): bytes { Transcode16To8(Str.OfInt(dec.n)) + (if dec.e10 == 0 then [] - else Str.ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) + else ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) } function KV(kv: (string, JSON)): bytes { - String(kv.0) + Str.ToBytes(":") + JSON(kv.1) + String(kv.0) + ToBytes(":") + JSON(kv.1) } function Join(sep: bytes, items: seq): bytes { @@ -72,21 +79,21 @@ module {:options "-functionSyntax:4"} JSON.Spec { } function Object(obj: seq<(string, JSON)>): bytes { - Str.ToBytes("{") + - Join(Str.ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KV(obj[i]))) + - Str.ToBytes("}") + ToBytes("{") + + Join(ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KV(obj[i]))) + + ToBytes("}") } function Array(arr: seq): bytes { - Str.ToBytes("[") + - Join(Str.ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => JSON(arr[i]))) + - Str.ToBytes("]") + ToBytes("[") + + Join(ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => JSON(arr[i]))) + + ToBytes("]") } function JSON(js: JSON): bytes { match js - case Null => Str.ToBytes("null") - case Bool(b) => if b then Str.ToBytes("true") else Str.ToBytes("false") + case Null => ToBytes("null") + case Bool(b) => if b then ToBytes("true") else ToBytes("false") case String(str) => String(str) case Number(dec) => Number(dec) case Object(obj) => Object(obj) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index ccfd3717ab5b7b9169a9cf0febffb15cf5c2d308..3a5f7d06852c5b2507b5871604852b0ee09eda3d 100644 GIT binary patch literal 3583 zcmds4U2fwx5Pr`o270lLDsr5(so`A*NbI!TV%rpJZwu543R$9Kr821{sr)nv(6{y$ zJwOl8`}71oLT5zMlC9(+yZcb|AqhDgeKVZ-_>oC@oD~TJ@2yTV)qTs;SR^b+7PsC` z>ef!B6jE&&kH34hW%-dwF%tQ0x49Gq-I2&L7VA`S)t#_+1>-R*M+Rjlnky}%Si?#r zMaJOk$=SETlqs!(A7qs0Ov2p`K)T9>)KI1yOtjoK*&S47ku%N`I1vR;Sn`T%wbNKM z8zj5F%}r<=ub5tn1jbA;nMPUq3v18S0Oq%vsSZe{3&|Cvfx1PDF>$M>m>xFY8Fr=oX=8MHdMx*3DYP|J9`neP*a zAe(`(#P!Wo+@jL7Y#i4+Qw_A6KGD>;JZ$v`&kFSh8{xX)+q$}Wy*qOSgJH8gy~C;K z6Nb&A(Rhbhu0NiFYF}Z;QBW{0d-u{X%fM76`>xG;14@kpr+b68Fi{B*&^jkft%M<&}y4qQHW{f z@k<%4R#8nH9cX~sT)Dj#*RRgV+ZJ_*aU14 zr+${7E0{LcT1UJQ-U6sb*u-lY#LFnisn2pA<3PP^wKi)|2QoQ~W?(w`p#M zrIwVnCk<?23fD7oA@OEmuXOX+VtvHtd7?}r$YFoS;Kw=%CwRhK9@GU#z7_jr7yzm8l z0B^uKRa@KBJu`c85Y9uin(<6`S9MkOsngZtzyFb;9Lv5O%9-?}FPXIDTPb9q_aWMI zX`^Q#zoO@Td9P;tm$EHS`s$3XZjRpM}!LKfS9LjI{wwQ*8kjdXzi>(P~=GB!w=q%JP z#yx{?Cy39Pu0$rmqaB@{F>A~BI{sASJA%xG>dNpmgnU?mZPmqjlJhSfP@ z>2emh|CA|!RU##L>$V)BwU8GYkxc62zpN3w1L-~JDKttecpItg8U7N-nJmjs_{(v1 zdb*G_%s*2NjDH~;llb44l}U_atG3nlea({?UtvaH#d^Dl0NRxzw8^I6=QOKQv|pz+ zTUbN#%FkE@_cnO(A$<3>@kp_aBbPKX@z}hvhgwX+_1LP_VM`=Emv0`Xoi`(bD zv7Xn8%;}kB`;TXop+y*lE!5=Bc#@rH7pvD z9Co}?UMyf0YeVy$_14DNLW>=FKIN8y$}n~wgIjo_rc5pqy_Xef;>h!#bDW~$AfH$? zW2-ut<9ULhSiT2Fb6H#SHY(r2xXN3Gd^t{(jpYNUx)#=1FA<+h$Wl8!Aakq=SKvon z#&b(}igD@M%Y4+54%pAuj@tOtf=`UMw|fk%ua04;*%1;WJ4Ddt2>#Uh9a~m6lOc)o z*;7E)Jk?=Z`=-?*E29D8%F3J>l=g=`EUqTwYA%+tN}d+SGS!N`IwrN(&n?a^vBoDr z__;iR?L_Gzs$u7Jj(3@AO-aOL65G?B)OTgsxW1dlti(^KO_-zqebt%fJ@*065}S(HPai`yRp%jjn4?k+^VP4Zf7KeNrDs!~9j)fFj6FxJ+o%>^pl%?R z+oe&<;G5<=F7F%L;JtnPmcdI?Zd(ppXw<gf;G=A~RRa5_jCQc{<{LJjCB~)yOKCXHe?c)7~C>2l(+^ zXXg*+x~uFFUqIUd<_$a=^SR~u+q6rB?Bu`^V#ayV=(5C zvL(NXk|kMb+c#U*4gq<&axi=L(Ml9p3%@{+adWhF(PuL;HaYk>Loe@PA8`gIBhB77 zeDvacUEbliGw?eJ_nGhTpM@$9M8428PoXEa+2?4yPQ(~;xa#hO ze>3v;xPtl$xkHxVS*^@(L{IufAg)dMwU~@ufb+r^d!F86J}jSIsC8D&dlkJUF8>$D CtcQdE diff --git a/src/JSON/Cursors.dfy b/src/JSON/Utils/Cursors.dfy similarity index 98% rename from src/JSON/Cursors.dfy rename to src/JSON/Utils/Cursors.dfy index c775e367..2e346331 100644 --- a/src/JSON/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -1,9 +1,9 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" +include "../../BoundedInts.dfy" +include "../../Wrappers.dfy" include "Views.dfy" include "Lexers.dfy" -module {:options "-functionSyntax:4"} Cursors { +module {:options "-functionSyntax:4"} JSON.Utils.Cursors { import opened BoundedInts import opened Wrappers diff --git a/src/JSON/Lexers.dfy b/src/JSON/Utils/Lexers.dfy similarity index 91% rename from src/JSON/Lexers.dfy rename to src/JSON/Utils/Lexers.dfy index aee4788f..b63f4a59 100644 --- a/src/JSON/Lexers.dfy +++ b/src/JSON/Utils/Lexers.dfy @@ -1,7 +1,7 @@ -include "../Wrappers.dfy" -include "../BoundedInts.dfy" +include "../../Wrappers.dfy" +include "../../BoundedInts.dfy" -module {:options "-functionSyntax:4"} Lexers { +module {:options "-functionSyntax:4"} JSON.Utils.Lexers { module Core { import opened Wrappers import opened BoundedInts diff --git a/src/JSON/Parsers.dfy b/src/JSON/Utils/Parsers.dfy similarity index 94% rename from src/JSON/Parsers.dfy rename to src/JSON/Utils/Parsers.dfy index 7cd8de62..102fe595 100644 --- a/src/JSON/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -1,8 +1,8 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" +include "../../BoundedInts.dfy" +include "../../Wrappers.dfy" include "Cursors.dfy" -module {:options "-functionSyntax:4"} Parsers { +module {:options "-functionSyntax:4"} JSON.Utils.Parsers { import opened BoundedInts import opened Wrappers diff --git a/src/JSON/Str.dfy b/src/JSON/Utils/Str.dfy similarity index 95% rename from src/JSON/Str.dfy rename to src/JSON/Utils/Str.dfy index 2ee820af..38e4e251 100644 --- a/src/JSON/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,8 +1,8 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" -include "../Math.dfy" +include "../../BoundedInts.dfy" +include "../../Wrappers.dfy" +include "../../Math.dfy" -module {:options "-functionSyntax:4"} Str { +module {:options "-functionSyntax:4"} JSON.Utils.Str { import opened Wrappers import Math @@ -241,13 +241,4 @@ module {:options "-functionSyntax:4"} Str { function Concat(strs: seq) : string { Join("", strs) } - - import opened BoundedInts - - function ToBytes(s: string) : seq - requires forall c: char | c in s :: c as int < 256 - { - seq(|s|, i requires 0 <= i < |s| => - assert s[i] in s; s[i] as byte) - } } diff --git a/src/JSON/UtfUtils.dfy b/src/JSON/Utils/Unicode.dfy similarity index 97% rename from src/JSON/UtfUtils.dfy rename to src/JSON/Utils/Unicode.dfy index 1599a4d0..518e1df1 100644 --- a/src/JSON/UtfUtils.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -1,6 +1,6 @@ -include "../BoundedInts.dfy" +include "../../BoundedInts.dfy" -module {:options "-functionSyntax:4"} UtfUtils { +module {:options "-functionSyntax:4"} JSON.Utils.Unicode { import opened BoundedInts function Utf16Decode1(c: uint16): uint32 diff --git a/src/JSON/Vectors.dfy b/src/JSON/Utils/Vectors.dfy similarity index 97% rename from src/JSON/Vectors.dfy rename to src/JSON/Utils/Vectors.dfy index 8c052fa4..7065290e 100644 --- a/src/JSON/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -1,7 +1,7 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" +include "../../BoundedInts.dfy" +include "../../Wrappers.dfy" -module {:options "-functionSyntax:4"} Vectors { +module {:options "-functionSyntax:4"} JSON.Utils.Vectors { import opened BoundedInts import opened Wrappers diff --git a/src/JSON/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy similarity index 95% rename from src/JSON/Views.Writers.dfy rename to src/JSON/Utils/Views.Writers.dfy index 4e92f66c..081b044b 100644 --- a/src/JSON/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -1,8 +1,8 @@ -include "../BoundedInts.dfy" -include "../Wrappers.dfy" +include "../../BoundedInts.dfy" +include "../../Wrappers.dfy" include "Views.dfy" -module {:options "-functionSyntax:4"} Views.Writers { +module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { import opened BoundedInts import opened Wrappers diff --git a/src/JSON/Views.dfy b/src/JSON/Utils/Views.dfy similarity index 97% rename from src/JSON/Views.dfy rename to src/JSON/Utils/Views.dfy index e517f859..6e7ab42f 100644 --- a/src/JSON/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -1,6 +1,6 @@ -include "../BoundedInts.dfy" +include "../../BoundedInts.dfy" -module {:options "-functionSyntax:4"} Views.Core { +module {:options "-functionSyntax:4"} JSON.Utils.Views.Core { import opened BoundedInts type View = v: View_ | v.Valid? witness View([], 0, 0) diff --git a/src/JSON/JSON.ZeroCopy.API.dfy b/src/JSON/ZeroCopy/API.dfy similarity index 85% rename from src/JSON/JSON.ZeroCopy.API.dfy rename to src/JSON/ZeroCopy/API.dfy index 87238c9f..bf55622e 100644 --- a/src/JSON/JSON.ZeroCopy.API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,13 +1,13 @@ -include "JSON.Errors.dfy" -include "JSON.Grammar.dfy" -include "JSON.LowLevel.Spec.dfy" -include "JSON.ZeroCopy.Serializer.dfy" -include "JSON.ZeroCopy.Deserializer.dfy" +include "../Errors.dfy" +include "../Grammar.dfy" +include "../LowLevel.Spec.dfy" +include "Serializer.dfy" +include "Deserializer.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { import opened BoundedInts import opened Wrappers - import Vs = Views.Core + import Vs = Utils.Views.Core import opened Grammar import opened Errors diff --git a/src/JSON/JSON.ZeroCopy.Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy similarity index 97% rename from src/JSON/JSON.ZeroCopy.Deserializer.dfy rename to src/JSON/ZeroCopy/Deserializer.dfy index f2240b11..e9ed18f1 100644 --- a/src/JSON/JSON.ZeroCopy.Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,8 +1,8 @@ -include "JSON.Errors.dfy" -include "JSON.Grammar.dfy" -include "JSON.LowLevel.Spec.dfy" -include "JSON.LowLevel.SpecProperties.dfy" -include "Parsers.dfy" +include "../Errors.dfy" +include "../Grammar.dfy" +include "../LowLevel.Spec.dfy" +include "../LowLevel.SpecProperties.dfy" +include "../Utils/Parsers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { module Core { @@ -10,9 +10,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Wrappers import LowLevel.Spec - import Vs = Views.Core - import opened Cursors - import opened Parsers + import Vs = Utils.Views.Core + import opened Utils.Cursors + import opened Utils.Parsers import opened Grammar import Errors @@ -82,7 +82,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Grammar - import opened Cursors + import opened Utils.Cursors import opened Core const OPEN: byte @@ -105,10 +105,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Params: SequenceParams import LowLevel.SpecProperties - import opened Vs = Views.Core + import opened Vs = Utils.Views.Core import opened Grammar - import opened Cursors - import Parsers + import opened Utils.Cursors + import Utils.Parsers import opened Core const SEPARATOR: byte := ',' as byte @@ -280,11 +280,11 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Wrappers - import opened Vs = Views.Core + import opened Vs = Utils.Views.Core import opened Grammar import opened Core import opened Errors - import Cursors + import Utils.Cursors import Values function LiftCursorError(err: Cursors.CursorError): DeserializationError { @@ -322,7 +322,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Wrappers import opened Grammar - import opened Cursors + import opened Utils.Cursors import opened Core import Strings @@ -377,7 +377,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Grammar import opened Core - import opened Cursors + import opened Utils.Cursors function {:opaque} Constant(cs: FreshCursor, expected: bytes) : (pr: ParseResult) requires |expected| < TWO_TO_THE_32 @@ -394,10 +394,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Grammar - import opened Cursors - import opened LC = Lexers.Core - import opened Lexers.Strings - import opened Parsers + import opened Utils.Cursors + import opened LC = Utils.Lexers.Core + import opened Utils.Lexers.Strings + import opened Utils.Parsers import opened Core function {:opaque} StringBody(cs: Cursor): (pr: CursorResult) @@ -446,7 +446,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Wrappers import opened Grammar - import opened Cursors + import opened Utils.Cursors import opened Core function {:opaque} Digits(cs: FreshCursor) : (sp: Split) diff --git a/src/JSON/JSON.ZeroCopy.Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy similarity index 97% rename from src/JSON/JSON.ZeroCopy.Serializer.dfy rename to src/JSON/ZeroCopy/Serializer.dfy index e93dc61e..c5cd7d8e 100644 --- a/src/JSON/JSON.ZeroCopy.Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,7 +1,7 @@ -include "JSON.Errors.dfy" -include "JSON.LowLevel.Spec.dfy" -include "JSON.LowLevel.SpecProperties.dfy" -include "Views.Writers.dfy" +include "../Errors.dfy" +include "../LowLevel.Spec.dfy" +include "../LowLevel.SpecProperties.dfy" +include "../Utils/Views.Writers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import opened BoundedInts @@ -11,8 +11,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import LowLevel.Spec import LowLevel.SpecProperties import opened Grammar - import opened Views.Writers - import opened Vs = Views.Core // DISCUSS: Module naming convention? + import opened Utils.Views.Writers + import opened Vs = Utils.Views.Core // DISCUSS: Module naming convention? method Serialize(js: JSON) returns (rbs: SerializationResult>) ensures rbs.Success? ==> fresh(rbs.value) From 6ba17352b281555bad60167406e3c86c7b364ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 1 Sep 2022 22:49:58 -0700 Subject: [PATCH 14/84] json: Clean up API files --- src/JSON/API.dfy | 41 +++++++++++++++------------------------ src/JSON/Tests.dfy | 2 +- src/JSON/ZeroCopy/API.dfy | 16 +++++++-------- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index ffd671fa..976d134e 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -1,46 +1,37 @@ -include "Errors.dfy" -include "Grammar.dfy" -include "Spec.dfy" include "Serializer.dfy" include "Deserializer.dfy" include "ZeroCopy/API.dfy" module {:options "-functionSyntax:4"} JSON.API { - // TODO: Propagate proofs - import opened BoundedInts - import opened Wrappers - import Vs = Utils.Views.Core - import opened Errors - import opened AST - import Spec - import S = Serializer - import DS = Deserializer - import ZAPI = ZeroCopy.API + import AST + import Serializer + import Deserializer + import ZeroCopy = ZeroCopy.API - function {:opaque} Serialize(js: JSON) : (bs: SerializationResult>) + function {:opaque} Serialize(js: AST.JSON) : (bs: SerializationResult>) { - var js :- S.JSON(js); - Success(ZAPI.Serialize(js)) + var js :- Serializer.JSON(js); + ZeroCopy.Serialize(js) } - method SerializeAlloc(js: JSON) returns (bs: SerializationResult>) + method SerializeAlloc(js: AST.JSON) returns (bs: SerializationResult>) { - var js :- S.JSON(js); - bs := ZAPI.SerializeAlloc(js); + var js :- Serializer.JSON(js); + bs := ZeroCopy.SerializeAlloc(js); } - method SerializeBlit(js: JSON, bs: array) returns (len: SerializationResult) + method SerializeBlit(js: AST.JSON, bs: array) returns (len: SerializationResult) modifies bs { - var js :- S.JSON(js); - len := ZAPI.SerializeBlit(js, bs); + var js :- Serializer.JSON(js); + len := ZeroCopy.SerializeBlit(js, bs); } - function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) + function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) { - var js :- ZAPI.Deserialize(bs); - DS.JSON(js) + var js :- ZeroCopy.Deserialize(bs); + Deserializer.JSON(js) } } diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 3a5f7d06..981da8d7 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -59,7 +59,7 @@ module JSON.Tests.ZeroCopyWrapper refines Wrapper { method Serialize(js: JSON) returns (bs: SerializationResult) { // print "Count: ", wr.chain.Count(), "\n"; - bs := Success(API.Serialize(js)); + bs := API.Serialize(js); } method SpecSerialize(js: JSON) returns (bs: SerializationResult) { diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index bf55622e..17372a84 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,4 +1,3 @@ -include "../Errors.dfy" include "../Grammar.dfy" include "../LowLevel.Spec.dfy" include "Serializer.dfy" @@ -7,28 +6,27 @@ include "Deserializer.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { import opened BoundedInts import opened Wrappers - import Vs = Utils.Views.Core - import opened Grammar import opened Errors + import Grammar import LowLevel.Spec import Serializer import Deserializer - function {:opaque} Serialize(js: JSON) : (bs: seq) - ensures bs == Spec.JSON(js) + function {:opaque} Serialize(js: Grammar.JSON) : (bs: SerializationResult>) + ensures bs == Success(Spec.JSON(js)) { - Serializer.Text(js).Bytes() + Success(Serializer.Text(js).Bytes()) } - method SerializeAlloc(js: JSON) returns (bs: SerializationResult>) + method SerializeAlloc(js: Grammar.JSON) returns (bs: SerializationResult>) ensures bs.Success? ==> fresh(bs.value) ensures bs.Success? ==> bs.value[..] == Spec.JSON(js) { bs := Serializer.Serialize(js); } - method SerializeBlit(js: JSON, bs: array) returns (len: SerializationResult) + method SerializeBlit(js: Grammar.JSON, bs: array) returns (len: SerializationResult) modifies bs ensures len.Success? ==> len.value as int <= bs.Length ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) @@ -38,7 +36,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { len := Serializer.SerializeTo(js, bs); } - function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) + function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) ensures js.Success? ==> bs == Spec.JSON(js.value) { Deserializer.API.OfBytes(bs) From 74f9b0bf8c094aaf74e44c42d2eadc81474fba6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 00:33:06 -0700 Subject: [PATCH 15/84] json: Merge latest changes to the vectors library --- src/JSON/Utils/Vectors.dfy | 76 ++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/src/JSON/Utils/Vectors.dfy b/src/JSON/Utils/Vectors.dfy index 7065290e..ee42ff26 100644 --- a/src/JSON/Utils/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -6,9 +6,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { import opened Wrappers datatype VectorError = OutOfMemory + const OOM_FAILURE := Fail(OutOfMemory) class Vector { - ghost var Repr : seq + ghost var items : seq + ghost var Repr : set const a: A var size: uint32 @@ -19,31 +21,59 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { const MAX_CAPACITY_BEFORE_DOUBLING: uint32 := UINT32_MAX / 2 ghost predicate Valid?() - reads this, data + reads this, Repr + ensures Valid?() ==> this in Repr { + && Repr == {this, data} && capacity != 0 && data.Length == capacity as int && size <= capacity - && Repr == data[..size] + && size as int == |items| + && items == data[..size] } constructor(a0: A, initial_capacity: uint32 := 8) requires initial_capacity > 0 + ensures size == 0 + ensures items == [] + ensures fresh(Repr) ensures Valid?() { a := a0; - Repr := []; + items := []; size := 0; capacity := initial_capacity; data := new A[initial_capacity](_ => a0); + Repr := {this, data}; } - method At(idx: uint32) returns (a: A) + function At(idx: uint32) : (a: A) + reads this, this.data requires idx < size requires Valid?() - ensures a == data[idx] == Repr[idx] + ensures a == data[idx] == items[idx] { - return data[idx]; + data[idx] + } + + function Top() : (a: A) + reads this, this.data + requires 0 < size + requires Valid?() + ensures a == data[size - 1] == items[size - 1] + { + data[size - 1] + } + + method Put(idx: uint32, a: A) + requires idx < size + requires Valid?() + modifies data, `items + ensures Valid?() + ensures items == old(items)[idx := a] + { + data[idx] := a; + items := items[idx := a]; } method Blit(new_data: array, count: uint32) @@ -67,16 +97,15 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { method Realloc(new_capacity: uint32) requires Valid?() requires new_capacity > capacity - modifies this, `data + modifies `capacity, `data, `Repr, data ensures Valid?() - ensures Repr == old(Repr) - ensures size == old(size) ensures capacity == new_capacity ensures fresh(data) { var old_data, old_capacity := data, capacity; data, capacity := new A[new_capacity](_ => a), new_capacity; Blit(old_data, old_capacity); + Repr := {this, data}; } function DefaultNewCapacity(capacity: uint32) : uint32 { @@ -87,11 +116,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { method ReallocDefault() returns (o: Outcome) requires Valid?() - modifies this, `data + modifies `capacity, `data, `Repr, data ensures Valid?() - ensures Repr == old(Repr) - ensures size == old(size) - ensures old(capacity) == MAX_CAPACITY <==> o.Fail? + ensures o.Fail? <==> old(capacity) == MAX_CAPACITY ensures o.Fail? ==> && unchanged(this) && unchanged(data) @@ -113,8 +140,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { requires Valid?() modifies this, `data ensures Valid?() - ensures Repr == old(Repr) - ensures size == old(size) + modifies `capacity, `data, `Repr, data ensures reserved <= capacity - size ==> o.Pass? ensures o.Pass? ==> @@ -139,29 +165,29 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { return Pass; } - method PopFast(a: A) + method PopFast() requires Valid?() requires size > 0 - modifies this, data + modifies `size, `items ensures Valid?() ensures size == old(size) - 1 - ensures Repr == old(Repr[..|Repr| - 1]) + ensures items == old(items[..|items| - 1]) { size := size - 1; - Repr := Repr[..|Repr| - 1]; + items := items[..|items| - 1]; } method PushFast(a: A) requires Valid?() requires size < capacity - modifies this, data + modifies `size, `items, data ensures Valid?() ensures size == old(size) + 1 - ensures Repr == old(Repr) + [a] + ensures items == old(items) + [a] { data[size] := a; size := size + 1; - Repr := Repr + [a]; + items := items + [a]; } method Push(a: A) returns (o: Outcome) @@ -174,7 +200,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { ensures o.Pass? ==> && old(size) < MAX_CAPACITY && size == old(size) + 1 - && Repr == old(Repr) + [a] + && items == old(items) + [a] + && capacity >= old(capacity) + && if old(size == capacity) then fresh(data) else unchanged(`data) { if size == capacity { var d := ReallocDefault(); From a0a816698a2d1a239caf9d53e3e6de8e7cadbfac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 01:40:59 -0700 Subject: [PATCH 16/84] json: Move IntPow and IntLog to their own utility module --- src/JSON/Deserializer.dfy | 4 +- src/JSON/Utils/Math.dfy | 108 ++++++++++++++++++++++++ src/JSON/Utils/Str.dfy | 2 +- src/Math.dfy | 134 ------------------------------ src/NonlinearArithmetic/Power.dfy | 3 - 5 files changed, 111 insertions(+), 140 deletions(-) create mode 100644 src/JSON/Utils/Math.dfy diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 96d8a1d5..8a1aaf13 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -6,7 +6,7 @@ include "../Collections/Sequences/Seq.dfy" include "../BoundedInts.dfy" -include "../Math.dfy" +include "Utils/Math.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" @@ -18,7 +18,7 @@ include "Spec.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { import Seq - import Math + import Utils.Math import opened Wrappers import opened BoundedInts diff --git a/src/JSON/Utils/Math.dfy b/src/JSON/Utils/Math.dfy new file mode 100644 index 00000000..d953393e --- /dev/null +++ b/src/JSON/Utils/Math.dfy @@ -0,0 +1,108 @@ +// include "../.s./NonlinearArithmetic/DivMod.dfy" // Only needed for unused proofs + +module JSON.Utils.Math { + function method {:opaque} IntPow(base: int, n: nat): int { + if n == 0 then 1 + else base * IntPow(base, n - 1) + } + + lemma {:induction n} IntPow_Nat(base: nat, n: nat) + ensures IntPow(base, n) >= 0 + { + reveal IntPow(); + } + + lemma {:induction n} IntPow_NonZero(base: int, n: nat) + requires base != 0 + ensures IntPow(base, n) != 0 + { + reveal IntPow(); + } + + function method {:opaque} IntLog(base: nat, pow: nat): nat + requires base > 1 + decreases pow + { + if pow < base then 0 + else + assert pow < pow * base; assert pow / base < pow; + 1 + IntLog(base, pow / base) + } + + lemma {:induction false} Divide_Increasing(base: nat, pow: nat, pow': nat) + requires base > 1 + requires pow <= pow' + ensures pow / base <= pow' / base + { + var q, q', r, r' := pow / base, pow' / base, pow % base, pow' % base; + assert pow == q * base + r; + assert pow' == q' * base + r'; + assert 0 <= r < base && 0 <= r' < base; + var dp, dq, dr := pow' - pow, q' - q, r' - r; + assert dp == dq * base + dr; + if dq < 0 { + assert dq * base < -base; + assert dr <= base; + assert dp == dq * base + dr < 0; + assert false; + } + } + + lemma {:induction false} IntLog_Increasing(base: nat, pow: nat, pow': nat) + requires base > 1 + requires pow <= pow' + ensures IntLog(base, pow) <= IntLog(base, pow') + decreases pow + { + reveal IntLog(); + if pow' < base { + assert IntLog(base, pow) == 0 == IntLog(base, pow'); + } else if pow < base { + assert IntLog(base, pow) == 0; + } else { + assert pow < pow * base; assert pow / base < pow; + assert pow' < pow' * base; assert pow' / base < pow'; + assert pow / base <= pow' / base by { Divide_Increasing(base, pow, pow'); } + IntLog_Increasing(base, pow / base, pow' / base); + } + } + + /* + import DivMod + + lemma {:induction false} Multiply_Divide(p: int, q: nat) + requires q != 0 + ensures q * p / q == p + { + DivMod.LemmaDivMultiplesVanish(p, q); + } + + lemma {:induction false} IntLog_IntPow(base: nat, n: nat) + requires base > 1 + ensures (IntPow_Nat(base, n); IntLog(base, IntPow(base, n)) == n) + { + if n == 0 { + reveal IntPow(); + reveal IntLog(); + } else { + IntPow_Nat(base, n); + calc { + IntLog(base, IntPow(base, n)); + { reveal IntPow(); } + IntLog(base, base * IntPow(base, n - 1)); + { reveal IntLog(); IntPow_Nat(base, n - 1); IntPow_NonZero(base, n - 1); } + 1 + IntLog(base, base * IntPow(base, n - 1) / base); + { Multiply_Divide(IntPow(base, n - 1), base); } + 1 + IntLog(base, IntPow(base, n - 1)); + { IntLog_IntPow(base, n - 1); } + 1 + (n - 1); + } + } + } + + lemma {:induction false} IntPow_IntLog(base: nat, pow: nat) + requires base > 1 + requires pow > 0 + ensures IntPow(base, IntLog(base, pow)) <= pow < IntPow(base, IntLog(base, pow) + 1) + */ +} diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 38e4e251..f93c3492 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,6 +1,6 @@ include "../../BoundedInts.dfy" include "../../Wrappers.dfy" -include "../../Math.dfy" +include "Math.dfy" module {:options "-functionSyntax:4"} JSON.Utils.Str { import opened Wrappers diff --git a/src/Math.dfy b/src/Math.dfy index 73edcf41..17b0f2bc 100644 --- a/src/Math.dfy +++ b/src/Math.dfy @@ -27,138 +27,4 @@ module Math { { if a >= 0 then a else -a } - - function method {:opaque} IntPow(base: int, n: nat): int { - if n == 0 then 1 - else base * IntPow(base, n - 1) - } - - function method {:opaque} IntLog(base: nat, pow: nat): nat - requires base > 1 - decreases pow - // TODO: pow > 0 ==> IntPow(base, IntLog(base, pow)) <= pow < IntPow(base, IntLog(base, pow) + 1) - { - if pow < base then 0 - else - assert pow < pow * base; assert pow / base < pow; - 1 + IntLog(base, pow / base) - } - - lemma {:induction false} Divide_Increasing(base: nat, pow: nat, pow': nat) - requires base > 1 - requires pow <= pow' - ensures pow / base <= pow' / base - { - var q, q', r, r' := pow / base, pow' / base, pow % base, pow' % base; - assert pow == q * base + r; - assert pow' == q' * base + r'; - assert 0 <= r < base && 0 <= r' < base; - var dp, dq, dr := pow' - pow, q' - q, r' - r; - assert dp == dq * base + dr; - if dq < 0 { - assert dq * base < -base; - assert dr <= base; - assert dp == dq * base + dr < 0; - assert false; - } - } - - lemma {:induction false} IntLog_Increasing(base: nat, pow: nat, pow': nat) - requires base > 1 - requires pow <= pow' - ensures IntLog(base, pow) <= IntLog(base, pow') - decreases pow - { - reveal IntLog(); - if pow' < base { - assert IntLog(base, pow) == 0 == IntLog(base, pow'); - } else if pow < base { - assert IntLog(base, pow) == 0; - } else { - assert pow < pow * base; assert pow / base < pow; - assert pow' < pow' * base; assert pow' / base < pow'; - assert pow / base <= pow' / base by { Divide_Increasing(base, pow, pow'); } - IntLog_Increasing(base, pow / base, pow' / base); - } - } - - // lemma {:induction n} IntPow_Nat(base: nat, n: nat) - // ensures IntPow(base, n) >= 0 - // { - // reveal IntPow(); - // } - - // lemma {:induction n} IntPow_NonZero(base: int, n: nat) - // requires base != 0 - // ensures IntPow(base, n) != 0 - // { - // reveal IntPow(); - // } - - // function method {:opaque} NatPow(base: nat, n: nat): nat { - // IntPow_Nat(base, n); - // IntPow(base, n) - // } - - // lemma {:induction false} NatPow_NonZero(base: nat, n: nat) - // requires base != 0 - // ensures NatPow(base, n) != 0 - // { - // reveal NatPow(); IntPow_NonZero(base, n); - // } - - // function method {:opaque} IntLogSup(base: nat, pow: nat): (log: nat) - // requires base > 1 - // ensures log > 0 - // decreases pow - // // IntPow(base, IntLogSup(base, pow) - 1) - 1 <= pow <= IntPow(base, IntLogSup(base, pow)) - // { - // if pow < base then 1 - // else - // assert pow < pow * base; - // assert pow / base < pow; - // 1 + IntLog(base, pow / base) - // } - - // // lemma {:induction false} IntLogSup_transfer(base: nat, n: nat, n': nat) - // // requires base > 1 - // // requires n > n' - // // ensures - // // (NatPow_NonZero(base, n); NatPow_NonZero(base, n'); - // // IntLogSup(base, NatPow(base, n), NatPow(base, n')) == IntLogSup(base, NatPow(base, n - n'))) - // // { - // // reveal NatPow(); - // // reveal IntPow(); - // // reveal IntLogSup(); - // // if n' == 0 { - // // } else { - // // NatPow_NonZero(base, n); NatPow_NonZero(base, n'); NatPow_NonZero(base, n' - 1); - // // calc { - // // IntLogSup(base, NatPow(base, n), NatPow(base, n')); - // // 1 + IntLogSup(base, NatPow(base, n), NatPow(base, n' - 1) * base) - 1; - // // { assume NatPow(base, n) > NatPow(base, n'); } // FIXME wrong - // // IntLogSup(base, NatPow(base, n), NatPow(base, n' - 1)) - 1; - // // { IntLogSup_transfer(); } - // // IntLogSup(base, NatPow(base, n - n')) - // // } - // // } - // // } - - // lemma {:induction false} IntPow_IntLogSup(base: nat, n: nat) - // requires base > 1 - // ensures IntLogSup(base, NatPow(base, n)) - 1 < n <= IntLogSup(base, NatPow(base, n)) - // { - // reveal IntLogSup(); - // reveal IntPow(); - // reveal NatPow(); - // if n == 0 { - // } else { - // calc { - // IntLogSup(base, NatPow(base, n)); - // IntLogSup(base, NatPow(base, n - 1) * base); - // IntLogSup(base, NatPow(base, n - 1) * base); - // 1 + IntLogSup(base, NatPow(base, n - 1)); - // } - // } - // } } diff --git a/src/NonlinearArithmetic/Power.dfy b/src/NonlinearArithmetic/Power.dfy index 61f74bec..bda7a18a 100644 --- a/src/NonlinearArithmetic/Power.dfy +++ b/src/NonlinearArithmetic/Power.dfy @@ -12,7 +12,6 @@ former takes arguments and may be more stable and less reliant on Z3 heuristics. The latter includes automation and its use requires less effort */ -include "../Math.dfy" include "DivMod.dfy" include "Internals/GeneralInternals.dfy" include "Mul.dfy" @@ -27,9 +26,7 @@ module Power { function method {:opaque} Pow(b: int, e: nat): int decreases e - ensures Pow(b, e) == Math.IntPow(b, e) { - reveal Math.IntPow(); if e == 0 then 1 else From c032f8be86c089d3be81af6ea27a5e3f890c6b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 02:14:09 -0700 Subject: [PATCH 17/84] json: Add a README --- src/JSON/README.md | 108 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 src/JSON/README.md diff --git a/src/JSON/README.md b/src/JSON/README.md new file mode 100644 index 00000000..b5937d72 --- /dev/null +++ b/src/JSON/README.md @@ -0,0 +1,108 @@ +# JSON + +JSON serialization and deserialization in Dafny, as described in [RFC 8259](https://datatracker.ietf.org/doc/html/rfc8259). + +This library provides two APIs: + +- A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use (in particular, its JSON AST exposes strings as unescaped, undecoded utf-8 byte sequences of type `seq`). + +- A high-level API built on top of the zero-copy API that is unverified and less efficient, but is more convenient to use (in particular, it handles encoding and escaping: its JSON AST uses Dafny's `string` type). + +Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you nead to read or produce JSON text in other encodings. + +## Library usage + +The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, call the high-level API, and use the low-level API to make localized modifications to a partial parse of a JSON AST. The main entry points are `API.Serialize` (to go from utf-8 bytes to a JSON AST), and `API.Deserialize` (for the reverse operation): + +```dafny +var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4584.2}]}"); + +var CITY_AST := Object([("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1)))])]))]); + +expect API.Deserialize(CITY_JS) == Success(CITY_AST); + +expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( + @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" +)); +``` + +## What is verified? + +The zero-copy serializer is proved sound and complete against a simple functional specification found in [`LowLevel.Spec.dfy`](./LowLevel.Spec.dfy). The low-level deserializer is proven sound, but not complete, against that same specification: if a value is decoded + +### Useful submodules + +Most of the contents of the `Utils/` directory are not specific to JSON. They are not sufficiently documented to be moved to the main library yet, but they provide useful functionality: + +- [`Views.dfy`](Utils/Views.dfy) and [`Views.Writers.dfy`](Utils/Views.Writers.dfy) implement slices over byte strings whose bounds are representable as `int32` native integers. Adjacent slices can be merged in time `O(1)` (but see [“Caveats”](#caveats) below). To be part of the main library, it would have to be generalized to slices over any sequences and to indices represented using arbitrary integers (though Dafny does not currently have a way to have modularity over int width). + +- [`Cursors.dfy`](Utils/Cursors.dfy) implements slices augmented with an inner pointer tracking a position. Functions are provided to skip over a specific byte or bytes, over bytes matching a given predicate, or over bytes recognized by a given lexer, and ghost functions are provided to express the fact that a cursor was obtained by trimming a prefix of another cursor. Cursors are used in the implementation of the parser: after skipping over a given construct, a cursor can be split into a view (capturing the part of the string matching the construct) and a new cursor starting at the previous position, with its pointer reset to the beginning of the string. To become part of the main library, cursors would need to be generalized in the same way as views above. + +- [`Lexers.dfy`](Utils/Lexers.dfy) contains a state machine that recognizes quoted strings and skips over backslash-escaped quotes. State-machine-based lexers are used in `Cursors.dfy` to skip over complex constructs (like quoted strings), though in practice the low-level parser uses a custom function-by-method to speed up this specific case. + +- [`Unicode.dfy`](Utils/Unicode.dfy) implements conversions between UTF-8 and UTF-16. These conversions would have to be more efficient to be included in the main library. [This page](http://bjoern.hoehrmann.de/utf-8/decoder/dfa/) provides reasonably efficient DFAs for this task; it would be an interesting project to port them to Dafny and verify them against the Unicode library in [`../Unicode/`](../Unicode/). + +- [`Vectors.dfy`](Utils/Vectors.dfy) implements resizable arrays (the C++ standard library calls them vectors). This module is reasonably close to being generally useful, but the way it tracks references needs to be revised if it if to be used for storing reference types. It would also be good to generalize the index type that it uses (at the moment, it only accepts `int32` indices). + +- [`Str.dfy`](Utils/Str.dfy) implements functions to convert between (big-endian) strings and numbers. This module shares similarities with [LittleEndianNat.dfy](../Collections/Sequences/LittleEndianNat.dfy) (except for the digit order), although `LittleEndianNat.dfy` makes the base a module parameter instead of a function argument. + +- [`Math.dfy`](Utils/Math.dfy) implements `IntPow` and its left inverse `IntLog`. These functions are used to characterize the length of strings produced from numbers and the value of numbers produced from strings. They could be migrated to [`NonlinearArithmetic/Power.dfy`](../NonlinearArithmetic/Power.dfy), but the `Nonlinear` part of the library uses custom Dafny flags for arithmetic and Dafny does not (yet) support specifying these flags per-module. + +- [`Parsers.dfy`](Utils/Parsers.dfy) has definitions of well-formedness for parsers (stating that they must consume part of their input). This file would have to be significantly expanded to create a composable library of parsing combinators to be useful as part of the main library. + +## Caveats + +- Not all parts of this library are verified. In particular, the high-level API is not verified (the most complex part of it is the string escaping code). + +- The optimization used to merge contiguous slices can have poor performance in corner cases. Specifically, the function `View.Adjacent`, which checks whether two slices can be concatenated efficiently, uses Dafny's value equality, not reference equality, to check whether both slices point into the same string. The value-equality check is `O(1)` when the strings are physically equal, but may take linear time otherwise. + + The worst-case behavior happens when serializing a low-level JSON object in which the `View`s all point to (physically) different strings with equal contents. This cannot happen when modifying the result of a previous parse (in that case, all views will share the same underlying storage, and the comparison will be fast). + + This issue could be fixed by using reference equality, but doing so requires defining an external function to expose (a restricted form of) reference equality on values (exposing reference equality in general on values is not sound). A good description of the relevant technique can be found in chapter 9 of “Verasco: a Formally Verified C Static Analyzer” by Jacques-Henri Jourdan. + +### Implementation notes + +- The division into a low-level and a high-level API achieves two desirable outcomes: + + - Programs that modify a small part of a JSON object can be implemented much more efficiently that using a traditional JSON AST ↔ bytes API, since unmodified parts of the decoded JSON object can be reserialized very quickly. + + - Specs and proofs for the low-level API are simple because the low-level API captures all encoding details (including the amount of whitespace used), which makes serialization and deserialization uniquely determined. As a result, serialization is really the (left) inverse of deserializations. + +- Most of the low-level API uses bounded integers (`int32`), so the library cannot deserialize more than 4GB of JSON text. + +- To support in-place updates, the low-level API supports serializing into an existing buffer. Instead of repeatedly checking for overflows, the implementation uses saturating addition and checks for overflow only once, after writing the whole object. This optimization was inspired by Go's error-handling style, described in [Errors are values](https://go.dev/blog/errors-are-values). + +- Dafny does not support floating point numbers (and the JSON spec does not mandate a specific precision), so the library uses a pair `(n, e10)` instead representing the number `n * 10^e10`. + +- Workarounds for known Dafny bugs are indicated by the keyword [`BUG`](https://github.com/dafny-lang/libraries/search?q=BUG) in the sources. + +## TODOs and contribution opportunities + +### Verification + +The high-level API is unverified. Unlike the low-level API, general JSON serialization and deserialization are not inverses of each other, because deserializing an object and re-serializing it may add or remove whitespace or change the way strings or numbers are represented (e.g. the string `"a"` may be serialized as `"\u0061"` and the number 1 as 1.0e0, 10e-1, etc.). As a result, an executable serializer is not a sufficient specification of the format. Techniques to write specifications and verify serializers and deserializers in such cases are described in e.g. [Narcissus: correct-by-construction derivation of decoders and encoders from binary formats](https://dl.acm.org/doi/abs/10.1145/3341686). + +Note that it would be possible (and a great start) to show that the deserializer supports everything that the serializer may produce. This would not require a generalized specification (one that allows variation in serialization choices) of the format. + +Beyond this large task, some small lemmas that should hold but were not needed by this codebase are commented out and marked as `// TODO`. + +### Performance problems in the high-level API + +The high-level API does string encoding and escaping as two separate passes that both construct sequences. Adding `by method` blocks and fusing these two passes would provide significant gains. + +### Performance opportunities in the low-level API + +The low-level API is written in the pure subset of Dafny. As a result, it creates lots of short-lived records (especially cursors, defined in [`Cursors.dfy`](Utils/Cursors.dfy)). A linear version of Dafny would be able to mutate these structures in place. An traditional reuse analysis (see e.g. Schulte & Grieskamp, “Generating Efficient Portable Code for a Strict Applicative Language”, or any of the OPAL papers) would achieve the same result. Large gains could already be achieved simply by improving the compilation of records in the C# backend (e.g. using structs for small records and removing unnecessary interfaces). + +The optimization to merge contiguous string could be made to use physical equality. This would provide little gain in the typical case, but would eliminate the worst-case described in [“Caveats”](#caveats) above. + +The low-level serializer does not re-serialize unchanged parts of an AST, but it still re-writes the whole object. This could be eliminated when changes to the object do not change its overall length. From 2de6306eccfa3bb8c2f50bce0caae9ec3ad2ca18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 02:18:33 -0700 Subject: [PATCH 18/84] json: Add a useful postcondition to Views.Merge --- src/JSON/Utils/Views.dfy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/JSON/Utils/Views.dfy b/src/JSON/Utils/Views.dfy index 6e7ab42f..a0081d8e 100644 --- a/src/JSON/Utils/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -112,8 +112,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Core { && lv.s == rv.s } - function Merge(lv: View, rv: View) : View + function Merge(lv: View, rv: View) : (v: View) requires Adjacent(lv, rv) + ensures v.Bytes() == lv.Bytes() + rv.Bytes() { lv.(end := rv.end) } From 92a8f8dbd9f26eb0c731b0f243dcab983f66d378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 11:04:31 -0700 Subject: [PATCH 19/84] json: Speed up a proof --- src/JSON/Utils/Unicode.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index 518e1df1..bb0b7c3d 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -46,7 +46,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Unicode { function Utf16Encode2(cp: uint32): seq requires cp < 0x100000 { - var bv := cp as bv32; + var bv := cp as bv20; [(0xD800 | (bv >> 10)) as char, (0xDC00 | (bv & 0x03FF)) as char] } From 390e288110ef07bebeca6c3dc55ab06a47716998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 11:04:25 -0700 Subject: [PATCH 20/84] json: Move all math code to NonlinearArithmetic/ and use /noNLArith --- src/JSON/Deserializer.dfy | 8 +- src/JSON/README.md | 2 - src/JSON/Spec.dfy | 8 +- src/JSON/Utils/Math.dfy | 108 -------------------------- src/JSON/Utils/Str.dfy | 60 ++++++++------ src/NonlinearArithmetic/DivMod.dfy | 16 ++++ src/NonlinearArithmetic/Logarithm.dfy | 103 ++++++++++++++++++++++++ src/NonlinearArithmetic/Power.dfy | 1 - 8 files changed, 166 insertions(+), 140 deletions(-) delete mode 100644 src/JSON/Utils/Math.dfy create mode 100644 src/NonlinearArithmetic/Logarithm.dfy diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 8a1aaf13..92c2ee95 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -6,7 +6,6 @@ include "../Collections/Sequences/Seq.dfy" include "../BoundedInts.dfy" -include "Utils/Math.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" @@ -18,10 +17,11 @@ include "Spec.dfy" module {:options "-functionSyntax:4"} JSON.Deserializer { import Seq - import Utils.Math import opened Wrappers import opened BoundedInts + import opened Logarithm + import opened Power import opened Utils.Str import Utils.Unicode @@ -56,7 +56,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { else var tl :- Unescape(str, start + 6); var hd := Str.ToNat(code, 16); - assert hd < 0x10000 by { reveal Math.IntPow(); } + assert hd < 0x10000 by { reveal Pow(); } Success([hd as char] + tl) else var unescaped: uint16 := match c @@ -118,7 +118,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { case NonEmpty(JFrac(_, num)) => var pow10 := num.Length() as int; var frac :- ToInt(minus, num); - Success(AST.Decimal(n * Math.IntPow(10, pow10) + frac, e10 - pow10)) + Success(AST.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) } function KV(js: Grammar.jkv): DeserializationResult<(string, AST.JSON)> { diff --git a/src/JSON/README.md b/src/JSON/README.md index b5937d72..646d0c5c 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -55,8 +55,6 @@ Most of the contents of the `Utils/` directory are not specific to JSON. They a - [`Str.dfy`](Utils/Str.dfy) implements functions to convert between (big-endian) strings and numbers. This module shares similarities with [LittleEndianNat.dfy](../Collections/Sequences/LittleEndianNat.dfy) (except for the digit order), although `LittleEndianNat.dfy` makes the base a module parameter instead of a function argument. -- [`Math.dfy`](Utils/Math.dfy) implements `IntPow` and its left inverse `IntLog`. These functions are used to characterize the length of strings produced from numbers and the value of numbers produced from strings. They could be migrated to [`NonlinearArithmetic/Power.dfy`](../NonlinearArithmetic/Power.dfy), but the `Nonlinear` part of the library uses custom Dafny flags for arithmetic and Dafny does not (yet) support specifying these flags per-module. - - [`Parsers.dfy`](Utils/Parsers.dfy) has definitions of well-formedness for parsers (stating that they must consume part of their input). This file would have to be significantly expanded to create a composable library of parsing combinators to be useful as part of the main library. ## Caveats diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 9bce4ebe..46200703 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -6,6 +6,7 @@ /// ``JSON.Serializer.dfy``. include "../BoundedInts.dfy" +include "../NonlinearArithmetic/Logarithm.dfy" include "AST.dfy" include "Utils/Unicode.dfy" @@ -17,6 +18,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { import opened Utils.Str import opened AST import opened Utils.Unicode + import opened Logarithm type bytes = seq @@ -24,10 +26,10 @@ module {:options "-functionSyntax:4"} JSON.Spec { var s := Str.OfNat(c as nat, 16); assert |s| <= 4 by { assert c as nat <= 0xFFFF; - assert Math.IntLog(16, c as nat) <= Math.IntLog(16, 0xFFFF) by { - Math.IntLog_Increasing(16, c as nat, 0xFFFF); + assert Log(16, c as nat) <= Log(16, 0xFFFF) by { + LemmaLogIsOrdered(16, c as nat, 0xFFFF); } - assert Math.IntLog(16, 0xFFFF) == 3 by { reveal Math.IntLog(); } + assert Log(16, 0xFFFF) == 3 by { reveal Log(); } } s + seq(4 - |s|, _ => ' ') } diff --git a/src/JSON/Utils/Math.dfy b/src/JSON/Utils/Math.dfy deleted file mode 100644 index d953393e..00000000 --- a/src/JSON/Utils/Math.dfy +++ /dev/null @@ -1,108 +0,0 @@ -// include "../.s./NonlinearArithmetic/DivMod.dfy" // Only needed for unused proofs - -module JSON.Utils.Math { - function method {:opaque} IntPow(base: int, n: nat): int { - if n == 0 then 1 - else base * IntPow(base, n - 1) - } - - lemma {:induction n} IntPow_Nat(base: nat, n: nat) - ensures IntPow(base, n) >= 0 - { - reveal IntPow(); - } - - lemma {:induction n} IntPow_NonZero(base: int, n: nat) - requires base != 0 - ensures IntPow(base, n) != 0 - { - reveal IntPow(); - } - - function method {:opaque} IntLog(base: nat, pow: nat): nat - requires base > 1 - decreases pow - { - if pow < base then 0 - else - assert pow < pow * base; assert pow / base < pow; - 1 + IntLog(base, pow / base) - } - - lemma {:induction false} Divide_Increasing(base: nat, pow: nat, pow': nat) - requires base > 1 - requires pow <= pow' - ensures pow / base <= pow' / base - { - var q, q', r, r' := pow / base, pow' / base, pow % base, pow' % base; - assert pow == q * base + r; - assert pow' == q' * base + r'; - assert 0 <= r < base && 0 <= r' < base; - var dp, dq, dr := pow' - pow, q' - q, r' - r; - assert dp == dq * base + dr; - if dq < 0 { - assert dq * base < -base; - assert dr <= base; - assert dp == dq * base + dr < 0; - assert false; - } - } - - lemma {:induction false} IntLog_Increasing(base: nat, pow: nat, pow': nat) - requires base > 1 - requires pow <= pow' - ensures IntLog(base, pow) <= IntLog(base, pow') - decreases pow - { - reveal IntLog(); - if pow' < base { - assert IntLog(base, pow) == 0 == IntLog(base, pow'); - } else if pow < base { - assert IntLog(base, pow) == 0; - } else { - assert pow < pow * base; assert pow / base < pow; - assert pow' < pow' * base; assert pow' / base < pow'; - assert pow / base <= pow' / base by { Divide_Increasing(base, pow, pow'); } - IntLog_Increasing(base, pow / base, pow' / base); - } - } - - /* - import DivMod - - lemma {:induction false} Multiply_Divide(p: int, q: nat) - requires q != 0 - ensures q * p / q == p - { - DivMod.LemmaDivMultiplesVanish(p, q); - } - - lemma {:induction false} IntLog_IntPow(base: nat, n: nat) - requires base > 1 - ensures (IntPow_Nat(base, n); IntLog(base, IntPow(base, n)) == n) - { - if n == 0 { - reveal IntPow(); - reveal IntLog(); - } else { - IntPow_Nat(base, n); - calc { - IntLog(base, IntPow(base, n)); - { reveal IntPow(); } - IntLog(base, base * IntPow(base, n - 1)); - { reveal IntLog(); IntPow_Nat(base, n - 1); IntPow_NonZero(base, n - 1); } - 1 + IntLog(base, base * IntPow(base, n - 1) / base); - { Multiply_Divide(IntPow(base, n - 1), base); } - 1 + IntLog(base, IntPow(base, n - 1)); - { IntLog_IntPow(base, n - 1); } - 1 + (n - 1); - } - } - } - - lemma {:induction false} IntPow_IntLog(base: nat, pow: nat) - requires base > 1 - requires pow > 0 - ensures IntPow(base, IntLog(base, pow)) <= pow < IntPow(base, IntLog(base, pow) + 1) - */ -} diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index f93c3492..4cf17104 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,14 +1,23 @@ +// RUN: %dafny /compile:0 /noNLarith "%s" + include "../../BoundedInts.dfy" include "../../Wrappers.dfy" -include "Math.dfy" +include "../../NonlinearArithmetic/Mul.dfy" +include "../../NonlinearArithmetic/DivMod.dfy" +include "../../NonlinearArithmetic/Logarithm.dfy" +include "../../NonlinearArithmetic/Power.dfy" module {:options "-functionSyntax:4"} JSON.Utils.Str { import opened Wrappers - import Math + import opened Power + import opened Logarithm abstract module ParametricConversion { import opened Wrappers - import Math + import opened Mul + import opened DivMod + import opened Power + import opened Logarithm type Char(==) type String = seq @@ -19,24 +28,24 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { requires base > 1 decreases n ensures n == 0 ==> |digits| == 0 - ensures n > 0 ==> |digits| == Math.IntLog(base, n) + 1 + ensures n > 0 ==> |digits| == Log(base, n) + 1 ensures forall d | d in digits :: 0 <= d < base { if n == 0 then - assert Math.IntPow(base, 0) == 1 by { reveal Math.IntPow(); } + assert Pow(base, 0) == 1 by { reveal Pow(); } [] else - assert n < base * n; - assert n / base < n; + LemmaDivPosIsPosAuto(); LemmaDivDecreasesAuto(); var digits' := Digits(n / base, base); var digits := digits' + [n % base]; - assert |digits| == Math.IntLog(base, n) + 1 by { + assert |digits| == Log(base, n) + 1 by { + assert |digits| == |digits'| + 1; if n < base { - assert |digits| == 1; - assert Math.IntLog(base, n) == 0 by { reveal Math.IntLog(); } + LemmaLog0(base, n); + assert n / base == 0 by { LemmaBasicDiv(base); } } else { - assert |digits'| == |digits| - 1; - assert Math.IntLog(base, n) == Math.IntLog(base, n / base) + 1 by { reveal Math.IntLog(); } + LemmaLogS(base, n); + assert n / base > 0 by { LemmaDivNonZeroAuto(); } } } digits @@ -56,11 +65,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { function OfNat_any(n: nat, chars: seq) : (str: String) requires |chars| > 1 - ensures |str| == Math.IntLog(|chars|, n) + 1 + ensures |str| == Log(|chars|, n) + 1 ensures forall c | c in str :: c in chars { var base := |chars|; - if n == 0 then reveal Math.IntLog(); [chars[0]] + if n == 0 then reveal Log(); [chars[0]] else OfDigits(Digits(n, base), chars) } @@ -89,24 +98,31 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { requires forall c | c in str :: c in digits { if str == [] then 0 - else ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]] + else + LemmaMulNonnegativeAuto(); + ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]] } lemma {:induction false} ToNat_bound(str: String, base: nat, digits: map) requires base > 0 requires forall c | c in str :: c in digits requires forall c | c in str :: digits[c] < base - ensures ToNat_any(str, base, digits) < Math.IntPow(base, |str|) + ensures ToNat_any(str, base, digits) < Pow(base, |str|) { - reveal Math.IntPow(); if str == [] { + reveal Pow(); } else { calc <= { ToNat_any(str, base, digits); ToNat_any(str[..|str| - 1], base, digits) * base + digits[str[|str| - 1]]; - { ToNat_bound(str[..|str| - 1], base, digits); } - (Math.IntPow(base, |str| - 1) - 1) * base + base - 1; - Math.IntPow(base, |str| - 1) * base - 1; + ToNat_any(str[..|str| - 1], base, digits) * base + (base - 1); + { ToNat_bound(str[..|str| - 1], base, digits); + LemmaMulInequalityAuto(); } + (Pow(base, |str| - 1) - 1) * base + base - 1; + { LemmaMulIsDistributiveAuto(); } + Pow(base, |str| - 1) * base - 1; + { reveal Pow(); LemmaMulIsCommutativeAuto(); } + Pow(base, |str|) - 1; } } } @@ -178,7 +194,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { function OfNat(n: nat, base: int := 10) : (str: string) requires 2 <= base <= 16 - ensures |str| == Math.IntLog(base, n) + 1 + ensures |str| == Log(base, n) + 1 ensures forall c | c in str :: c in HEX_DIGITS[..base] { CharStrConversion.OfNat_any(n, HEX_DIGITS[..base]) @@ -194,7 +210,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { function ToNat(str: string, base: int := 10) : (n: nat) requires 2 <= base <= 16 requires forall c | c in str :: c in HEX_TABLE && HEX_TABLE[c] as int < base - ensures n < Math.IntPow(base, |str|) + ensures n < Pow(base, |str|) { CharStrConversion.ToNat_bound(str, base, HEX_TABLE); CharStrConversion.ToNat_any(str, base, HEX_TABLE) diff --git a/src/NonlinearArithmetic/DivMod.dfy b/src/NonlinearArithmetic/DivMod.dfy index 5567ef8c..a3642151 100644 --- a/src/NonlinearArithmetic/DivMod.dfy +++ b/src/NonlinearArithmetic/DivMod.dfy @@ -111,6 +111,22 @@ module DivMod { } } + lemma LemmaDivNonZero(x: int, d: int) + requires x >= d > 0 + ensures x / d > 0 + { + LemmaDivPosIsPosAuto(); + if x / d == 0 { + LemmaSmallDivConverseAuto(); + } + } + + lemma LemmaDivNonZeroAuto() + ensures forall x, d {:trigger x / d } | x >= d > 0 :: x / d > 0 + { + forall x, d | x >= d > 0 { LemmaDivNonZero(x, d); } + } + /* given two fractions with the same numerator, the order of numbers is determined by the denominators. However, if the numerator is 0, the fractions are equal regardless of the denominators' values */ diff --git a/src/NonlinearArithmetic/Logarithm.dfy b/src/NonlinearArithmetic/Logarithm.dfy new file mode 100644 index 00000000..fc438f2e --- /dev/null +++ b/src/NonlinearArithmetic/Logarithm.dfy @@ -0,0 +1,103 @@ +// RUN: %dafny /compile:0 /noNLarith "%s" + +include "Mul.dfy" +include "DivMod.dfy" +include "Power.dfy" + +module Logarithm { + import opened Mul + import opened DivMod + import opened Power + + function method {:opaque} Log(base: nat, pow: nat): nat + requires base > 1 + decreases pow + { + if pow < base then 0 + else + LemmaDivPosIsPosAuto(); LemmaDivDecreasesAuto(); + 1 + Log(base, pow / base) + } + + lemma {:induction false} LemmaLog0(base: nat, pow: nat) + requires base > 1 + requires pow < base + ensures Log(base, pow) == 0 + { + reveal Log(); + } + + lemma {:induction false} LemmaLogS(base: nat, pow: nat) + requires base > 1 + requires pow >= base + ensures pow / base >= 0 + ensures Log(base, pow) == 1 + Log(base, pow / base) + { + LemmaDivPosIsPosAuto(); + reveal Log(); + } + + lemma {:induction false} LemmaLogSAuto() + ensures forall base: nat, pow: nat + {:trigger Log(base, pow / base)} + | && base > 1 + && pow >= base + :: && pow / base >= 0 + && Log(base, pow) == 1 + Log(base, pow / base) + { + forall base: nat, pow: nat | && base > 1 && pow >= base + ensures && pow / base >= 0 + && Log(base, pow) == 1 + Log(base, pow / base) + { + LemmaLogS(base, pow); + } + } + + lemma {:induction false} LemmaLogIsOrdered(base: nat, pow: nat, pow': nat) + requires base > 1 + requires pow <= pow' + ensures Log(base, pow) <= Log(base, pow') + decreases pow + { + reveal Log(); + if pow' < base { + assert Log(base, pow) == 0 == Log(base, pow'); + } else if pow < base { + assert Log(base, pow) == 0; + } else { + LemmaDivPosIsPosAuto(); LemmaDivDecreasesAuto(); LemmaDivIsOrderedAuto(); + LemmaLogIsOrdered(base, pow / base, pow' / base); + } + } + + lemma {:induction false} LemmaLogPow(base: nat, n: nat) + requires base > 1 + ensures (LemmaPowPositive(base, n); Log(base, Pow(base, n)) == n) + { + if n == 0 { + reveal Pow(); + reveal Log(); + } else { + LemmaPowPositive(base, n); + calc { + Log(base, Pow(base, n)); + { reveal Pow(); } + Log(base, base * Pow(base, n - 1)); + { LemmaPowPositive(base, n - 1); + LemmaMulIncreases(Pow(base, n - 1), base); + LemmaLogS(base, base * Pow(base, n - 1)); } + 1 + Log(base, (base * Pow(base, n - 1)) / base); + { LemmaDivMultiplesVanish(Pow(base, n - 1), base); } + 1 + Log(base, Pow(base, n - 1)); + { LemmaLogPow(base, n - 1); } + 1 + (n - 1); + } + } + } + + // TODO + // lemma {:induction false} Pow_Log(base: nat, pow: nat) + // requires base > 1 + // requires pow > 0 + // ensures Pow(base, Log(base, pow)) <= pow < Pow(base, Log(base, pow) + 1) +} diff --git a/src/NonlinearArithmetic/Power.dfy b/src/NonlinearArithmetic/Power.dfy index bda7a18a..5d86e94c 100644 --- a/src/NonlinearArithmetic/Power.dfy +++ b/src/NonlinearArithmetic/Power.dfy @@ -18,7 +18,6 @@ include "Mul.dfy" include "Internals/MulInternals.dfy" module Power { - import Math import opened DivMod import opened GeneralInternals import opened Mul From dcd06cc4075adf9a680ca09d5a29271fc0a9b5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 2 Sep 2022 15:11:17 -0700 Subject: [PATCH 21/84] json: Add a tutorial --- src/JSON/AST.dfy | 5 + src/JSON/Tutorial.dfy | 269 +++++++++++++++++++++++++++++++++++ src/JSON/Tutorial.dfy.expect | 4 + src/JSON/Utils/Unicode.dfy | 8 ++ 4 files changed, 286 insertions(+) create mode 100644 src/JSON/Tutorial.dfy create mode 100644 src/JSON/Tutorial.dfy.expect diff --git a/src/JSON/AST.dfy b/src/JSON/AST.dfy index 0664c6a7..4d3f6b01 100644 --- a/src/JSON/AST.dfy +++ b/src/JSON/AST.dfy @@ -1,6 +1,11 @@ module {:options "-functionSyntax:4"} JSON.AST { datatype Decimal = Decimal(n: int, e10: int) // (n) * 10^(e10) + + function Int(n: int): Decimal { + Decimal(n, 0) + } + datatype JSON = | Null | Bool(b: bool) diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy new file mode 100644 index 00000000..ec508b88 --- /dev/null +++ b/src/JSON/Tutorial.dfy @@ -0,0 +1,269 @@ +// RUN: %dafny -compile:3 -runAllTests:1 "%s" +/// # Using the JSON library + +include "API.dfy" +include "ZeroCopy/API.dfy" + +/// This library offers two APIs: a high-level one and a low-level one. +/// +/// ## High-level API + +module {:options "-functionSyntax:4"} JSON.Examples.HighLevel { + import API + import Utils.Unicode + import opened AST + import opened Wrappers + +/// The high-level API works with fairly simple ASTs that contain native Dafny +/// strings: + + method {:test} Test() { + +/// Use `API.Deserialize` to deserialize a byte string. +/// +/// For example, here is how to decode the JSON test `"[true]"`. (We need to +/// convert from Dafny's native strings to byte strings because Dafny does not +/// have syntax for byte strings; in a real application, we would be reading and +/// writing raw bytes directly from disk or from the network instead). + + var SIMPLE_JS := Unicode.Transcode16To8("[true]"); + var SIMPLE_AST := Array([Bool(true)]); + expect API.Deserialize(SIMPLE_JS) == Success(SIMPLE_AST); + +/// Here is a larger object, written using a verbatim string (with `@"`). In +/// verbatim strings `""` represents a single double-quote character): + + var CITIES_JS := Unicode.Transcode16To8(@"{ + ""Cities"": [ + { + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4584.2 + }, { + ""Name"": ""Rome"", + ""Founded"": -753, + ""Population"": 2.873e6, + ""Area (km2)"": 1285 + }, { + ""Name"": ""Paris"", + ""Founded"": null, + ""Population"": 2.161e6, + ""Area (km2)"": 2383.5 + } + ] + }"); + var CITIES_AST := Object([ + ("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1))) + ]), + Object([ + ("Name", String("Rome")), + ("Founded", Number(Int(-753))), + ("Population", Number(Decimal(2873, 3))), + ("Area (km2)", Number(Int(1285))) + ]), + Object([ + ("Name", String("Paris")), + ("Founded", Null), + ("Population", Number(Decimal(2161, 3))), + ("Area (km2)", Number(Decimal(23835, -1))) + ]) + ])) + ]); + expect API.Deserialize(CITIES_JS) == Success(CITIES_AST); + +/// Serialization works similarly, with `API.Serialize`. For this first example +/// the generated string matches what we started with exactly: + + expect API.Serialize(SIMPLE_AST) == Success(SIMPLE_JS); + +/// For more complex object, the generated layout may not be exactly the same; note in particular how the representation of numbers and the whitespace have changed. + + expect API.Serialize(CITIES_AST) == Success(Unicode.Transcode16To8( + @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1},{""Name"":""Rome"",""Founded"":-753,""Population"":2873e3,""Area (km2)"":1285},{""Name"":""Paris"",""Founded"":null,""Population"":2161e3,""Area (km2)"":23835e-1}]}" + )); + +/// Additional methods are defined in `API.dfy` to serialize an object into an +/// existing buffer or into an array. Below is the smaller example from the +/// README, as a sanity check: + + var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4584.2}]}"); + + var CITY_AST := Object([("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1)))])]))]); + + expect API.Deserialize(CITY_JS) == Success(CITY_AST); + + expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( + @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" + )); + } +} + +/// ## Low-level API +/// +/// If you care about low-level performance, or about preserving existing +/// formatting as much as possible, you may prefer to use the lower-level API: + +module {:options "-functionSyntax:4"} JSON.Examples.LowLevel { + import ZeroCopy.API + import Utils.Unicode + import opened Grammar + import opened Wrappers + +/// The low-level API works with ASTs that record all details of formatting and +/// encoding: each node contains pointers to parts of a string, such that +/// concatenating the fields of all nodes reconstructs the serialized value. + + method {:test} Test() { + +/// The low-level API exposes the same functions and methods as the high-level +/// one, but the type that they consume and produce is `Grammar.JSON` (defined +/// in `Grammar.dfy` as a `Grammar.Value` surrounded by optional whitespace) +/// instead of `AST.JSON` (defined in `AST.dfy`). Since `Grammar.JSON` contains +/// all formatting information, re-serializing an object produces the original +/// value: + + var CITIES := Unicode.Transcode16To8(@"{ + ""Cities"": [ + { + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4600 + }, { + ""Name"": ""Rome"", + ""Founded"": -753, + ""Population"": 2.873e6, + ""Area (km2)"": 1285 + }, { + ""Name"": ""Paris"", + ""Founded"": null, + ""Population"": 2.161e6, + ""Area (km2)"": 2383.5 + } + ] + }"); + + var deserialized :- expect API.Deserialize(CITIES); + expect API.Serialize(deserialized) == Success(CITIES); + +/// Since the formatting is preserved, it is also possible to write +/// minimally-invasive transformations over an AST. For example, let's replace +/// `null` in the object above with `"Unknown"`. +/// +/// First, we construct a JSON value for the string `"Unknown"`; this could be +/// done by hand using `View.OfBytes()`, but using `API.Deserialize` is even +/// simpler: + + var UNKNOWN :- expect API.Deserialize(Unicode.Transcode16To8(@"""Unknown""")); + +/// `UNKNOWN` is of type `Grammar.JSON`, which contains optional whitespace and +/// a `Grammar.Value` under the name `UNKNOWN.t`, which we can use in the +/// replacement: + + var without_null := deserialized.(t := ReplaceNull(deserialized.t, UNKNOWN.t)); + +/// Then, if we reserialize, we see that all formatting (and, in fact, all of +/// the serialization work) has been reused: + + expect API.Serialize(without_null) == Success(Unicode.Transcode16To8(@"{ + ""Cities"": [ + { + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4600 + }, { + ""Name"": ""Rome"", + ""Founded"": -753, + ""Population"": 2.873e6, + ""Area (km2)"": 1285 + }, { + ""Name"": ""Paris"", + ""Founded"": ""Unknown"", + ""Population"": 2.161e6, + ""Area (km2)"": 2383.5 + } + ] + }")); + } + +/// All that remains is to write the recursive traversal: + + import Seq + + function ReplaceNull(js: Value, replacement: Value): Value { + match js + +/// Non-recursive cases are untouched: + + case Bool(_) => js + case String(_) => js + case Number(_) => js + +/// `Null` is replaced with the new `replacement` value: + + case Null(_) => replacement + +/// … and objects and arrays are traversed recursively (only the data part of is +/// traversed: other fields record information about the formatting of braces, +/// square brackets, and whitespace, and can thus be reused without +/// modifications): + + case Object(obj) => + Object(obj.(data := MapSuffixedSequence(obj.data, (s: Suffixed) requires s in obj.data => + s.t.(v := ReplaceNull(s.t.v, replacement))))) + case Array(arr) => + Array(arr.(data := MapSuffixedSequence(arr.data, (s: Suffixed) requires s in arr.data => + ReplaceNull(s.t, replacement)))) + } + +/// Note that well-formedness criteria on the low-level AST are enforced using +/// subset types, which is why we need a bit more work to iterate over the +/// sequences of key-value paris and of values in objects and arrays. +/// Specifically, we need to prove that mapping over these sequences doesn't +/// introduce dangling punctuation (`NoTrailingSuffix`). We package this +/// reasoning into a `MapSuffixedSequence` function: + + function MapSuffixedSequence(sq: SuffixedSequence, fn: Suffixed --> D) + : SuffixedSequence + requires forall suffixed | suffixed in sq :: fn.requires(suffixed) + { + // BUG(https://github.com/dafny-lang/dafny/issues/2184) + // BUG(https://github.com/dafny-lang/dafny/issues/2690) + var fn' := (sf: Suffixed) requires (ghost var in_sq := sf => sf in sq; in_sq(sf)) => sf.(t := fn(sf)); + var sq' := Seq.Map(fn', sq); + + assert NoTrailingSuffix(sq') by { + forall idx | 0 <= idx < |sq'| ensures sq'[idx].suffix.Empty? <==> idx == |sq'| - 1 { + calc { + sq'[idx].suffix.Empty?; + fn'(sq[idx]).suffix.Empty?; + sq[idx].suffix.Empty?; + idx == |sq| - 1; + idx == |sq'| - 1; + } + } + } + + sq' + } +} + +/// The examples in this file can be run with `Dafny -compile:4 -runAllTests:1` +/// (the tests produce no output, but their calls to `expect` will be checked +/// dynamically). diff --git a/src/JSON/Tutorial.dfy.expect b/src/JSON/Tutorial.dfy.expect new file mode 100644 index 00000000..048c3f8a --- /dev/null +++ b/src/JSON/Tutorial.dfy.expect @@ -0,0 +1,4 @@ + +Dafny program verifier finished with 3 verified, 0 errors +JSON.Examples.HighLevel.Test: PASSED +JSON.Examples.LowLevel.Test: PASSED diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index bb0b7c3d..d990e10e 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -143,4 +143,12 @@ module {:options "-functionSyntax:4"} JSON.Utils.Unicode { function Transcode8To16(s: seq): string { Utf16Encode(Utf8Decode(s)) } + + function ASCIIToBytes(s: string): seq + // Keep ASCII characters in `s` and discard all other characters + { + seq(|s|, idx requires 0 <= idx < |s| => + if s[idx] as uint16 < 128 then s[idx] as uint8 + else 0 as uint8) + } } From f23ed821fb74ef2810beb20eb93a6e7ddeb69046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 15 Sep 2022 12:58:21 -0400 Subject: [PATCH 22/84] Update src/JSON/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikaël Mayer --- src/JSON/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/README.md b/src/JSON/README.md index 646d0c5c..c0808fee 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -8,7 +8,7 @@ This library provides two APIs: - A high-level API built on top of the zero-copy API that is unverified and less efficient, but is more convenient to use (in particular, it handles encoding and escaping: its JSON AST uses Dafny's `string` type). -Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you nead to read or produce JSON text in other encodings. +Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you need to read or produce JSON text in other encodings. ## Library usage From efc4b0bc671385e876146cf7bb7e093b3bbb33fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 15 Sep 2022 12:58:21 -0400 Subject: [PATCH 23/84] json: Fix two typos in the README --- src/JSON/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/README.md b/src/JSON/README.md index c0808fee..36298079 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -37,7 +37,7 @@ expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( ## What is verified? -The zero-copy serializer is proved sound and complete against a simple functional specification found in [`LowLevel.Spec.dfy`](./LowLevel.Spec.dfy). The low-level deserializer is proven sound, but not complete, against that same specification: if a value is decoded +The zero-copy serializer is proved sound and complete against a simple functional specification found in [`LowLevel.Spec.dfy`](./LowLevel.Spec.dfy). The low-level deserializer is proven sound, but not complete, against that same specification: if a value is deserialized successfully, then re-serializing recovers the original bytestring. ### Useful submodules From d7064d23f95a50bd6357ef444693e829d31f0ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 29 Dec 2022 20:52:41 +0100 Subject: [PATCH 24/84] json: Add a TODO --- src/JSON/Utils/Cursors.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/Utils/Cursors.dfy b/src/JSON/Utils/Cursors.dfy index 2e346331..164bfdc5 100644 --- a/src/JSON/Utils/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -35,7 +35,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { | ExpectingByte(expected: byte, b: opt_byte) | ExpectingAnyByte(expected_sq: seq, b: opt_byte) | OtherError(err: R) - { + { // TODO: Include positions in errors function ToString(pr: R -> string) : string { match this case EOF => "Reached EOF" From d75efc14e28c93d9df1fb7d071423949c0487797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 01:30:41 +0100 Subject: [PATCH 25/84] json: Rename parameters and functions based on review --- src/JSON/API.dfy | 4 ++-- src/JSON/Utils/Vectors.dfy | 4 ++-- src/JSON/Utils/Views.Writers.dfy | 28 ++++++++++++++-------------- src/JSON/Utils/Views.dfy | 16 ++++++++-------- src/JSON/ZeroCopy/API.dfy | 2 +- src/JSON/ZeroCopy/Serializer.dfy | 16 ++++++++-------- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index 976d134e..8ed913c0 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -22,11 +22,11 @@ module {:options "-functionSyntax:4"} JSON.API { bs := ZeroCopy.SerializeAlloc(js); } - method SerializeBlit(js: AST.JSON, bs: array) returns (len: SerializationResult) + method SerializeInto(js: AST.JSON, bs: array) returns (len: SerializationResult) modifies bs { var js :- Serializer.JSON(js); - len := ZeroCopy.SerializeBlit(js, bs); + len := ZeroCopy.SerializeInto(js, bs); } function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) diff --git a/src/JSON/Utils/Vectors.dfy b/src/JSON/Utils/Vectors.dfy index ee42ff26..e10eb137 100644 --- a/src/JSON/Utils/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -76,7 +76,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { items := items[idx := a]; } - method Blit(new_data: array, count: uint32) + method CopyFrom(new_data: array, count: uint32) requires count as int <= new_data.Length requires count <= capacity requires data.Length == capacity as int @@ -104,7 +104,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { { var old_data, old_capacity := data, capacity; data, capacity := new A[new_capacity](_ => a), new_capacity; - Blit(old_data, old_capacity); + CopyFrom(old_data, old_capacity); Repr := {this, data}; } diff --git a/src/JSON/Utils/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy index 081b044b..5f29f674 100644 --- a/src/JSON/Utils/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -43,16 +43,16 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { Chain(this, v') } - method {:tailrecursion} Blit(bs: array, end: uint32) - requires end as int == Length() <= bs.Length - modifies bs - ensures bs[..end] == Bytes() - ensures bs[end..] == old(bs[end..]) + method {:tailrecursion} CopyTo(dest: array, end: uint32) + requires end as int == Length() <= dest.Length + modifies dest + ensures dest[..end] == Bytes() + ensures dest[end..] == old(dest[end..]) { if Chain? { var end := end - v.Length(); - v.Blit(bs, end); - previous.Blit(bs, end); + v.CopyTo(dest, end); + previous.CopyTo(dest, end); } } } @@ -100,15 +100,15 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { fn(this) } - method {:tailrecursion} Blit(bs: array) + method {:tailrecursion} CopyTo(dest: array) requires Valid? requires Unsaturated? - requires Length() <= bs.Length - modifies bs - ensures bs[..length] == Bytes() - ensures bs[length..] == old(bs[length..]) + requires Length() <= dest.Length + modifies dest + ensures dest[..length] == Bytes() + ensures dest[length..] == old(dest[length..]) { - chain.Blit(bs, length); + chain.CopyTo(dest, length); } method ToArray() returns (bs: array) @@ -118,7 +118,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { ensures bs[..] == Bytes() { bs := new byte[length]; - Blit(bs); + CopyTo(bs); } } } diff --git a/src/JSON/Utils/Views.dfy b/src/JSON/Utils/Views.dfy index a0081d8e..ef0f7c4f 100644 --- a/src/JSON/Utils/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -82,19 +82,19 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Core { else At(0) as opt_byte } - method Blit(bs: array, start: uint32 := 0) + method CopyTo(dest: array, start: uint32 := 0) requires Valid? - requires start as int + Length() as int <= bs.Length + requires start as int + Length() as int <= dest.Length requires start as int + Length() as int < TWO_TO_THE_32 - modifies bs - ensures bs[start..start + Length()] == Bytes() - ensures bs[start + Length()..] == old(bs[start + Length()..]) + modifies dest + ensures dest[start..start + Length()] == Bytes() + ensures dest[start + Length()..] == old(dest[start + Length()..]) { for idx := 0 to Length() - invariant bs[start..start + idx] == Bytes()[..idx] - invariant bs[start + Length()..] == old(bs[start + Length()..]) + invariant dest[start..start + idx] == Bytes()[..idx] + invariant dest[start + Length()..] == old(dest[start + Length()..]) { - bs[start + idx] := s[beg + idx]; + dest[start + idx] := s[beg + idx]; } } } diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index 17372a84..0fbdc91d 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -26,7 +26,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { bs := Serializer.Serialize(js); } - method SerializeBlit(js: Grammar.JSON, bs: array) returns (len: SerializationResult) + method SerializeInto(js: Grammar.JSON, bs: array) returns (len: SerializationResult) modifies bs ensures len.Success? ==> len.value as int <= bs.Length ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index c5cd7d8e..d5bfd7e1 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -24,17 +24,17 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { return Success(bs); } - method SerializeTo(js: JSON, bs: array) returns (len: SerializationResult) - modifies bs - ensures len.Success? ==> len.value as int <= bs.Length - ensures len.Success? ==> bs[..len.value] == Spec.JSON(js) - ensures len.Success? ==> bs[len.value..] == old(bs[len.value..]) - ensures len.Failure? ==> unchanged(bs) + method SerializeTo(js: JSON, dest: array) returns (len: SerializationResult) + modifies dest + ensures len.Success? ==> len.value as int <= dest.Length + ensures len.Success? ==> dest[..len.value] == Spec.JSON(js) + ensures len.Success? ==> dest[len.value..] == old(dest[len.value..]) + ensures len.Failure? ==> unchanged(dest) { var writer := Text(js); :- Need(writer.Unsaturated?, OutOfMemory); - :- Need(writer.length as int <= bs.Length, OutOfMemory); - writer.Blit(bs); + :- Need(writer.length as int <= dest.Length, OutOfMemory); + writer.CopyTo(dest); return Success(writer.length); } From ddea84fcf45a1b13b239aabdf7d0065d6ac18910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 01:31:02 +0100 Subject: [PATCH 26/84] json: Update to latest Dafny --- src/JSON/ZeroCopy/Serializer.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index d5bfd7e1..f9688df3 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -173,7 +173,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { // maintaining the termination argument? Maybe the lambda for elements will be enough? ghost function SequenceSpec(v: Value, items: seq, - spec: T -> bytes, impl: (ghost Value, T, Writer) --> Writer, + spec: T -> bytes, impl: (Value, T, Writer) --> Writer, writer: Writer) : (wr: Writer) requires forall item, wr | item in items :: impl.requires(v, item, wr) From 92dcba1a4afdd6a417a5b6a1f619044005696a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 01:56:02 +0100 Subject: [PATCH 27/84] json: Rename LowLevel and HighLevel to Concrete and Abstract --- .../{LowLevel.Spec.dfy => ConcreteSyntax.Spec.dfy} | 2 +- ...roperties.dfy => ConcreteSyntax.SpecProperties.dfy} | 4 ++-- src/JSON/Deserializer.dfy | 1 + src/JSON/README.md | 6 +++--- src/JSON/Tests.dfy | 6 +++--- src/JSON/Tutorial.dfy | 4 ++-- src/JSON/Tutorial.dfy.expect | 4 ++-- src/JSON/ZeroCopy/API.dfy | 4 ++-- src/JSON/ZeroCopy/Deserializer.dfy | 10 +++++----- src/JSON/ZeroCopy/Serializer.dfy | 8 ++++---- 10 files changed, 25 insertions(+), 24 deletions(-) rename src/JSON/{LowLevel.Spec.dfy => ConcreteSyntax.Spec.dfy} (97%) rename src/JSON/{LowLevel.SpecProperties.dfy => ConcreteSyntax.SpecProperties.dfy} (93%) diff --git a/src/JSON/LowLevel.Spec.dfy b/src/JSON/ConcreteSyntax.Spec.dfy similarity index 97% rename from src/JSON/LowLevel.Spec.dfy rename to src/JSON/ConcreteSyntax.Spec.dfy index 85a45f79..77c9393e 100644 --- a/src/JSON/LowLevel.Spec.dfy +++ b/src/JSON/ConcreteSyntax.Spec.dfy @@ -1,6 +1,6 @@ include "Grammar.dfy" -module {:options "-functionSyntax:4"} JSON.LowLevel.Spec { +module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.Spec { import opened BoundedInts import Vs = Utils.Views.Core diff --git a/src/JSON/LowLevel.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy similarity index 93% rename from src/JSON/LowLevel.SpecProperties.dfy rename to src/JSON/ConcreteSyntax.SpecProperties.dfy index 5862e591..3b815687 100644 --- a/src/JSON/LowLevel.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -1,6 +1,6 @@ -include "LowLevel.Spec.dfy" +include "ConcreteSyntax.Spec.dfy" -module {:options "-functionSyntax:4"} JSON.LowLevel.SpecProperties { +module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties { import opened BoundedInts import Vs = Utils.Views.Core diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 92c2ee95..79925641 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -37,6 +37,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { js.At(0) == 't' as byte } + // TODO: Verify this function function Unescape(str: string, start: nat := 0): DeserializationResult decreases |str| - start { // Assumes UTF-16 strings diff --git a/src/JSON/README.md b/src/JSON/README.md index 36298079..2ff40b6a 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -4,9 +4,9 @@ JSON serialization and deserialization in Dafny, as described in [RFC 8259](http This library provides two APIs: -- A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use (in particular, its JSON AST exposes strings as unescaped, undecoded utf-8 byte sequences of type `seq`). +- A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use (in particular, it works on a concrete syntax tree that represents strings as unescaped, undecoded utf-8 byte sequences of type `seq`). -- A high-level API built on top of the zero-copy API that is unverified and less efficient, but is more convenient to use (in particular, it handles encoding and escaping: its JSON AST uses Dafny's `string` type). +- A high-level API built on top of the zero-copy API that is unverified and less efficient, but is more convenient to use (in particular, it produces abstract syntax trees and it abstracts away details of encoding and escaping: its JSON AST uses Dafny's `string` type). Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you need to read or produce JSON text in other encodings. @@ -37,7 +37,7 @@ expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( ## What is verified? -The zero-copy serializer is proved sound and complete against a simple functional specification found in [`LowLevel.Spec.dfy`](./LowLevel.Spec.dfy). The low-level deserializer is proven sound, but not complete, against that same specification: if a value is deserialized successfully, then re-serializing recovers the original bytestring. +The zero-copy serializer is proved sound and complete against a simple functional specification found in [`ConcreteSyntax.Spec.dfy`](./ConcreteSyntax.Spec.dfy). The low-level deserializer is proven sound, but not complete, against that same specification: if a value is deserialized successfully, then re-serializing recovers the original bytestring. ### Useful submodules diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 981da8d7..c9e2f02a 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -49,7 +49,7 @@ module JSON.Tests.ZeroCopyWrapper refines Wrapper { import opened Wrappers import Grammar import ZeroCopy.API - import LowLevel.Spec + import ConcreteSyntax.Spec type JSON = Grammar.JSON @@ -72,7 +72,7 @@ module JSON.Tests.ZeroCopyWrapper refines Wrapper { } } -module JSON.Tests.HighLevelWrapper refines Wrapper { +module JSON.Tests.AbstractSyntaxWrapper refines Wrapper { import opened Wrappers import Grammar import API @@ -128,6 +128,6 @@ module JSON.Tests { method Main() { ZeroCopyWrapper.TestStrings(VECTORS); - HighLevelWrapper.TestStrings(VECTORS); + AbstractSyntaxWrapper.TestStrings(VECTORS); } } diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index ec508b88..937627a0 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -8,7 +8,7 @@ include "ZeroCopy/API.dfy" /// /// ## High-level API -module {:options "-functionSyntax:4"} JSON.Examples.HighLevel { +module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import API import Utils.Unicode import opened AST @@ -118,7 +118,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.HighLevel { /// If you care about low-level performance, or about preserving existing /// formatting as much as possible, you may prefer to use the lower-level API: -module {:options "-functionSyntax:4"} JSON.Examples.LowLevel { +module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { import ZeroCopy.API import Utils.Unicode import opened Grammar diff --git a/src/JSON/Tutorial.dfy.expect b/src/JSON/Tutorial.dfy.expect index 048c3f8a..db4fbc9c 100644 --- a/src/JSON/Tutorial.dfy.expect +++ b/src/JSON/Tutorial.dfy.expect @@ -1,4 +1,4 @@ Dafny program verifier finished with 3 verified, 0 errors -JSON.Examples.HighLevel.Test: PASSED -JSON.Examples.LowLevel.Test: PASSED +JSON.Examples.AbstractSyntax.Test: PASSED +JSON.Examples.ConcreteSyntax.Test: PASSED diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index 0fbdc91d..a6f041b9 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,5 +1,5 @@ include "../Grammar.dfy" -include "../LowLevel.Spec.dfy" +include "../ConcreteSyntax.Spec.dfy" include "Serializer.dfy" include "Deserializer.dfy" @@ -9,7 +9,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.API { import opened Errors import Grammar - import LowLevel.Spec + import ConcreteSyntax.Spec import Serializer import Deserializer diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index e9ed18f1..8d776725 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,7 +1,7 @@ include "../Errors.dfy" include "../Grammar.dfy" -include "../LowLevel.Spec.dfy" -include "../LowLevel.SpecProperties.dfy" +include "../ConcreteSyntax.Spec.dfy" +include "../ConcreteSyntax.SpecProperties.dfy" include "../Utils/Parsers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { @@ -9,7 +9,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Wrappers - import LowLevel.Spec + import ConcreteSyntax.Spec import Vs = Utils.Views.Core import opened Utils.Cursors import opened Utils.Parsers @@ -104,7 +104,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened BoundedInts import opened Params: SequenceParams - import LowLevel.SpecProperties + import ConcreteSyntax.SpecProperties import opened Vs = Utils.Views.Core import opened Grammar import opened Utils.Cursors @@ -331,7 +331,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import Arrays import Constants - import LowLevel.SpecProperties + import ConcreteSyntax.SpecProperties function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) decreases cs.Length(), 1 diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index f9688df3..7fb4c3a6 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,6 +1,6 @@ include "../Errors.dfy" -include "../LowLevel.Spec.dfy" -include "../LowLevel.SpecProperties.dfy" +include "../ConcreteSyntax.Spec.dfy" +include "../ConcreteSyntax.SpecProperties.dfy" include "../Utils/Views.Writers.dfy" module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { @@ -8,8 +8,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import opened Wrappers import opened Errors - import LowLevel.Spec - import LowLevel.SpecProperties + import ConcreteSyntax.Spec + import ConcreteSyntax.SpecProperties import opened Grammar import opened Utils.Views.Writers import opened Vs = Utils.Views.Core // DISCUSS: Module naming convention? From 515c6fbacb05a741ab58a32a19edbde48887d4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 01:56:22 +0100 Subject: [PATCH 28/84] json: More renamings --- src/JSON/ConcreteSyntax.Spec.dfy | 26 +++++++++++++------------- src/JSON/Deserializer.dfy | 4 ++-- src/JSON/Grammar.dfy | 14 +++++++------- src/JSON/Serializer.dfy | 6 +++--- src/JSON/Spec.dfy | 4 ++-- src/JSON/Tutorial.dfy | 10 ++++++---- src/JSON/ZeroCopy/Deserializer.dfy | 16 ++++++++-------- 7 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/JSON/ConcreteSyntax.Spec.dfy b/src/JSON/ConcreteSyntax.Spec.dfy index 77c9393e..b04c8593 100644 --- a/src/JSON/ConcreteSyntax.Spec.dfy +++ b/src/JSON/ConcreteSyntax.Spec.dfy @@ -10,37 +10,37 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.Spec { v.Bytes() } - function Structural(self: Structural, pt: T -> bytes): bytes { - View(self.before) + pt(self.t) + View(self.after) + function Structural(self: Structural, fT: T -> bytes): bytes { + View(self.before) + fT(self.t) + View(self.after) } function StructuralView(self: Structural): bytes { Structural(self, View) } - function Maybe(self: Maybe, pt: T -> bytes): (bs: bytes) + function Maybe(self: Maybe, fT: T -> bytes): (bs: bytes) ensures self.Empty? ==> bs == [] - ensures self.NonEmpty? ==> bs == pt(self.t) + ensures self.NonEmpty? ==> bs == fT(self.t) { - if self.Empty? then [] else pt(self.t) + if self.Empty? then [] else fT(self.t) } - function ConcatBytes(ts: seq, pt: T --> bytes) : bytes - requires forall d | d in ts :: pt.requires(d) + function ConcatBytes(ts: seq, fT: T --> bytes) : bytes + requires forall d | d in ts :: fT.requires(d) { if |ts| == 0 then [] - else pt(ts[0]) + ConcatBytes(ts[1..], pt) + else fT(ts[0]) + ConcatBytes(ts[1..], fT) } - function Bracketed(self: Bracketed, pdatum: Suffixed --> bytes): bytes - requires forall d | d < self :: pdatum.requires(d) + function Bracketed(self: Bracketed, fDatum: Suffixed --> bytes): bytes + requires forall d | d < self :: fDatum.requires(d) { StructuralView(self.l) + - ConcatBytes(self.data, pdatum) + + ConcatBytes(self.data, fDatum) + StructuralView(self.r) } - function KV(self: jkv): bytes { + function KeyValue(self: jKeyValue): bytes { String(self.k) + StructuralView(self.colon) + Value(self.v) } @@ -66,7 +66,7 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.Spec { } function Member(self: jmember) : bytes { - KV(self.t) + CommaSuffix(self.suffix) + KeyValue(self.t) + CommaSuffix(self.suffix) } function Item(self: jitem) : bytes { diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 79925641..79c172b0 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -122,14 +122,14 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(AST.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) } - function KV(js: Grammar.jkv): DeserializationResult<(string, AST.JSON)> { + function KeyValue(js: Grammar.jKeyValue): DeserializationResult<(string, AST.JSON)> { var k :- String(js.k); var v :- Value(js.v); Success((k, v)) } function Object(js: Grammar.jobject): DeserializationResult> { - Seq.MapWithResult(d requires d in js.data => KV(d.t), js.data) + Seq.MapWithResult(d requires d in js.data => KeyValue(d.t), js.data) } function Array(js: Grammar.jarray): DeserializationResult> { diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy index b5c326fb..f1e244d8 100644 --- a/src/JSON/Grammar.dfy +++ b/src/JSON/Grammar.dfy @@ -1,6 +1,6 @@ -/// ======================== -/// Low-level JSON grammar -/// ======================== +/// ========================================== +/// Low-level JSON grammar (Concrete syntax) +/// ========================================== /// /// See ``JSON.AST`` for the high-level interface. @@ -74,13 +74,13 @@ module {:options "-functionSyntax:4"} JSON.Grammar { type jint = v: View | Int?(v) witness View.OfBytes(['0' as byte]) type jstr = v: View | true witness View.OfBytes([]) // TODO: Enforce quoting and escaping datatype jstring = JString(lq: jquote, contents: jstr, rq: jquote) - datatype jkv = KV(k: jstring, colon: Structural, v: Value) + datatype jKeyValue = KeyValue(k: jstring, colon: Structural, v: Value) // TODO enforce no leading space before closing bracket to disambiguate WS { WS WS } WS - type jobject = Bracketed + type jobject = Bracketed type jarray = Bracketed - type jmembers = SuffixedSequence - type jmember = Suffixed + type jmembers = SuffixedSequence + type jmember = Suffixed type jitems = SuffixedSequence type jitem = Suffixed diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index 61b0b0d2..264ddbdb 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -127,10 +127,10 @@ module {:options "-functionSyntax:4"} JSON.Serializer { const COLON: Structural := MkStructural(Grammar.COLON) - function KV(kv: (string, AST.JSON)): Result { + function KeyValue(kv: (string, AST.JSON)): Result { var k :- String(kv.0); var v :- Value(kv.1); - Success(Grammar.KV(k, COLON, v)) + Success(Grammar.KeyValue(k, COLON, v)) } function MkSuffixedSequence(ds: seq, suffix: Structural, start: nat := 0) @@ -146,7 +146,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { MkStructural(Grammar.COMMA) function Object(obj: seq<(string, AST.JSON)>): Result { - var items :- Seq.MapWithResult(v requires v in obj => KV(v), obj); + var items :- Seq.MapWithResult(v requires v in obj => KeyValue(v), obj); Success(Bracketed(MkStructural(LBRACE), MkSuffixedSequence(items, COMMA), MkStructural(RBRACE))) diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 46200703..7a95a69a 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -70,7 +70,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { else ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) } - function KV(kv: (string, JSON)): bytes { + function KeyValue(kv: (string, JSON)): bytes { String(kv.0) + ToBytes(":") + JSON(kv.1) } @@ -82,7 +82,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { function Object(obj: seq<(string, JSON)>): bytes { ToBytes("{") + - Join(ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KV(obj[i]))) + + Join(ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KeyValue(obj[i]))) + ToBytes("}") } diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index 937627a0..95704000 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -4,9 +4,11 @@ include "API.dfy" include "ZeroCopy/API.dfy" -/// This library offers two APIs: a high-level one and a low-level one. +/// This library offers two APIs: a high-level one (giving abstract syntax trees +/// with no concrete syntactic details) and a low-level one (including all +/// information about blanks, separator positions, character escapes, etc.). /// -/// ## High-level API +/// ## High-level API (Abstract syntax) module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import API @@ -113,7 +115,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { } } -/// ## Low-level API +/// ## Low-level API (concrete syntax) /// /// If you care about low-level performance, or about preserving existing /// formatting as much as possible, you may prefer to use the lower-level API: @@ -225,7 +227,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// modifications): case Object(obj) => - Object(obj.(data := MapSuffixedSequence(obj.data, (s: Suffixed) requires s in obj.data => + Object(obj.(data := MapSuffixedSequence(obj.data, (s: Suffixed) requires s in obj.data => s.t.(v := ReplaceNull(s.t.v, replacement))))) case Array(arr) => Array(arr.(data := MapSuffixedSequence(arr.data, (s: Suffixed) requires s in arr.data => diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 8d776725..acd74952 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -588,7 +588,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Wrappers import Strings - type TElement = jkv + type TElement = jKeyValue const OPEN := '{' as byte const CLOSE := '}' as byte @@ -600,27 +600,27 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Success(cs.Split()) } - function {:opaque} KVFromParts(ghost cs: Cursor, k: Split, + function {:opaque} KeyValueFromParts(ghost cs: Cursor, k: Split, colon: Split>, v: Split) - : (sp: Split) + : (sp: Split) requires k.StrictlySplitFrom?(cs, Spec.String) requires colon.StrictlySplitFrom?(k.cs, c => Spec.Structural(c, SpecView)) requires v.StrictlySplitFrom?(colon.cs, Spec.Value) - ensures sp.StrictlySplitFrom?(cs, Spec.KV) + ensures sp.StrictlySplitFrom?(cs, Spec.KeyValue) { - var sp := SP(Grammar.KV(k.t, colon.t, v.t), v.cs); + var sp := SP(Grammar.KeyValue(k.t, colon.t, v.t), v.cs); calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: cs.Bytes(); Spec.String(k.t) + k.cs.Bytes(); Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + colon.cs.Bytes(); Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + Spec.Value(v.t) + v.cs.Bytes(); - Spec.KV(sp.t) + v.cs.Bytes(); + Spec.KeyValue(sp.t) + v.cs.Bytes(); } sp } function ElementSpec(t: TElement) : bytes { - Spec.KV(t) + Spec.KeyValue(t) } function {:opaque} Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) @@ -628,7 +628,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { var k :- Strings.String(cs); var colon :- Core.Structural(k.cs, Parsers.Parser(Colon, SpecView)); var v :- json.fn(colon.cs); - Success(KVFromParts(cs, k, colon, v)) + Success(KeyValueFromParts(cs, k, colon, v)) } } From 95462d29bf74a46ece13fe2361c45bcb7a02406f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 02:57:02 +0100 Subject: [PATCH 29/84] json: Apply @MikaelMayer's feedback --- src/JSON/ConcreteSyntax.SpecProperties.dfy | 6 ++++-- src/JSON/README.md | 4 ++-- src/JSON/Utils/Cursors.dfy | 9 +++++++-- src/JSON/Utils/Lexers.dfy | 6 +++++- src/JSON/Utils/Str.dfy | 13 ++++++++----- src/JSON/Utils/Unicode.dfy | 4 ++++ src/JSON/Utils/Views.Writers.dfy | 5 ----- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 3b815687..47d7e75b 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -1,13 +1,15 @@ include "ConcreteSyntax.Spec.dfy" -module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties { +module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties + // Some useful properties about the functions used in `ConcreteSyntax.Spec`. +{ import opened BoundedInts import Vs = Utils.Views.Core import opened Grammar import Spec - lemma Bracketed_Morphism(bracketed: Bracketed) // DISCUSS + lemma Bracketed_Morphism(bracketed: Bracketed) ensures forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d < bracketed :: pd0.requires(d)) && (forall d | d < bracketed :: pd1.requires(d)) diff --git a/src/JSON/README.md b/src/JSON/README.md index 2ff40b6a..48cded09 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -4,9 +4,9 @@ JSON serialization and deserialization in Dafny, as described in [RFC 8259](http This library provides two APIs: -- A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use (in particular, it works on a concrete syntax tree that represents strings as unescaped, undecoded utf-8 byte sequences of type `seq`). +- A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use. This API operates on concrete syntax trees that capture details of punctuation and blanks and represent strings using unescaped, undecoded utf-8 byte sequences. -- A high-level API built on top of the zero-copy API that is unverified and less efficient, but is more convenient to use (in particular, it produces abstract syntax trees and it abstracts away details of encoding and escaping: its JSON AST uses Dafny's `string` type). +- A high-level API built on top of the previous one. This API is more convenient to use, but it is unverified and less efficient. It produces abstract syntax trees that represent strings using Dafny's built-in `string` type. Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you need to read or produce JSON text in other encodings. diff --git a/src/JSON/Utils/Cursors.dfy b/src/JSON/Utils/Cursors.dfy index 164bfdc5..484f9014 100644 --- a/src/JSON/Utils/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -77,7 +77,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { s[beg..end] } - ghost function StrictlyAdvancedFrom?(other: Cursor): (b: bool) + ghost predicate StrictlyAdvancedFrom?(other: Cursor): (b: bool) requires Valid? ensures b ==> SuffixLength() < other.SuffixLength() @@ -156,6 +156,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { end - beg } + lemma PrefixSuffixLength() + requires Valid? + ensures Length() == PrefixLength() + SuffixLength() + {} + ghost predicate ValidIndex?(idx: uint32) { beg as int + idx as int < end as int } @@ -186,7 +191,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { else SuffixAt(0) as opt_byte } - function LookingAt(c: char): (b: bool) + predicate LookingAt(c: char): (b: bool) requires Valid? requires c as int < 256 ensures b <==> !EOF? && SuffixAt(0) == c as byte diff --git a/src/JSON/Utils/Lexers.dfy b/src/JSON/Utils/Lexers.dfy index b63f4a59..eaf12400 100644 --- a/src/JSON/Utils/Lexers.dfy +++ b/src/JSON/Utils/Lexers.dfy @@ -6,7 +6,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Lexers { import opened Wrappers import opened BoundedInts - datatype LexerResult<+T, +R> = Accept | Reject(err: R) | Partial(st: T) + datatype LexerResult<+T, +R> = + // A Lexer may return three results: + | Accept // The input is valid. + | Reject(err: R) // The input is not valid; `err` says why. + | Partial(st: T) // More input is needed to finish lexing. type Lexer = (T, opt_byte) -> LexerResult } diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 4cf17104..2d9f5f89 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -182,10 +182,8 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { type Char = char } - const HEX_DIGITS := [ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F' - ] + const HEX_DIGITS: seq := "0123456789ABCDEF" + const HEX_TABLE := map[ '0' := 0, '1' := 1, '2' := 2, '3' := 3, '4' := 4, '5' := 5, '6' := 6, '7' := 7, '8' := 8, '9' := 9, 'a' := 0xA, 'b' := 0xB, 'c' := 0xC, 'd' := 0xD, 'e' := 0xE, 'f' := 0xF, @@ -255,6 +253,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { } function Concat(strs: seq) : string { - Join("", strs) + if |strs| == 0 then "" + else strs[0] + Concat(strs[1..]) } + + lemma Concat_Join(strs: seq) + ensures Concat(strs) == Join("", strs) + {} } diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index d990e10e..f051fb07 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -1,5 +1,9 @@ include "../../BoundedInts.dfy" +// TODO: This module was written before Dafny got a Unicode library. It would +// be better to combine the two, especially given that the Unicode library has +// proofs! + module {:options "-functionSyntax:4"} JSON.Utils.Unicode { import opened BoundedInts diff --git a/src/JSON/Utils/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy index 5f29f674..7e6a4a3f 100644 --- a/src/JSON/Utils/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -8,11 +8,6 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { import opened Core - // export - // reveals Error, Writer - // provides Core, Wrappers - // provides Writer_, Writer_.Append, Writer_.Empty, Writer_.Valid? - datatype Chain = | Empty | Chain(previous: Chain, v: View) From 1670c1d15a5f115583dc1ec5d552ffde673a9797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Fri, 30 Dec 2022 12:25:37 +0100 Subject: [PATCH 30/84] json: Add lit headers to run verification and tests in CI --- src/JSON/API.dfy | 2 ++ src/JSON/AST.dfy | 2 ++ src/JSON/ConcreteSyntax.Spec.dfy | 2 ++ src/JSON/ConcreteSyntax.SpecProperties.dfy | 2 ++ src/JSON/Deserializer.dfy | 2 ++ src/JSON/Errors.dfy | 2 ++ src/JSON/Grammar.dfy | 2 ++ src/JSON/Serializer.dfy | 2 ++ src/JSON/Spec.dfy | 2 ++ src/JSON/Tests.dfy | 2 ++ src/JSON/Utils/Cursors.dfy | 2 ++ src/JSON/Utils/Lexers.dfy | 2 ++ src/JSON/Utils/Parsers.dfy | 2 ++ src/JSON/Utils/Str.dfy | 2 +- src/JSON/Utils/Unicode.dfy | 2 ++ src/JSON/Utils/Vectors.dfy | 2 ++ src/JSON/Utils/Views.Writers.dfy | 2 ++ src/JSON/Utils/Views.dfy | 2 ++ src/JSON/ZeroCopy/API.dfy | 2 ++ src/JSON/ZeroCopy/Deserializer.dfy | 2 ++ src/JSON/ZeroCopy/Serializer.dfy | 2 ++ 21 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index 8ed913c0..4ec7b4ab 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "Serializer.dfy" include "Deserializer.dfy" include "ZeroCopy/API.dfy" diff --git a/src/JSON/AST.dfy b/src/JSON/AST.dfy index 4d3f6b01..37cf20e4 100644 --- a/src/JSON/AST.dfy +++ b/src/JSON/AST.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + module {:options "-functionSyntax:4"} JSON.AST { datatype Decimal = Decimal(n: int, e10: int) // (n) * 10^(e10) diff --git a/src/JSON/ConcreteSyntax.Spec.dfy b/src/JSON/ConcreteSyntax.Spec.dfy index b04c8593..75784d70 100644 --- a/src/JSON/ConcreteSyntax.Spec.dfy +++ b/src/JSON/ConcreteSyntax.Spec.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "Grammar.dfy" module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.Spec { diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 47d7e75b..d295ecc8 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "ConcreteSyntax.Spec.dfy" module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 79c172b0..1a146582 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + /// =============================================== /// Deserialization from JSON.Grammar to JSON.AST /// =============================================== diff --git a/src/JSON/Errors.dfy b/src/JSON/Errors.dfy index 62ee1a0c..04f3e239 100644 --- a/src/JSON/Errors.dfy +++ b/src/JSON/Errors.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../Wrappers.dfy" include "../BoundedInts.dfy" include "Utils/Str.dfy" diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy index f1e244d8..c45ed0b0 100644 --- a/src/JSON/Grammar.dfy +++ b/src/JSON/Grammar.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + /// ========================================== /// Low-level JSON grammar (Concrete syntax) /// ========================================== diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index 264ddbdb..ec680097 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + /// ============================================= /// Serialization from JSON.AST to JSON.Grammar /// ============================================= diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 7a95a69a..58a744cc 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + /// ============================================= /// Serialization from AST.JSON to bytes (Spec) /// ============================================= diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index c9e2f02a..ba75945b 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:3 "%s" + include "Utils/Unicode.dfy" include "Errors.dfy" include "API.dfy" diff --git a/src/JSON/Utils/Cursors.dfy b/src/JSON/Utils/Cursors.dfy index 484f9014..eeddab20 100644 --- a/src/JSON/Utils/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" include "../../Wrappers.dfy" include "Views.dfy" diff --git a/src/JSON/Utils/Lexers.dfy b/src/JSON/Utils/Lexers.dfy index eaf12400..d2f3f65d 100644 --- a/src/JSON/Utils/Lexers.dfy +++ b/src/JSON/Utils/Lexers.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../Wrappers.dfy" include "../../BoundedInts.dfy" diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 102fe595..03f59f01 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" include "../../Wrappers.dfy" include "Cursors.dfy" diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 2d9f5f89..4407f2a4 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:0 /noNLarith "%s" +// RUN: %dafny -compile:0 -noNLarith "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index f051fb07..a375d66d 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" // TODO: This module was written before Dafny got a Unicode library. It would diff --git a/src/JSON/Utils/Vectors.dfy b/src/JSON/Utils/Vectors.dfy index e10eb137..62d3b0f2 100644 --- a/src/JSON/Utils/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy index 7e6a4a3f..170a980a 100644 --- a/src/JSON/Utils/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" include "../../Wrappers.dfy" include "Views.dfy" diff --git a/src/JSON/Utils/Views.dfy b/src/JSON/Utils/Views.dfy index ef0f7c4f..bf15268a 100644 --- a/src/JSON/Utils/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../../BoundedInts.dfy" module {:options "-functionSyntax:4"} JSON.Utils.Views.Core { diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index a6f041b9..9da7c950 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../Grammar.dfy" include "../ConcreteSyntax.Spec.dfy" include "Serializer.dfy" diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index acd74952..3df5c43c 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../Errors.dfy" include "../Grammar.dfy" include "../ConcreteSyntax.Spec.dfy" diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index 7fb4c3a6..c5faa9d6 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,3 +1,5 @@ +// RUN: %dafny -compile:0 "%s" + include "../Errors.dfy" include "../ConcreteSyntax.Spec.dfy" include "../ConcreteSyntax.SpecProperties.dfy" From 3c717385b345693631d139770593c29c3a9b9399 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 4 Mar 2023 09:51:28 -0800 Subject: [PATCH 31/84] Updating lit syntax and a few function methods --- src/JSON/API.dfy | 2 +- src/JSON/AST.dfy | 2 +- src/JSON/ConcreteSyntax.Spec.dfy | 2 +- src/JSON/ConcreteSyntax.SpecProperties.dfy | 2 +- src/JSON/Deserializer.dfy | 2 +- src/JSON/Errors.dfy | 2 +- src/JSON/Grammar.dfy | 2 +- src/JSON/Serializer.dfy | 2 +- src/JSON/Spec.dfy | 2 +- src/JSON/Utils/Cursors.dfy | 2 +- src/JSON/Utils/Lexers.dfy | 2 +- src/JSON/Utils/Parsers.dfy | 2 +- src/JSON/Utils/Str.dfy | 2 +- src/JSON/Utils/Unicode.dfy | 2 +- src/JSON/Utils/Vectors.dfy | 2 +- src/JSON/Utils/Views.Writers.dfy | 2 +- src/JSON/Utils/Views.dfy | 2 +- src/JSON/ZeroCopy/API.dfy | 2 +- src/JSON/ZeroCopy/Deserializer.dfy | 2 +- src/JSON/ZeroCopy/Serializer.dfy | 2 +- src/Math.dfy | 2 +- src/NonlinearArithmetic/Logarithm.dfy | 4 ++-- 22 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index 4ec7b4ab..b4924b20 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "Serializer.dfy" include "Deserializer.dfy" diff --git a/src/JSON/AST.dfy b/src/JSON/AST.dfy index 37cf20e4..17d5629f 100644 --- a/src/JSON/AST.dfy +++ b/src/JSON/AST.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" module {:options "-functionSyntax:4"} JSON.AST { datatype Decimal = diff --git a/src/JSON/ConcreteSyntax.Spec.dfy b/src/JSON/ConcreteSyntax.Spec.dfy index 75784d70..8bd4d4cc 100644 --- a/src/JSON/ConcreteSyntax.Spec.dfy +++ b/src/JSON/ConcreteSyntax.Spec.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "Grammar.dfy" diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index d295ecc8..bf2e8d60 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "ConcreteSyntax.Spec.dfy" diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 1a146582..d93f2e22 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" /// =============================================== /// Deserialization from JSON.Grammar to JSON.AST diff --git a/src/JSON/Errors.dfy b/src/JSON/Errors.dfy index 04f3e239..34fae3a4 100644 --- a/src/JSON/Errors.dfy +++ b/src/JSON/Errors.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../Wrappers.dfy" include "../BoundedInts.dfy" diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy index c45ed0b0..7f6013d9 100644 --- a/src/JSON/Grammar.dfy +++ b/src/JSON/Grammar.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" /// ========================================== /// Low-level JSON grammar (Concrete syntax) diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index ec680097..caf5b9ad 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" /// ============================================= /// Serialization from JSON.AST to JSON.Grammar diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 58a744cc..9a2db6fa 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" /// ============================================= /// Serialization from AST.JSON to bytes (Spec) diff --git a/src/JSON/Utils/Cursors.dfy b/src/JSON/Utils/Cursors.dfy index eeddab20..d6a22b5f 100644 --- a/src/JSON/Utils/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Lexers.dfy b/src/JSON/Utils/Lexers.dfy index d2f3f65d..5c3d879a 100644 --- a/src/JSON/Utils/Lexers.dfy +++ b/src/JSON/Utils/Lexers.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../Wrappers.dfy" include "../../BoundedInts.dfy" diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 03f59f01..9696808c 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 4407f2a4..6bc428af 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 -noNLarith "%s" +// RUN: %verify -noNLarith "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index a375d66d..c66b6928 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" diff --git a/src/JSON/Utils/Vectors.dfy b/src/JSON/Utils/Vectors.dfy index 62d3b0f2..8d56ab1b 100644 --- a/src/JSON/Utils/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy index 170a980a..5a9549fb 100644 --- a/src/JSON/Utils/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/JSON/Utils/Views.dfy b/src/JSON/Utils/Views.dfy index bf15268a..1ec8cc4e 100644 --- a/src/JSON/Utils/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../../BoundedInts.dfy" diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index 9da7c950..bc1ef17d 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../Grammar.dfy" include "../ConcreteSyntax.Spec.dfy" diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 3df5c43c..929a1e26 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../Errors.dfy" include "../Grammar.dfy" diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index c5faa9d6..1ea3d579 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:0 "%s" +// RUN: %verify "%s" include "../Errors.dfy" include "../ConcreteSyntax.Spec.dfy" diff --git a/src/Math.dfy b/src/Math.dfy index ef4c1fe2..17cf15f2 100644 --- a/src/Math.dfy +++ b/src/Math.dfy @@ -22,7 +22,7 @@ module {:options "-functionSyntax:4"} Math { a } - function method Abs(a: int): (a': int) + function Abs(a: int): (a': int) ensures a' >= 0 { if a >= 0 then a else -a diff --git a/src/NonlinearArithmetic/Logarithm.dfy b/src/NonlinearArithmetic/Logarithm.dfy index fc438f2e..da157ecf 100644 --- a/src/NonlinearArithmetic/Logarithm.dfy +++ b/src/NonlinearArithmetic/Logarithm.dfy @@ -4,12 +4,12 @@ include "Mul.dfy" include "DivMod.dfy" include "Power.dfy" -module Logarithm { +module {:options "-functionSyntax:4"} Logarithm { import opened Mul import opened DivMod import opened Power - function method {:opaque} Log(base: nat, pow: nat): nat + function {:opaque} Log(base: nat, pow: nat): nat requires base > 1 decreases pow { From d1bc8dcdbab0c1fce2ed548768d0100d24248ec2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 4 Mar 2023 10:39:04 -0800 Subject: [PATCH 32/84] Remaining lit syntax fixes --- lit.site.cfg | 2 ++ src/JSON/Tests.dfy | 2 +- src/JSON/Tutorial.dfy | 2 +- src/JSON/Utils/Str.dfy | 2 +- src/NonlinearArithmetic/Logarithm.dfy | 2 +- src/dafny/Collections/Seqs.dfy | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lit.site.cfg b/lit.site.cfg index 60666266..67a1314e 100644 --- a/lit.site.cfg +++ b/lit.site.cfg @@ -140,6 +140,7 @@ resolveArgs = ' resolve --use-basename-for-filename ' + standardArguments verifyArgs = ' verify --use-basename-for-filename --cores:2 --verification-time-limit:300 ' + standardArguments buildArgs = ' build --use-basename-for-filename --cores:2 --verification-time-limit:300 ' + standardArguments runArgs = ' run --use-basename-for-filename --cores:2 --verification-time-limit:300 ' + standardArguments +testArgs = ' test --use-basename-for-filename --cores:2 --verification-time-limit:300 ' + standardArguments config.substitutions.append( ('%trargs', '--use-basename-for-filename --cores:2 --verification-time-limit:300' ) ) @@ -148,6 +149,7 @@ config.substitutions.append( ('%verify', dafnyExecutable + verifyArgs ) ) config.substitutions.append( ('%translate', dafnyExecutable + ' translate' ) ) config.substitutions.append( ('%build', dafnyExecutable + buildArgs ) ) config.substitutions.append( ('%run', dafnyExecutable + runArgs ) ) +config.substitutions.append( ('%test', dafnyExecutable + runArgs ) ) # config.substitutions.append( ('%repositoryRoot', repositoryRoot) ) # config.substitutions.append( ('%binaryDir', binaryDir) ) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index ba75945b..15d08397 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:3 "%s" +// RUN: %run "%s" include "Utils/Unicode.dfy" include "Errors.dfy" diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index 95704000..b61a084b 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny -compile:3 -runAllTests:1 "%s" +// RUN: %test "%s" /// # Using the JSON library include "API.dfy" diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 6bc428af..6b4115c6 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -1,4 +1,4 @@ -// RUN: %verify -noNLarith "%s" +// RUN: %verify --disable-nonlinear-arithmetic "%s" include "../../BoundedInts.dfy" include "../../Wrappers.dfy" diff --git a/src/NonlinearArithmetic/Logarithm.dfy b/src/NonlinearArithmetic/Logarithm.dfy index da157ecf..c04f3e88 100644 --- a/src/NonlinearArithmetic/Logarithm.dfy +++ b/src/NonlinearArithmetic/Logarithm.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:0 /noNLarith "%s" +// RUN: %verify --disable-nonlinear-arithmetic "%s" include "Mul.dfy" include "DivMod.dfy" diff --git a/src/dafny/Collections/Seqs.dfy b/src/dafny/Collections/Seqs.dfy index 7babe61f..e014cef5 100644 --- a/src/dafny/Collections/Seqs.dfy +++ b/src/dafny/Collections/Seqs.dfy @@ -656,7 +656,7 @@ module {:options "-functionSyntax:4"} Dafny.Collections.Seq { requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) ensures |result| <= |xs| ensures forall i: nat {:trigger result[i]} :: i < |result| && f.requires(result[i]) ==> f(result[i]) - reads f.reads + reads set i, o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o { if |xs| == 0 then [] else (if f(xs[0]) then [xs[0]] else []) + Filter(f, xs[1..]) From 634f88c33f3cb1bdb3d3146c14839bca987d19d9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 4 Mar 2023 12:40:42 -0800 Subject: [PATCH 33/84] Updating CI to use 3.13.1 and 4.0.0 Since this code hits bugs in 3.11 and 3.12 --- .github/workflows/nightly.yml | 8 +------- .github/workflows/reusable-tests.yml | 11 +---------- .github/workflows/tests.yml | 7 +------ 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a772f9b8..ff144cfa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,13 +14,7 @@ jobs: verification: strategy: matrix: - # nightly-latest to catch anything that breaks these tests in current development - # 2/18/2023 version is the first that supports logging - # 3.11.0 supports new CLI but does not support logging - # setup-dafny-action does not yet support 3.13.1 or recent nightly-lates -t - ## version: [ nightly-latest, nightly-2023-02-18-ef4f346, 3.11.0, 3.12.0, 3.13.1 ] - version: [ nightly-2023-02-18-ef4f346, 3.11.0, 3.12.0 ] + version: [ nightly-latest ] uses: ./.github/workflows/reusable-tests.yml with: diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index bd98122e..37d489b0 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v3 - name: Install Dafny - uses: dafny-lang/setup-dafny-action@v1.6.0 + uses: dafny-lang/setup-dafny-action@v1.6.1 with: dafny-version: ${{ inputs.dafny-version }} @@ -39,23 +39,14 @@ jobs: - name: Set up JS dependencies run: npm install bignumber.js - - name: Verify Code and Examples without logging - id: nolog - if: inputs.dafny-version == '3.11.0' - run: lit --time-tests -v . - - name: Verify Code and Examples - id: withlog - if: steps.nolog.conclusion == 'skipped' run: | lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv' . - name: Generate Report - if: always() && steps.withlog.conclusion != 'skipped' run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-duration-seconds 10 - uses: actions/upload-artifact@v2 # upload test results - if: always() && steps.withlog.conclusion != 'skipped' with: name: verification-results path: '**/TestResults/*.trx' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 322f75f3..b42bc958 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,12 +13,7 @@ jobs: verification: strategy: matrix: - # nightly-latest to catch anything that breaks these tests in current development - # 2/18/2023 version is the first that supports logging - # 3.11.0 supports new CLI but does not support logging - # setup-dafny-action does not yet support 3.13.1 or recent nightly-latest - ## version: [ nightly-2023-02-18-ef4f346, 3.11.0, 3.12.0, 3.13.1 ] - version: [ nightly-2023-02-18-ef4f346, 3.11.0, 3.12.0 ] + version: [ 3.13.1, 4.0.0 ] uses: ./.github/workflows/reusable-tests.yml with: From 08cf34d0210f6b9066e9f7bac3d2e1749d5ba5a2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 6 Mar 2023 13:25:41 -0800 Subject: [PATCH 34/84] Split up the most expensive assertion batch --- .github/workflows/tests.yml | 2 +- src/JSON/ZeroCopy/Deserializer.dfy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b42bc958..d38911c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: verification: strategy: matrix: - version: [ 3.13.1, 4.0.0 ] + version: [ 3.13.1 ] uses: ./.github/workflows/reusable-tests.yml with: diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 929a1e26..ab5c04b4 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -492,7 +492,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { else Success(sp) } - function {:opaque} Exp(cs: FreshCursor) : (pr: ParseResult>) + function {:opaque} {:vcs_split_on_every_assert} Exp(cs: FreshCursor) : (pr: ParseResult>) ensures pr.Success? ==> pr.value.SplitFrom?(cs, exp => Spec.Maybe(exp, Spec.Exp)) { var SP(e, cs) := From 578cb95b72b5d3812a12117593c010571f48ec10 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 01:34:22 +0100 Subject: [PATCH 35/84] drop support for old Dafny versions --- .github/workflows/nightly.yml | 7 +------ .github/workflows/tests.yml | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a4cca8d2..ff144cfa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -14,12 +14,7 @@ jobs: verification: strategy: matrix: - # nightly-latest to catch anything that breaks these tests in current development - # 2/18/2023 version is the first that supports logging, but it is not supported by setup-dafny-action 1.6.1 - # 3.11.0 supports new CLI but does not support logging - # setup-dafny-action does not yet support 3.13.1 or recent nightly-lates - - version: [ nightly-latest, nightly-2023-02-18-ef4f346, 3.11.0, 3.12.0, 3.13.1, 4.0.0 ] + version: [ nightly-latest ] uses: ./.github/workflows/reusable-tests.yml with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1379bd25..f8ca51ef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,12 +12,7 @@ concurrency: jobs: verification: strategy: - matrix: - # nightly-latest to catch anything that breaks these tests in current development - # 2/18/2023 version is the first that supports logging, but it is not supported by dafny-setup-action 1.6.1 - # 3.11.0 supports new CLI but does not support logging - # setup-dafny-action does not yet support 3.13.1 or recent nightly-latest - version: [ 3.11.0, 3.12.0, 3.13.1, 4.0.0 ] + version: [ 3.13.1, 4.0.0 ] uses: ./.github/workflows/reusable-tests.yml with: From b96bbae1b4ed9b6bb966eecc55a88004cb492550 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:17:24 +0100 Subject: [PATCH 36/84] format --- src/JSON/ConcreteSyntax.SpecProperties.dfy | 14 +-- src/JSON/Deserializer.dfy | 49 +++++----- src/JSON/Errors.dfy | 34 +++---- src/JSON/Serializer.dfy | 12 +-- src/JSON/Spec.dfy | 36 ++++---- src/JSON/Tutorial.dfy | 90 ++++++++++--------- src/JSON/Utils/Cursors.dfy | 46 +++++----- src/JSON/Utils/Lexers.dfy | 20 ++--- src/JSON/Utils/Parsers.dfy | 2 +- src/JSON/Utils/Str.dfy | 13 +-- src/JSON/Utils/Unicode.dfy | 16 ++-- src/JSON/Utils/Vectors.dfy | 32 +++---- src/JSON/Utils/Views.Writers.dfy | 4 +- src/JSON/Utils/Views.dfy | 12 +-- src/JSON/ZeroCopy/Deserializer.dfy | 56 ++++++------ src/JSON/ZeroCopy/Serializer.dfy | 32 +++---- .../Internals/ModInternals.dfy | 2 +- src/NonlinearArithmetic/Logarithm.dfy | 10 +-- 18 files changed, 244 insertions(+), 236 deletions(-) diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index bf2e8d60..8613de04 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -3,7 +3,7 @@ include "ConcreteSyntax.Spec.dfy" module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties - // Some useful properties about the functions used in `ConcreteSyntax.Spec`. +// Some useful properties about the functions used in `ConcreteSyntax.Spec`. { import opened BoundedInts @@ -13,15 +13,15 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties lemma Bracketed_Morphism(bracketed: Bracketed) ensures forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes - | && (forall d | d < bracketed :: pd0.requires(d)) - && (forall d | d < bracketed :: pd1.requires(d)) - && (forall d | d < bracketed :: pd0(d) == pd1(d)) - :: Spec.Bracketed(bracketed, pd0) == Spec.Bracketed(bracketed, pd1) + | && (forall d | d < bracketed :: pd0.requires(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) + :: Spec.Bracketed(bracketed, pd0) == Spec.Bracketed(bracketed, pd1) { forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d < bracketed :: pd0.requires(d)) - && (forall d | d < bracketed :: pd1.requires(d)) - && (forall d | d < bracketed :: pd0(d) == pd1(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) { calc { Spec.Bracketed(bracketed, pd0); diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index d93f2e22..e23e3eb7 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -63,14 +63,14 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success([hd as char] + tl) else var unescaped: uint16 := match c - case '\"' => 0x22 as uint16 // quotation mark - case '\\' => 0x5C as uint16 // reverse solidus - case 'b' => 0x08 as uint16 // backspace - case 'f' => 0x0C as uint16 // form feed - case 'n' => 0x0A as uint16 // line feed - case 'r' => 0x0D as uint16 // carriage return - case 't' => 0x09 as uint16 // tab - case _ => 0 as uint16; + case '\"' => 0x22 as uint16 // quotation mark + case '\\' => 0x5C as uint16 // reverse solidus + case 'b' => 0x08 as uint16 // backspace + case 'f' => 0x0C as uint16 // form feed + case 'n' => 0x0A as uint16 // line feed + case 'r' => 0x0D as uint16 // carriage return + case 't' => 0x09 as uint16 // tab + case _ => 0 as uint16; if unescaped == 0 as uint16 then Failure(UnsupportedEscape(str[start..start+2])) else @@ -96,11 +96,12 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { type Char = byte } - const DIGITS := map[ - '0' as uint8 := 0, '1' as uint8 := 1, '2' as uint8 := 2, '3' as uint8 := 3, - '4' as uint8 := 4, '5' as uint8 := 5, '6' as uint8 := 6, '7' as uint8 := 7, - '8' as uint8 := 8, '9' as uint8 := 9 - ] + const DIGITS := + map[ + '0' as uint8 := 0, '1' as uint8 := 1, '2' as uint8 := 2, '3' as uint8 := 3, + '4' as uint8 := 4, '5' as uint8 := 5, '6' as uint8 := 6, '7' as uint8 := 7, + '8' as uint8 := 8, '9' as uint8 := 9 + ] const MINUS := '-' as uint8 @@ -117,11 +118,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { case Empty => Success(0) case NonEmpty(JExp(_, sign, num)) => ToInt(sign, num); match frac - case Empty => Success(AST.Decimal(n, e10)) - case NonEmpty(JFrac(_, num)) => - var pow10 := num.Length() as int; - var frac :- ToInt(minus, num); - Success(AST.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) + case Empty => Success(AST.Decimal(n, e10)) + case NonEmpty(JFrac(_, num)) => + var pow10 := num.Length() as int; + var frac :- ToInt(minus, num); + Success(AST.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) } function KeyValue(js: Grammar.jKeyValue): DeserializationResult<(string, AST.JSON)> { @@ -140,12 +141,12 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { function Value(js: Grammar.Value): DeserializationResult { match js - case Null(_) => Success(AST.Null()) - case Bool(b) => Success(AST.Bool(Bool(b))) - case String(str) => var s :- String(str); Success(AST.String(s)) - case Number(dec) => var n :- Number(dec); Success(AST.Number(n)) - case Object(obj) => var o :- Object(obj); Success(AST.Object(o)) - case Array(arr) => var a :- Array(arr); Success(AST.Array(a)) + case Null(_) => Success(AST.Null()) + case Bool(b) => Success(AST.Bool(Bool(b))) + case String(str) => var s :- String(str); Success(AST.String(s)) + case Number(dec) => var n :- Number(dec); Success(AST.Number(n)) + case Object(obj) => var o :- Object(obj); Success(AST.Object(o)) + case Array(arr) => var a :- Array(arr); Success(AST.Array(a)) } function JSON(js: Grammar.JSON): DeserializationResult { diff --git a/src/JSON/Errors.dfy b/src/JSON/Errors.dfy index 34fae3a4..d8aae9db 100644 --- a/src/JSON/Errors.dfy +++ b/src/JSON/Errors.dfy @@ -22,20 +22,20 @@ module {:options "-functionSyntax:4"} JSON.Errors { { function ToString() : string { match this - case UnterminatedSequence => "Unterminated sequence" - case UnsupportedEscape(str) => "Unsupported escape sequence: " + str - case EscapeAtEOS => "Escape character at end of string" - case EmptyNumber => "Number must contain at least one digit" - case ExpectingEOF => "Expecting EOF" - case IntOverflow => "Input length does not fit in a 32-bit counter" - case ReachedEOF => "Reached EOF" - case ExpectingByte(b0, b) => - var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; - "Expecting '" + [b0 as char] + "', read " + c - case ExpectingAnyByte(bs0, b) => - var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; - var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); - "Expecting one of '" + c0s + "', read " + c + case UnterminatedSequence => "Unterminated sequence" + case UnsupportedEscape(str) => "Unsupported escape sequence: " + str + case EscapeAtEOS => "Escape character at end of string" + case EmptyNumber => "Number must contain at least one digit" + case ExpectingEOF => "Expecting EOF" + case IntOverflow => "Input length does not fit in a 32-bit counter" + case ReachedEOF => "Reached EOF" + case ExpectingByte(b0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + "Expecting '" + [b0 as char] + "', read " + c + case ExpectingAnyByte(bs0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); + "Expecting one of '" + c0s + "', read " + c } } @@ -46,9 +46,9 @@ module {:options "-functionSyntax:4"} JSON.Errors { { function ToString() : string { match this - case OutOfMemory => "Out of memory" - case IntTooLarge(i: int) => "Integer too large: " + Str.OfInt(i) - case StringTooLong(s: string) => "String too long: " + s + case OutOfMemory => "Out of memory" + case IntTooLarge(i: int) => "Integer too large: " + Str.OfInt(i) + case StringTooLong(s: string) => "String too long: " + s } } diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index caf5b9ad..b0d65933 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -164,12 +164,12 @@ module {:options "-functionSyntax:4"} JSON.Serializer { function Value(js: AST.JSON): Result { match js - case Null => Success(Grammar.Null(View.OfBytes(NULL))) - case Bool(b) => Success(Grammar.Bool(Bool(b))) - case String(str) => var s :- String(str); Success(Grammar.String(s)) - case Number(dec) => var n :- Number(dec); Success(Grammar.Number(n)) - case Object(obj) => var o :- Object(obj); Success(Grammar.Object(o)) - case Array(arr) => var a :- Array(arr); Success(Grammar.Array(a)) + case Null => Success(Grammar.Null(View.OfBytes(NULL))) + case Bool(b) => Success(Grammar.Bool(Bool(b))) + case String(str) => var s :- String(str); Success(Grammar.String(s)) + case Number(dec) => var n :- Number(dec); Success(Grammar.Number(n)) + case Object(obj) => var o :- Object(obj); Success(Grammar.Object(o)) + case Array(arr) => var a :- Array(arr); Success(Grammar.Array(a)) } function JSON(js: AST.JSON): Result { diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 9a2db6fa..33cd3ff8 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -42,16 +42,16 @@ module {:options "-functionSyntax:4"} JSON.Spec { if start >= |str| then [] else (match str[start] as uint16 - case 0x22 => "\\\"" // quotation mark - case 0x5C => "\\\\" // reverse solidus - case 0x08 => "\\b" // backspace - case 0x0C => "\\f" // form feed - case 0x0A => "\\n" // line feed - case 0x0D => "\\r" // carriage return - case 0x09 => "\\t" // tab - case c => - if c < 0x001F then "\\u" + EscapeUnicode(c) - else [str[start]]) + case 0x22 => "\\\"" // quotation mark + case 0x5C => "\\\\" // reverse solidus + case 0x08 => "\\b" // backspace + case 0x0C => "\\f" // form feed + case 0x0A => "\\n" // line feed + case 0x0D => "\\r" // carriage return + case 0x09 => "\\t" // tab + case c => + if c < 0x001F then "\\u" + EscapeUnicode(c) + else [str[start]]) + Escape(str, start + 1) } @@ -68,8 +68,8 @@ module {:options "-functionSyntax:4"} JSON.Spec { function Number(dec: Decimal): bytes { Transcode16To8(Str.OfInt(dec.n)) + - (if dec.e10 == 0 then [] - else ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) + (if dec.e10 == 0 then [] + else ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) } function KeyValue(kv: (string, JSON)): bytes { @@ -96,11 +96,11 @@ module {:options "-functionSyntax:4"} JSON.Spec { function JSON(js: JSON): bytes { match js - case Null => ToBytes("null") - case Bool(b) => if b then ToBytes("true") else ToBytes("false") - case String(str) => String(str) - case Number(dec) => Number(dec) - case Object(obj) => Object(obj) - case Array(arr) => Array(arr) + case Null => ToBytes("null") + case Bool(b) => if b then ToBytes("true") else ToBytes("false") + case String(str) => String(str) + case Number(dec) => Number(dec) + case Object(obj) => Object(obj) + case Array(arr) => Array(arr) } } diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index b61a084b..a87384ce 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -55,28 +55,29 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { } ] }"); - var CITIES_AST := Object([ - ("Cities", Array([ - Object([ - ("Name", String("Boston")), - ("Founded", Number(Int(1630))), - ("Population", Number(Int(689386))), - ("Area (km2)", Number(Decimal(45842, -1))) - ]), - Object([ - ("Name", String("Rome")), - ("Founded", Number(Int(-753))), - ("Population", Number(Decimal(2873, 3))), - ("Area (km2)", Number(Int(1285))) - ]), - Object([ - ("Name", String("Paris")), - ("Founded", Null), - ("Population", Number(Decimal(2161, 3))), - ("Area (km2)", Number(Decimal(23835, -1))) - ]) - ])) - ]); + var CITIES_AST := + Object([ + ("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1))) + ]), + Object([ + ("Name", String("Rome")), + ("Founded", Number(Int(-753))), + ("Population", Number(Decimal(2873, 3))), + ("Area (km2)", Number(Int(1285))) + ]), + Object([ + ("Name", String("Paris")), + ("Founded", Null), + ("Population", Number(Decimal(2161, 3))), + ("Area (km2)", Number(Decimal(23835, -1))) + ]) + ])) + ]); expect API.Deserialize(CITIES_JS) == Success(CITIES_AST); /// Serialization works similarly, with `API.Serialize`. For this first example @@ -86,9 +87,11 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { /// For more complex object, the generated layout may not be exactly the same; note in particular how the representation of numbers and the whitespace have changed. - expect API.Serialize(CITIES_AST) == Success(Unicode.Transcode16To8( + var EXPECTED := Unicode.Transcode16To8( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1},{""Name"":""Rome"",""Founded"":-753,""Population"":2873e3,""Area (km2)"":1285},{""Name"":""Paris"",""Founded"":null,""Population"":2161e3,""Area (km2)"":23835e-1}]}" - )); + ); + + expect API.Serialize(CITIES_AST) == Success(EXPECTED); /// Additional methods are defined in `API.dfy` to serialize an object into an /// existing buffer or into an array. Below is the smaller example from the @@ -100,18 +103,21 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { ""Population"": 689386, ""Area (km2)"": 4584.2}]}"); - var CITY_AST := Object([("Cities", Array([ - Object([ - ("Name", String("Boston")), - ("Founded", Number(Int(1630))), - ("Population", Number(Int(689386))), - ("Area (km2)", Number(Decimal(45842, -1)))])]))]); + var CITY_AST := + Object([("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1)))])]))]); expect API.Deserialize(CITY_JS) == Success(CITY_AST); - expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( + var EXPECTED' := Unicode.Transcode16To8( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" - )); + ); + + expect API.Serialize(CITY_AST) == Success(EXPECTED'); } } @@ -213,25 +219,25 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// Non-recursive cases are untouched: - case Bool(_) => js - case String(_) => js - case Number(_) => js + case Bool(_) => js + case String(_) => js + case Number(_) => js /// `Null` is replaced with the new `replacement` value: - case Null(_) => replacement + case Null(_) => replacement /// … and objects and arrays are traversed recursively (only the data part of is /// traversed: other fields record information about the formatting of braces, /// square brackets, and whitespace, and can thus be reused without /// modifications): - case Object(obj) => - Object(obj.(data := MapSuffixedSequence(obj.data, (s: Suffixed) requires s in obj.data => - s.t.(v := ReplaceNull(s.t.v, replacement))))) - case Array(arr) => - Array(arr.(data := MapSuffixedSequence(arr.data, (s: Suffixed) requires s in arr.data => - ReplaceNull(s.t, replacement)))) + case Object(obj) => + Object(obj.(data := MapSuffixedSequence(obj.data, (s: Suffixed) requires s in obj.data => + s.t.(v := ReplaceNull(s.t.v, replacement))))) + case Array(arr) => + Array(arr.(data := MapSuffixedSequence(arr.data, (s: Suffixed) requires s in arr.data => + ReplaceNull(s.t, replacement)))) } /// Note that well-formedness criteria on the low-level AST are enforced using diff --git a/src/JSON/Utils/Cursors.dfy b/src/JSON/Utils/Cursors.dfy index d6a22b5f..28d1f490 100644 --- a/src/JSON/Utils/Cursors.dfy +++ b/src/JSON/Utils/Cursors.dfy @@ -40,15 +40,15 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { { // TODO: Include positions in errors function ToString(pr: R -> string) : string { match this - case EOF => "Reached EOF" - case ExpectingByte(b0, b) => - var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; - "Expecting '" + [b0 as char] + "', read " + c - case ExpectingAnyByte(bs0, b) => - var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; - var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); - "Expecting one of '" + c0s + "', read " + c - case OtherError(err) => pr(err) + case EOF => "Reached EOF" + case ExpectingByte(b0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + "Expecting '" + [b0 as char] + "', read " + c + case ExpectingAnyByte(bs0, b) => + var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; + var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); + "Expecting one of '" + c0s + "', read " + c + case OtherError(err) => pr(err) } } type CursorResult<+R> = Result> @@ -82,10 +82,10 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { ghost predicate StrictlyAdvancedFrom?(other: Cursor): (b: bool) requires Valid? ensures b ==> - SuffixLength() < other.SuffixLength() + SuffixLength() < other.SuffixLength() ensures b ==> - beg == other.beg && end == other.end ==> - forall idx | beg <= idx < point :: s[idx] == other.s[idx] + beg == other.beg && end == other.end ==> + forall idx | beg <= idx < point :: s[idx] == other.s[idx] { && s == other.s && beg == other.beg @@ -103,7 +103,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { ghost predicate StrictSuffixOf?(other: Cursor) requires Valid? ensures StrictSuffixOf?(other) ==> - Length() < other.Length() + Length() < other.Length() { && s == other.s && beg > other.beg @@ -307,11 +307,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { ensures pr.Success? ==> pr.value.AdvancedFrom?(this) { match step(st, Peek()) - case Accept => Success(this) - case Reject(err) => Failure(OtherError(err)) - case Partial(st) => - if EOF? then Failure(EOF) - else Skip(1).SkipWhileLexer(step, st) + case Accept => Success(this) + case Reject(err) => Failure(OtherError(err)) + case Partial(st) => + if EOF? then Failure(EOF) + else Skip(1).SkipWhileLexer(step, st) } by method { var point' := point; var end := this.end; @@ -325,11 +325,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Cursors { var minusone: opt_byte := -1; // BUG(https://github.com/dafny-lang/dafny/issues/2191) var c := if eof then minusone else this.s[point'] as opt_byte; match step(st', c) - case Accept => return Success(Cursor(this.s, this.beg, point', this.end)); - case Reject(err) => return Failure(OtherError(err)); - case Partial(st'') => - if eof { return Failure(EOF); } - else { st' := st''; point' := point' + 1; } + case Accept => return Success(Cursor(this.s, this.beg, point', this.end)); + case Reject(err) => return Failure(OtherError(err)); + case Partial(st'') => + if eof { return Failure(EOF); } + else { st' := st''; point' := point' + 1; } } } } diff --git a/src/JSON/Utils/Lexers.dfy b/src/JSON/Utils/Lexers.dfy index 5c3d879a..a600b3a5 100644 --- a/src/JSON/Utils/Lexers.dfy +++ b/src/JSON/Utils/Lexers.dfy @@ -9,7 +9,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Lexers { import opened BoundedInts datatype LexerResult<+T, +R> = - // A Lexer may return three results: + // A Lexer may return three results: | Accept // The input is valid. | Reject(err: R) // The input is not valid; `err` says why. | Partial(st: T) // More input is needed to finish lexing. @@ -39,15 +39,15 @@ module {:options "-functionSyntax:4"} JSON.Utils.Lexers { : LexerResult { match st - case Start() => - if byte == '\"' as opt_byte then Partial(Body(false)) - else Reject("String must start with double quote") - case End() => - Accept - case Body(escaped) => - if byte == '\\' as opt_byte then Partial(Body(!escaped)) - else if byte == '\"' as opt_byte && !escaped then Partial(End) - else Partial(Body(false)) + case Start() => + if byte == '\"' as opt_byte then Partial(Body(false)) + else Reject("String must start with double quote") + case End() => + Accept + case Body(escaped) => + if byte == '\\' as opt_byte then Partial(Body(!escaped)) + else if byte == '\"' as opt_byte && !escaped then Partial(End) + else Partial(Body(false)) } } } diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 9696808c..8ae42ae2 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -14,7 +14,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> type Parser = p: Parser_ | p.Valid?() - // BUG(https://github.com/dafny-lang/dafny/issues/2103) + // BUG(https://github.com/dafny-lang/dafny/issues/2103) witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, ghost spec: T -> bytes) { diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 6b4115c6..554ca937 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -118,7 +118,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { ToNat_any(str[..|str| - 1], base, digits) * base + (base - 1); { ToNat_bound(str[..|str| - 1], base, digits); LemmaMulInequalityAuto(); } - (Pow(base, |str| - 1) - 1) * base + base - 1; + (Pow(base, |str| - 1) - 1) * base + base - 1; { LemmaMulIsDistributiveAuto(); } Pow(base, |str| - 1) * base - 1; { reveal Pow(); LemmaMulIsCommutativeAuto(); } @@ -184,11 +184,12 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { const HEX_DIGITS: seq := "0123456789ABCDEF" - const HEX_TABLE := map[ - '0' := 0, '1' := 1, '2' := 2, '3' := 3, '4' := 4, '5' := 5, '6' := 6, '7' := 7, '8' := 8, '9' := 9, - 'a' := 0xA, 'b' := 0xB, 'c' := 0xC, 'd' := 0xD, 'e' := 0xE, 'f' := 0xF, - 'A' := 0xA, 'B' := 0xB, 'C' := 0xC, 'D' := 0xD, 'E' := 0xE, 'F' := 0xF - ] + const HEX_TABLE := + map[ + '0' := 0, '1' := 1, '2' := 2, '3' := 3, '4' := 4, '5' := 5, '6' := 6, '7' := 7, '8' := 8, '9' := 9, + 'a' := 0xA, 'b' := 0xB, 'c' := 0xC, 'd' := 0xD, 'e' := 0xE, 'f' := 0xF, + 'A' := 0xA, 'B' := 0xB, 'C' := 0xC, 'D' := 0xD, 'E' := 0xE, 'F' := 0xF + ] function OfNat(n: nat, base: int := 10) : (str: string) requires 2 <= base <= 16 diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy index c66b6928..872d85e3 100644 --- a/src/JSON/Utils/Unicode.dfy +++ b/src/JSON/Utils/Unicode.dfy @@ -20,7 +20,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Unicode { { (0x10000 + ((((c0 as bv32) & 0x03FF) << 10) - | ((c1 as bv32) & 0x03FF))) + | ((c1 as bv32) & 0x03FF))) as uint32 } @@ -104,19 +104,19 @@ module {:options "-functionSyntax:4"} JSON.Utils.Unicode { if (c0 as bv32 & 0x80) == 0 then (c0 as uint32, 1) else if (c0 as bv32 & 0xE0) == 0xC0 && c1 > -1 then - (( (((c0 as bv32) & 0x1F) << 6) - | ((c1 as bv32) & 0x3F )) as uint32, + (( (((c0 as bv32) & 0x1F) << 6) + | ((c1 as bv32) & 0x3F )) as uint32, 2) else if (c0 as bv32 & 0xF0) == 0xE0 && c1 > -1 && c2 > -1 then (( (((c0 as bv32) & 0x0F) << 12) - | (((c1 as bv32) & 0x3F) << 6) - | ( (c2 as bv32) & 0x3F )) as uint32, + | (((c1 as bv32) & 0x3F) << 6) + | ( (c2 as bv32) & 0x3F )) as uint32, 3) else if (c0 as bv32 & 0xF8) == 0xF0 && c1 > -1 && c2 > -1 && c3 > -1 then (( (((c0 as bv32) & 0x07) << 18) - | (((c1 as bv32) & 0x3F) << 12) - | (((c2 as bv32) & 0x3F) << 6) - | ( (c3 as bv32) & 0x3F )) as uint32, + | (((c1 as bv32) & 0x3F) << 12) + | (((c2 as bv32) & 0x3F) << 6) + | ( (c3 as bv32) & 0x3F )) as uint32, 4) else (0xFFFD, 1) // Replacement character diff --git a/src/JSON/Utils/Vectors.dfy b/src/JSON/Utils/Vectors.dfy index 8d56ab1b..726e4d4d 100644 --- a/src/JSON/Utils/Vectors.dfy +++ b/src/JSON/Utils/Vectors.dfy @@ -122,13 +122,13 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { ensures Valid?() ensures o.Fail? <==> old(capacity) == MAX_CAPACITY ensures o.Fail? ==> - && unchanged(this) - && unchanged(data) + && unchanged(this) + && unchanged(data) ensures o.Pass? ==> - && fresh(data) - && old(capacity) < MAX_CAPACITY - && capacity == old(if capacity < MAX_CAPACITY_BEFORE_DOUBLING - then 2 * capacity else MAX_CAPACITY) + && fresh(data) + && old(capacity) < MAX_CAPACITY + && capacity == old(if capacity < MAX_CAPACITY_BEFORE_DOUBLING + then 2 * capacity else MAX_CAPACITY) { if capacity == MAX_CAPACITY { @@ -144,11 +144,11 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { ensures Valid?() modifies `capacity, `data, `Repr, data ensures reserved <= capacity - size ==> - o.Pass? + o.Pass? ensures o.Pass? ==> - old(size as int + reserved as int) <= capacity as int + old(size as int + reserved as int) <= capacity as int ensures o.Fail? ==> - reserved > MAX_CAPACITY - size + reserved > MAX_CAPACITY - size { if reserved > MAX_CAPACITY - size { return Fail(OutOfMemory); @@ -197,14 +197,14 @@ module {:options "-functionSyntax:4"} JSON.Utils.Vectors { modifies this, data ensures Valid?() ensures o.Fail? ==> - && unchanged(this) - && unchanged(data) + && unchanged(this) + && unchanged(data) ensures o.Pass? ==> - && old(size) < MAX_CAPACITY - && size == old(size) + 1 - && items == old(items) + [a] - && capacity >= old(capacity) - && if old(size == capacity) then fresh(data) else unchanged(`data) + && old(size) < MAX_CAPACITY + && size == old(size) + 1 + && items == old(items) + [a] + && capacity >= old(capacity) + && if old(size == capacity) then fresh(data) else unchanged(`data) { if size == capacity { var d := ReallocDefault(); diff --git a/src/JSON/Utils/Views.Writers.dfy b/src/JSON/Utils/Views.Writers.dfy index 5a9549fb..07656761 100644 --- a/src/JSON/Utils/Views.Writers.dfy +++ b/src/JSON/Utils/Views.Writers.dfy @@ -66,8 +66,8 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Writers { ghost const Valid? := length == // length is a saturating counter - if chain.Length() >= TWO_TO_THE_32 then UINT32_MAX - else chain.Length() as uint32 + if chain.Length() >= TWO_TO_THE_32 then UINT32_MAX + else chain.Length() as uint32 function Bytes() : (bs: bytes) ensures |bs| == Length() diff --git a/src/JSON/Utils/Views.dfy b/src/JSON/Utils/Views.dfy index 1ec8cc4e..d655b359 100644 --- a/src/JSON/Utils/Views.dfy +++ b/src/JSON/Utils/Views.dfy @@ -105,12 +105,12 @@ module {:options "-functionSyntax:4"} JSON.Utils.Views.Core { // Compare endpoints first to short-circuit the potentially-costly string // comparison && lv.end == rv.beg - // We would prefer to use reference equality here, but doing so in a sound - // way is tricky (see chapter 9 of ‘Verasco: a Formally Verified C Static - // Analyzer’ by Jacques-Henri Jourdan for details). The runtime optimizes - // the common case of physical equality and otherwise performs a length - // check, so the worst case (checking for adjacency in two slices that have - // equal but not physically-equal contents) is hopefully not too common. + // We would prefer to use reference equality here, but doing so in a sound + // way is tricky (see chapter 9 of ‘Verasco: a Formally Verified C Static + // Analyzer’ by Jacques-Henri Jourdan for details). The runtime optimizes + // the common case of physical equality and otherwise performs a length + // check, so the worst case (checking for adjacency in two slices that have + // equal but not physically-equal contents) is hopefully not too common. && lv.s == rv.s } diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index ab5c04b4..75e5444e 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -75,7 +75,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } type ValueParser = sp: SubParser | - forall t :: sp.spec(t) == Spec.Value(t) + forall t :: sp.spec(t) == Spec.Value(t) witness * } type Error = Core.Error @@ -149,9 +149,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} BracketedFromParts(ghost cs: Cursor, - open: Split>, - elems: Split>, - close: Split>) + open: Split>, + elems: Split>, + close: Split>) : (sp: Split) requires Grammar.NoTrailingSuffix(elems.t) requires open.StrictlySplitFrom?(cs, c => Spec.Structural(c, SpecView)) @@ -171,10 +171,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} AppendWithSuffix(ghost cs0: FreshCursor, - ghost json: ValueParser, - elems: Split>, - elem: Split, - sep: Split>) + ghost json: ValueParser, + elems: Split>, + elem: Split, + sep: Split>) : (elems': Split>) requires elems.cs.StrictlySplitFrom?(json.cs) requires elems.SplitFrom?(cs0, SuffixedElementsSpec) @@ -193,10 +193,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} AppendLast(ghost cs0: FreshCursor, - ghost json: ValueParser, - elems: Split>, - elem: Split, - sep: Split>) + ghost json: ValueParser, + elems: Split>, + elem: Split, + sep: Split>) : (elems': Split>) requires elems.cs.StrictlySplitFrom?(json.cs) requires elems.SplitFrom?(cs0, SuffixedElementsSpec) @@ -223,7 +223,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { open: Split>, elems: Split> ) // DISCUSS: Why is this function reverified once per instantiation of the module? - : (pr: ParseResult) + : (pr: ParseResult) requires open.StrictlySplitFrom?(cs0, c => Spec.Structural(c, SpecView)) requires elems.cs.StrictlySplitFrom?(json.cs) requires elems.SplitFrom?(open.cs, SuffixedElementsSpec) @@ -264,7 +264,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures x.r.t.Byte?(CLOSE) ensures NoTrailingSuffix(x.data) ensures forall pf | pf in x.data :: - pf.suffix.NonEmpty? ==> pf.suffix.t.t.Byte?(SEPARATOR) + pf.suffix.NonEmpty? ==> pf.suffix.t.t.Byte?(SEPARATOR) { // DISCUSS: Why is this lemma needed? Why does it require a body? var xlt: jopen := x.l.t; var xrt: jclose := x.r.t; @@ -291,10 +291,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { function LiftCursorError(err: Cursors.CursorError): DeserializationError { match err - case EOF => ReachedEOF - case ExpectingByte(expected, b) => ExpectingByte(expected, b) - case ExpectingAnyByte(expected_sq, b) => ExpectingAnyByte(expected_sq, b) - case OtherError(err) => err + case EOF => ReachedEOF + case ExpectingByte(expected, b) => ExpectingByte(expected, b) + case ExpectingAnyByte(expected_sq, b) => ExpectingAnyByte(expected_sq, b) + case OtherError(err) => err } function {:opaque} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) @@ -361,16 +361,16 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { else var SP(num, cs) :- Numbers.Number(cs); Success(SP(Grammar.Number(num), cs)) - } + } - function {:opaque} ValueParser(cs: FreshCursor) : (p: ValueParser) - decreases cs.Length(), 0 - ensures cs.SplitFrom?(p.cs) - { - var pre := (ps': FreshCursor) => ps'.Length() < cs.Length(); - var fn := (ps': FreshCursor) requires pre(ps') => Value(ps'); - Parsers.SubParser(cs, pre, fn, Spec.Value) - } + function {:opaque} ValueParser(cs: FreshCursor) : (p: ValueParser) + decreases cs.Length(), 0 + ensures cs.SplitFrom?(p.cs) + { + var pre := (ps': FreshCursor) => ps'.Length() < cs.Length(); + var fn := (ps': FreshCursor) requires pre(ps') => Value(ps'); + Parsers.SubParser(cs, pre, fn, Spec.Value) + } } module Constants { @@ -603,7 +603,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} KeyValueFromParts(ghost cs: Cursor, k: Split, - colon: Split>, v: Split) + colon: Split>, v: Split) : (sp: Split) requires k.StrictlySplitFrom?(cs, Spec.String) requires colon.StrictlySplitFrom?(k.cs, c => Spec.Structural(c, SpecView)) diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index 1ea3d579..dd992215 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -50,9 +50,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { ensures wr.Bytes() == writer.Bytes() + Spec.JSON(js) { writer - .Append(js.before) - .Then(wr => Value(js.t, wr)) - .Append(js.after) + .Append(js.before) + .Then(wr => Value(js.t, wr)) + .Append(js.after) } function {:opaque} Value(v: Value, writer: Writer) : (wr: Writer) @@ -60,12 +60,12 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { match v - case Null(n) => writer.Append(n) - case Bool(b) => writer.Append(b) - case String(str) => String(str, writer) - case Number(num) => Number(num, writer) - case Object(obj) => Object(obj, writer) - case Array(arr) => Array(arr, writer) + case Null(n) => writer.Append(n) + case Bool(b) => writer.Append(b) + case String(str) => String(str, writer) + case Number(num) => Number(num, writer) + case Object(obj) => Object(obj, writer) + case Array(arr) => Array(arr, writer) } function {:opaque} String(str: jstring, writer: Writer) : (wr: Writer) @@ -73,9 +73,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { ensures wr.Bytes() == writer.Bytes() + Spec.String(str) { writer - .Append(str.lq) - .Append(str.contents) - .Append(str.rq) + .Append(str.lq) + .Append(str.contents) + .Append(str.rq) } function {:opaque} Number(num: jnumber, writer: Writer) : (wr: Writer) @@ -84,11 +84,11 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { { var writer := writer.Append(num.minus).Append(num.num); var writer := if num.frac.NonEmpty? then - writer.Append(num.frac.t.period).Append(num.frac.t.num) - else writer; + writer.Append(num.frac.t.period).Append(num.frac.t.num) + else writer; var writer := if num.exp.NonEmpty? then - writer.Append(num.exp.t.e).Append(num.exp.t.sign).Append(num.exp.t.num) - else writer; + writer.Append(num.exp.t.e).Append(num.exp.t.sign).Append(num.exp.t.num) + else writer; writer } diff --git a/src/NonlinearArithmetic/Internals/ModInternals.dfy b/src/NonlinearArithmetic/Internals/ModInternals.dfy index d192970e..8a367ff8 100644 --- a/src/NonlinearArithmetic/Internals/ModInternals.dfy +++ b/src/NonlinearArithmetic/Internals/ModInternals.dfy @@ -91,7 +91,7 @@ module {:options "-functionSyntax:4"} ModInternals { LemmaFundamentalDivMod(x, n); LemmaFundamentalDivMod(x + n, n); var zp := (x + n) / n - x / n - 1; - forall ensures 0 == n * zp + ((x + n) % n) - (x % n) { LemmaMulAuto(); } + assert 0 == n * zp + ((x + n) % n) - (x % n) by { LemmaMulAuto(); } if (zp > 0) { LemmaMulInequality(1, zp, n); } if (zp < 0) { LemmaMulInequality(zp, -1, n); } } diff --git a/src/NonlinearArithmetic/Logarithm.dfy b/src/NonlinearArithmetic/Logarithm.dfy index c04f3e88..936d7ac2 100644 --- a/src/NonlinearArithmetic/Logarithm.dfy +++ b/src/NonlinearArithmetic/Logarithm.dfy @@ -39,11 +39,11 @@ module {:options "-functionSyntax:4"} Logarithm { lemma {:induction false} LemmaLogSAuto() ensures forall base: nat, pow: nat - {:trigger Log(base, pow / base)} - | && base > 1 - && pow >= base - :: && pow / base >= 0 - && Log(base, pow) == 1 + Log(base, pow / base) + {:trigger Log(base, pow / base)} + | && base > 1 + && pow >= base + :: && pow / base >= 0 + && Log(base, pow) == 1 + Log(base, pow / base) { forall base: nat, pow: nat | && base > 1 && pow >= base ensures && pow / base >= 0 From eb4158434927aca5c8631e3a8e9b3a9d82d75719 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:37:07 +0100 Subject: [PATCH 37/84] fix doctest --- src/JSON/README.md | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/JSON/README.md b/src/JSON/README.md index 48cded09..bedd0d7d 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -14,25 +14,35 @@ Both APIs provides functions for serialization (utf-8 bytes to AST) and deserial The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, call the high-level API, and use the low-level API to make localized modifications to a partial parse of a JSON AST. The main entry points are `API.Serialize` (to go from utf-8 bytes to a JSON AST), and `API.Deserialize` (for the reverse operation): + ```dafny -var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ - ""Name"": ""Boston"", - ""Founded"": 1630, - ""Population"": 689386, - ""Area (km2)"": 4584.2}]}"); - -var CITY_AST := Object([("Cities", Array([ - Object([ - ("Name", String("Boston")), - ("Founded", Number(Int(1630))), - ("Population", Number(Int(689386))), - ("Area (km2)", Number(Decimal(45842, -1)))])]))]); - -expect API.Deserialize(CITY_JS) == Success(CITY_AST); - -expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( - @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" -)); +include "src/JSON/API.dfy" + +import JSON.API +import JSON.Utils.Unicode +import opened JSON.AST +import opened Wrappers + +method Test(){ + var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ + ""Name"": ""Boston"", + ""Founded"": 1630, + ""Population"": 689386, + ""Area (km2)"": 4584.2}]}"); + + var CITY_AST := Object([("Cities", Array([ + Object([ + ("Name", String("Boston")), + ("Founded", Number(Int(1630))), + ("Population", Number(Int(689386))), + ("Area (km2)", Number(Decimal(45842, -1)))])]))]); + + expect API.Deserialize(CITY_JS) == Success(CITY_AST); + + expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( + @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" + )); +} ``` ## What is verified? From 899fcb7f7e4d89c59e48c64612965785fbaebe1a Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:40:03 +0100 Subject: [PATCH 38/84] fix workflow --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8ca51ef..b42bc958 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,6 +12,7 @@ concurrency: jobs: verification: strategy: + matrix: version: [ 3.13.1, 4.0.0 ] uses: ./.github/workflows/reusable-tests.yml From 9a321aa0b68fab9f6effcecfbf51f9d0fc1af3a8 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:41:44 +0100 Subject: [PATCH 39/84] drop old Dafny in the doc tests as well --- .github/workflows/check-examples-in-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-examples-in-docs.yml b/.github/workflows/check-examples-in-docs.yml index 7635cd64..a30d8b46 100644 --- a/.github/workflows/check-examples-in-docs.yml +++ b/.github/workflows/check-examples-in-docs.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - version: [ 3.12.0, 3.13.1, 4.0.0 ] + version: [ 3.13.1, 4.0.0 ] os: [ ubuntu-latest ] runs-on: ${{ matrix.os }} From 06dd2c67b9fd32802c87435cff1d37b573b6f6a8 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:53:50 +0100 Subject: [PATCH 40/84] stable formatting --- .github/workflows/check-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-format.yml b/.github/workflows/check-format.yml index 8ac260af..31dddd4e 100644 --- a/.github/workflows/check-format.yml +++ b/.github/workflows/check-format.yml @@ -15,7 +15,7 @@ jobs: - name: Install Dafny uses: dafny-lang/setup-dafny-action@v1 with: - dafny-version: "nightly-2023-02-15-567a5ba" + dafny-version: "4.0.0" - name: Install lit run: pip install lit OutputCheck From 22f2258c11a95eea8c7c68df52644217b03ac2e3 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 02:57:20 +0100 Subject: [PATCH 41/84] update `setup-dafny-action` --- .github/workflows/check-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-format.yml b/.github/workflows/check-format.yml index 31dddd4e..56fdcbe5 100644 --- a/.github/workflows/check-format.yml +++ b/.github/workflows/check-format.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - name: Install Dafny - uses: dafny-lang/setup-dafny-action@v1 + uses: dafny-lang/setup-dafny-action@v1.6.1 with: dafny-version: "4.0.0" From 829ae34d5f7498100a9dc4e054872c58c333950e Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 03:32:37 +0100 Subject: [PATCH 42/84] update literals --- src/JSON/Tests.dfy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 15d08397..2fe3a347 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -122,8 +122,8 @@ module JSON.Tests { " true ", " { } ", "\"\\t\\r\\n\\f\"", - "\"\u2200ABC // \\u2200ABC\"", // ∀ - "\"\uD83C\uDDEB\uD83C\uDDF7ABC // \\ud83c\\udDeB\\ud83c\\uDDF7ABC\"", // 🇫🇷 + "\"\U{2200}ABC // \\u2200ABC\"", // ∀ + "\"\U{1F1EB}\U{1F1F7} // \\u1f1eb\\u1f1EBABC\"", // 🇫🇷 "[true, false , null, { \"some\" : \"string\", \"and\": [ \"a number\", -123.456e-18 ] } ] " ]; From 8c7efb860a20090870be9f1d7a75e7b7992abaa3 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 03:56:47 +0100 Subject: [PATCH 43/84] weaken test --- src/JSON/Tests.dfy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 2fe3a347..255cfd9c 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -122,8 +122,8 @@ module JSON.Tests { " true ", " { } ", "\"\\t\\r\\n\\f\"", - "\"\U{2200}ABC // \\u2200ABC\"", // ∀ - "\"\U{1F1EB}\U{1F1F7} // \\u1f1eb\\u1f1EBABC\"", // 🇫🇷 + "\"∀ABC // \\u2200ABC\"", // ∀ + "\"🇫🇷 // \\u1f1eb\\u1f1EBABC\"", // 🇫🇷 "[true, false , null, { \"some\" : \"string\", \"and\": [ \"a number\", -123.456e-18 ] } ] " ]; From 9b4bd4a1d8f266dc56933b3dbd053c7602993df9 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 16 Mar 2023 03:58:01 +0100 Subject: [PATCH 44/84] revert assert by --- src/NonlinearArithmetic/Internals/ModInternals.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NonlinearArithmetic/Internals/ModInternals.dfy b/src/NonlinearArithmetic/Internals/ModInternals.dfy index 8a367ff8..d192970e 100644 --- a/src/NonlinearArithmetic/Internals/ModInternals.dfy +++ b/src/NonlinearArithmetic/Internals/ModInternals.dfy @@ -91,7 +91,7 @@ module {:options "-functionSyntax:4"} ModInternals { LemmaFundamentalDivMod(x, n); LemmaFundamentalDivMod(x + n, n); var zp := (x + n) / n - x / n - 1; - assert 0 == n * zp + ((x + n) % n) - (x % n) by { LemmaMulAuto(); } + forall ensures 0 == n * zp + ((x + n) % n) - (x % n) { LemmaMulAuto(); } if (zp > 0) { LemmaMulInequality(1, zp, n); } if (zp < 0) { LemmaMulInequality(zp, -1, n); } } From 7fb9e0241e810e74ddf9c9025badce45251539ee Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Mon, 20 Mar 2023 20:32:19 +0100 Subject: [PATCH 45/84] Dafny 4 --- src/JSON/Deserializer.dfy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index e23e3eb7..07b359a6 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -60,7 +60,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { var tl :- Unescape(str, start + 6); var hd := Str.ToNat(code, 16); assert hd < 0x10000 by { reveal Pow(); } - Success([hd as char] + tl) + if 0xD7FF < hd then + Failure(UnsupportedEscape(code)) + else + Success([hd as char]) + else var unescaped: uint16 := match c case '\"' => 0x22 as uint16 // quotation mark From a1cae956a410be6735dd0c261127e40b0afade9f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 6 Apr 2023 16:23:52 -0700 Subject: [PATCH 46/84] Optimize UnicodeEncodingForm.EncodeScalarSequence --- src/Collections/Sequences/Seq.dfy | 20 ++++++++++++++++++++ src/Unicode/UnicodeEncodingForm.dfy | 17 +++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/Collections/Sequences/Seq.dfy b/src/Collections/Sequences/Seq.dfy index 815970b9..39c57a9d 100644 --- a/src/Collections/Sequences/Seq.dfy +++ b/src/Collections/Sequences/Seq.dfy @@ -803,6 +803,26 @@ module {:options "-functionSyntax:4"} Seq { } } + /* Optimized implementation of Flatten(Map(f, xs)). */ + function {:opaque} FlatMap(f: (T ~> seq), xs: seq): (result: seq) + requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) + ensures result == Flatten(Map(f, xs)); + reads set i, o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o + { + Flatten(Map(f, xs)) + } + by method { + result := []; + ghost var unflattened: seq> := []; + for i := |xs| downto 0 + invariant unflattened == Map(f, xs[i..]) + invariant result == Flatten(unflattened) + { + var next := f(xs[i]); + unflattened := [next] + unflattened; + result := next + result; + } + } /********************************************************** * diff --git a/src/Unicode/UnicodeEncodingForm.dfy b/src/Unicode/UnicodeEncodingForm.dfy index d7f8a463..854ed1fb 100644 --- a/src/Unicode/UnicodeEncodingForm.dfy +++ b/src/Unicode/UnicodeEncodingForm.dfy @@ -239,6 +239,23 @@ abstract module {:options "-functionSyntax:4"} UnicodeEncodingForm { LemmaFlattenMinimalWellFormedCodeUnitSubsequences(ms); Seq.Flatten(ms) } + by method { + // Optimize to to avoid allocating the intermediate unflattened sequence. + // We can't quite use Seq.FlatMap easily because we need to prove the result + // is not just a seq but a WellFormedCodeUnitSeq. + // TODO: We can be even more efficient by using a JSON.Utils.Vectors.Vector instead. + s := []; + ghost var unflattened: seq := []; + for i := |vs| downto 0 + invariant unflattened == Seq.Map(EncodeScalarValue, vs[i..]) + invariant s == Seq.Flatten(unflattened) + { + var next: MinimalWellFormedCodeUnitSeq := EncodeScalarValue(vs[i]); + unflattened := [next] + unflattened; + LemmaPrependMinimalWellFormedCodeUnitSubsequence(next, s); + s := next + s; + } + } /** * Returns the scalar value sequence encoded by the given well-formed Unicode string. From 5553fb8edfdcaf3c5e0c77f65148b5e5a9ee1106 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Tue, 11 Apr 2023 21:32:30 +0200 Subject: [PATCH 47/84] fix Serializer --- src/JSON/Utils/Seq.dfy | 18 +++ src/JSON/ZeroCopy/Serializer.dfy | 181 +++++++++++++++++++++++-------- 2 files changed, 151 insertions(+), 48 deletions(-) create mode 100644 src/JSON/Utils/Seq.dfy diff --git a/src/JSON/Utils/Seq.dfy b/src/JSON/Utils/Seq.dfy new file mode 100644 index 00000000..dcc75afd --- /dev/null +++ b/src/JSON/Utils/Seq.dfy @@ -0,0 +1,18 @@ +module {:options "-functionSyntax:4"} JSON.Utils.Seq { + lemma Neutral(l: seq) + ensures l == l + [] + {} + + lemma Assoc(a: seq, b: seq, c: seq) + ensures a + b + c == a + (b + c) + {} + + + lemma Assoc'(a: seq, b: seq, c: seq) + ensures a + (b + c) == a + b + c + {} + + lemma Assoc2(a: seq, b: seq, c: seq, d: seq) + ensures a + b + c + d == a + (b + c + d) + {} +} diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index dd992215..84ea62fd 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,5 +1,6 @@ // RUN: %verify "%s" +include "../Utils/Seq.dfy" include "../Errors.dfy" include "../ConcreteSyntax.Spec.dfy" include "../ConcreteSyntax.SpecProperties.dfy" @@ -9,6 +10,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { import opened BoundedInts import opened Wrappers + import opened Seq = Utils.Seq import opened Errors import ConcreteSyntax.Spec import ConcreteSyntax.SpecProperties @@ -49,23 +51,31 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { function {:opaque} JSON(js: JSON, writer: Writer := Writer.Empty) : (wr: Writer) ensures wr.Bytes() == writer.Bytes() + Spec.JSON(js) { + Seq.Assoc2(writer.Bytes(),js.before.Bytes(), Spec.Value(js.t), js.after.Bytes()); writer .Append(js.before) .Then(wr => Value(js.t, wr)) .Append(js.after) } - function {:opaque} Value(v: Value, writer: Writer) : (wr: Writer) + lemma UnfoldValueNumber(v: Value) + requires v.Number? + ensures Spec.Value(v) == Spec.Number(v.num) + { + assert Spec.Value(v) == match v { case Number(num) => Spec.Number(num) case _ => []}; + } + + function {:opaque} {:vcs_split_on_every_assert} Value(v: Value, writer: Writer) : (wr: Writer) decreases v, 4 ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) { match v case Null(n) => writer.Append(n) - case Bool(b) => writer.Append(b) - case String(str) => String(str, writer) - case Number(num) => Number(num, writer) - case Object(obj) => Object(obj, writer) - case Array(arr) => Array(arr, writer) + case Bool(b) => var wr := writer.Append(b); wr + case String(str) => var wr := String(str, writer); wr + case Number(num) => assert Grammar.Number(num) == v by { UnfoldValueNumber(v); } var wr := Number(num, writer); wr + case Object(obj) => assert Grammar.Object(obj) == v; assert Spec.Value(v) == Spec.Object(obj); var wr := Object(obj, writer); wr + case Array(arr) => assert Grammar.Array(arr) == v; assert Spec.Value(v) == Spec.Array(arr); var wr := Array(arr, writer); assert wr.Bytes() == writer.Bytes() + Spec.Value(v); wr } function {:opaque} String(str: jstring, writer: Writer) : (wr: Writer) @@ -82,14 +92,25 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { decreases num, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Number(num) { - var writer := writer.Append(num.minus).Append(num.num); - var writer := if num.frac.NonEmpty? then - writer.Append(num.frac.t.period).Append(num.frac.t.num) - else writer; - var writer := if num.exp.NonEmpty? then - writer.Append(num.exp.t.e).Append(num.exp.t.sign).Append(num.exp.t.num) - else writer; - writer + var wr := writer.Append(num.minus).Append(num.num); + + var wr := if num.frac.NonEmpty? then + wr.Append(num.frac.t.period).Append(num.frac.t.num) + else wr; + assert wr.Bytes() == writer.Bytes() + Spec.View(num.minus) + Spec.View(num.num) + Spec.Maybe(num.frac, Spec.Frac) by { + assert num.frac.Empty? ==> wr.Bytes() == writer.Bytes() + Spec.View(num.minus) + Spec.View(num.num) + []; + } + + var wr := if num.exp.NonEmpty? then + wr.Append(num.exp.t.e).Append(num.exp.t.sign).Append(num.exp.t.num) + else wr; + assert wr.Bytes() == writer.Bytes() + Spec.View(num.minus) + Spec.View(num.num) + Spec.Maybe(num.frac, Spec.Frac) + Spec.Maybe(num.exp, Spec.Exp) by { + if num.exp.NonEmpty? {} else { + assert wr.Bytes() == writer.Bytes() + Spec.View(num.minus) + Spec.View(num.num) + Spec.Maybe(num.frac, Spec.Frac); + assert wr.Bytes() == writer.Bytes() + Spec.View(num.minus) + Spec.View(num.num) + Spec.Maybe(num.frac, Spec.Frac) + []; + } + } + wr } // DISCUSS: Can't be opaque, due to the lambda @@ -99,40 +120,73 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { writer.Append(st.before).Append(st.t).Append(st.after) } + lemma StructuralViewEns(st: Structural, writer: Writer) + ensures StructuralView(st, writer).Bytes() == writer.Bytes() + Spec.Structural(st, Spec.View) + {} + lemma {:axiom} Assume(b: bool) ensures b // FIXME refactor below to merge + lemma BracketedToObject(obj: jobject) + ensures Spec.Bracketed(obj, Spec.Member) == Spec.Object(obj) + { + var rMember := (d: jmember) requires d < obj => Spec.Member(d); + assert Spec.Bracketed(obj, Spec.Member) == Spec.Bracketed(obj, rMember) by { + // We call ``ConcatBytes`` with ``Spec.Member``, whereas the spec calls it + // with ``(d: jmember) requires d in obj.data => Spec.Member(d)``. That's + // why we need an explicit cast, which is performed by the lemma below. + SpecProperties.Bracketed_Morphism(obj); + assert forall d | d < obj :: Spec.Member(d) == rMember(d); + } + calc { + Spec.Bracketed(obj, Spec.Member); + Spec.Bracketed(obj, rMember); + Spec.Object(obj); + } + } + function {:opaque} Object(obj: jobject, writer: Writer) : (wr: Writer) decreases obj, 3 ensures wr.Bytes() == writer.Bytes() + Spec.Object(obj) { - var writer := StructuralView(obj.l, writer); - var writer := Members(obj, writer); - var writer := StructuralView(obj.r, writer); - - // We call ``ConcatBytes`` with ``Spec.Member``, whereas the spec calls it - // with ``(d: jmember) requires d in obj.data => Spec.Member(d)``. That's - // why we need an explicit cast, which is performed by the lemma below. - SpecProperties.Bracketed_Morphism(obj); - assert Spec.Object(obj) == Spec.Bracketed(obj, Spec.Member); - writer + var wr := StructuralView(obj.l, writer); + StructuralViewEns(obj.l, writer); + var wr := Members(obj, wr); + var wr := StructuralView(obj.r, wr); + Seq.Assoc2(writer.Bytes(), Spec.Structural(obj.l, Spec.View), Spec.ConcatBytes(obj.data, Spec.Member), Spec.Structural(obj.r, Spec.View)); + assert wr.Bytes() == writer.Bytes() + Spec.Bracketed(obj, Spec.Member); + assert Spec.Bracketed(obj, Spec.Member) == Spec.Object(obj) by { BracketedToObject(obj); } + wr + } + + lemma BracketedToArray(arr: jarray) + ensures Spec.Bracketed(arr, Spec.Item) == Spec.Array(arr) + { + var rItem := (d: jitem) requires d < arr => Spec.Item(d); + assert Spec.Bracketed(arr, Spec.Item) == Spec.Bracketed(arr, rItem) by { + SpecProperties.Bracketed_Morphism(arr); + assert forall d | d < arr :: Spec.Item(d) == rItem(d); + } + calc { + Spec.Bracketed(arr, Spec.Item); + Spec.Bracketed(arr, rItem); + Spec.Array(arr); + } } function {:opaque} Array(arr: jarray, writer: Writer) : (wr: Writer) decreases arr, 3 ensures wr.Bytes() == writer.Bytes() + Spec.Array(arr) { - var writer := StructuralView(arr.l, writer); - var writer := Items(arr, writer); - var writer := StructuralView(arr.r, writer); - - // We call ``ConcatBytes`` with ``Spec.Item``, whereas the spec calls it - // with ``(d: jitem) requires d in arr.data => Spec.Item(d)``. That's - // why we need an explicit cast, which is performed by the lemma below. - SpecProperties.Bracketed_Morphism(arr); // DISCUSS - assert Spec.Array(arr) == Spec.Bracketed(arr, Spec.Item); - writer + var wr := StructuralView(arr.l, writer); + StructuralViewEns(arr.l, writer); + var wr := Items(arr, wr); + var wr := StructuralView(arr.r, wr); + Seq.Assoc2(writer.Bytes(), Spec.Structural(arr.l, Spec.View), Spec.ConcatBytes(arr.data, Spec.Item), Spec.Structural(arr.r, Spec.View)); + assert wr.Bytes() == writer.Bytes() + Spec.Bracketed(arr, Spec.Item); + assert Spec.Bracketed(arr, Spec.Item) == Spec.Array(arr) by { BracketedToArray(arr); } + wr } function {:opaque} Members(obj: jobject, writer: Writer) : (wr: Writer) @@ -163,10 +217,15 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { // function is only used as a spec for Members. if members == [] then writer else - var writer := MembersSpec(obj, members[..|members|-1], writer); - assert members == members[..|members|-1] + [members[|members|-1]]; - SpecProperties.ConcatBytes_Linear(members[..|members|-1], [members[|members|-1]], Spec.Member); - Member(obj, members[|members|-1], writer) + var butLast, last := members[..|members|-1], members[|members|-1]; + assert members == butLast + [last]; + var wr := MembersSpec(obj, butLast, writer); + var wr := Member(obj, last, wr); + assert wr.Bytes() == writer.Bytes() + (Spec.ConcatBytes(butLast, Spec.Member) + Spec.ConcatBytes([last], Spec.Member)) by { + Seq.Assoc(writer.Bytes(), Spec.ConcatBytes(butLast, Spec.Member), Spec.ConcatBytes([last], Spec.Member)); + } + SpecProperties.ConcatBytes_Linear(butLast, [last], Spec.Member); + wr } // No by method block here, because the loop invariant in the method version // needs to call MembersSpec and the termination checker gets confused by // that. Instead, see Members above. // DISCUSS @@ -203,10 +262,15 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { // function is only used as a spec for Items. if items == [] then writer else - var writer := ItemsSpec(arr, items[..|items|-1], writer); - assert items == items[..|items|-1] + [items[|items|-1]]; - SpecProperties.ConcatBytes_Linear(items[..|items|-1], [items[|items|-1]], Spec.Item); - Item(arr, items[|items|-1], writer) + var butLast, last := items[..|items|-1], items[|items|-1]; + assert items == butLast + [last]; + var wr := ItemsSpec(arr, butLast, writer); + var wr := Item(arr, last, wr); + assert wr.Bytes() == writer.Bytes() + (Spec.ConcatBytes(butLast, Spec.Item) + Spec.ConcatBytes([last], Spec.Item)) by { + Seq.Assoc(writer.Bytes(), Spec.ConcatBytes(butLast, Spec.Item), Spec.ConcatBytes([last], Spec.Item)); + } + SpecProperties.ConcatBytes_Linear(butLast, [last], Spec.Item); + wr } // No by method block here, because the loop invariant in the method version // needs to call ItemsSpec and the termination checker gets confused by // that. Instead, see Items above. // DISCUSS @@ -225,6 +289,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { wr := Member(obj, members[i], wr); } assert members[..|members|] == members; + assert wr == MembersSpec(obj, members, writer); } method ItemsImpl(arr: jarray, writer: Writer) returns (wr: Writer) @@ -248,10 +313,26 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { decreases obj, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Member(m) { - var writer := String(m.t.k, writer); - var writer := StructuralView(m.t.colon, writer); - var writer := Value(m.t.v, writer); - if m.suffix.Empty? then writer else StructuralView(m.suffix.t, writer) + var wr := String(m.t.k, writer); + var wr := StructuralView(m.t.colon, wr); + var wr := Value(m.t.v, wr); + assert wr.Bytes() == writer.Bytes() + (Spec.String(m.t.k) + Spec.Structural(m.t.colon, Spec.View) + Spec.Value(m.t.v)) by { + Seq.Assoc2( writer.Bytes(), Spec.String(m.t.k), Spec.Structural(m.t.colon, Spec.View), Spec.Value(m.t.v)); + } + var wr := if m.suffix.Empty? then wr else StructuralView(m.suffix.t, wr); + assert wr.Bytes() == writer.Bytes() + Spec.KeyValue(m.t) + Spec.CommaSuffix(m.suffix) by { + if m.suffix.Empty? { + Neutral(Spec.KeyValue(m.t)); + Seq.Assoc'(writer.Bytes(), Spec.KeyValue(m.t), []); + } + else { + assert Spec.StructuralView(m.suffix.t) == Spec.CommaSuffix(m.suffix); + } + } + assert wr.Bytes() == writer.Bytes() + (Spec.KeyValue(m.t) + Spec.CommaSuffix(m.suffix)) by { + Seq.Assoc(writer.Bytes(), Spec.KeyValue(m.t), Spec.CommaSuffix(m.suffix)); + } + wr } function {:opaque} Item(ghost arr: jarray, m: jitem, writer: Writer) : (wr: Writer) @@ -259,7 +340,11 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { decreases arr, 0 ensures wr.Bytes() == writer.Bytes() + Spec.Item(m) { - var writer := Value(m.t, writer); - if m.suffix.Empty? then writer else StructuralView(m.suffix.t, writer) + var wr := Value(m.t, writer); + var wr := if m.suffix.Empty? then wr else StructuralView(m.suffix.t, wr); + assert wr.Bytes() == writer.Bytes() + (Spec.Value(m.t) + Spec.CommaSuffix(m.suffix)) by { + Seq.Assoc(writer.Bytes(), Spec.Value(m.t), Spec.CommaSuffix(m.suffix)); + } + wr } } From 7723fb506321d1ebf96e7eef8097c2c2bbf97ff2 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Tue, 11 Apr 2023 23:03:49 +0200 Subject: [PATCH 48/84] `RUN: %verify` --- src/JSON/Utils/Seq.dfy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/JSON/Utils/Seq.dfy b/src/JSON/Utils/Seq.dfy index dcc75afd..18d336f4 100644 --- a/src/JSON/Utils/Seq.dfy +++ b/src/JSON/Utils/Seq.dfy @@ -1,3 +1,5 @@ +// RUN: %verify "%s" + module {:options "-functionSyntax:4"} JSON.Utils.Seq { lemma Neutral(l: seq) ensures l == l + [] From e0827c523eb0ed550f979c7b7cfe525aa1e867b7 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 20 Apr 2023 00:25:01 +0200 Subject: [PATCH 49/84] fix Deserializer --- src/JSON/ConcreteSyntax.Spec.dfy | 21 ++++ src/JSON/ZeroCopy/Deserializer.dfy | 196 +++++++++++++++++++++++------ src/JSON/ZeroCopy/Serializer.dfy | 9 +- 3 files changed, 179 insertions(+), 47 deletions(-) diff --git a/src/JSON/ConcreteSyntax.Spec.dfy b/src/JSON/ConcreteSyntax.Spec.dfy index 8bd4d4cc..20510aab 100644 --- a/src/JSON/ConcreteSyntax.Spec.dfy +++ b/src/JSON/ConcreteSyntax.Spec.dfy @@ -94,6 +94,27 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.Spec { } } + lemma UnfoldValueNumber(v: Value) + requires v.Number? + ensures Value(v) == Number(v.num) + { + assert Value(v) == match v { case Number(num) => Number(num) case _ => []}; + } + + lemma UnfoldValueObject(v: Value) + requires v.Object? + ensures Value(v) == Object(v.obj) + { + assert Value(v) == match v { case Object(obj) => Object(obj) case _ => []}; + } + + lemma UnfoldValueArray(v: Value) + requires v.Array? + ensures Value(v) == Array(v.arr) + { + assert Value(v) == match v { case Array(arr) => Array(arr) case _ => []}; + } + function JSON(js: JSON) : bytes { Structural(js, Value) } diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 75e5444e..42e2b39c 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,5 +1,6 @@ // RUN: %verify "%s" +include "../Utils/Seq.dfy" include "../Errors.dfy" include "../Grammar.dfy" include "../ConcreteSyntax.Spec.dfy" @@ -17,6 +18,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import opened Utils.Parsers import opened Grammar import Errors + import opened Seq = Utils.Seq + type JSONError = Errors.DeserializationError type Error = CursorError @@ -92,7 +95,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type TElement - ghost function ElementSpec(t: TElement) : bytes + ghost function ElementSpec(t: TElement): bytes function Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) @@ -120,7 +123,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type TBracketed = Bracketed type TSuffixedElement = Suffixed - ghost function SuffixedElementSpec(e: TSuffixedElement) : bytes { + ghost function SuffixedElementSpec(e: TSuffixedElement): bytes { ElementSpec(e.t) + Spec.CommaSuffix(e.suffix) } @@ -160,13 +163,16 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures sp.StrictlySplitFrom?(cs, BracketedSpec) { var sp := SP(Grammar.Bracketed(open.t, elems.t, close.t), close.cs); - calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: - cs.Bytes(); - Spec.Structural(open.t, SpecView) + open.cs.Bytes(); - Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); - Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + Spec.Structural(close.t, SpecView) + close.cs.Bytes(); - Spec.Bracketed(sp.t, SuffixedElementSpec) + close.cs.Bytes(); + assert cs.Bytes() == Spec.Bracketed(sp.t, SuffixedElementSpec) + close.cs.Bytes() by { + assert cs.Bytes() == Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + Spec.Structural(close.t, SpecView) + close.cs.Bytes() by { + assert cs.Bytes() == Spec.Structural(open.t, SpecView) + open.cs.Bytes(); + assert open.cs.Bytes() == SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); + assert elems.cs.Bytes() == Spec.Structural(close.t, SpecView) + close.cs.Bytes(); + Seq.Assoc'(Spec.Structural(open.t, SpecView), SuffixedElementsSpec(elems.t), elems.cs.Bytes()); + Seq.Assoc'(Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t), Spec.Structural(close.t, SpecView), close.cs.Bytes()); + } } + assert sp.StrictlySplitFrom?(cs, BracketedSpec); sp } @@ -188,7 +194,28 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { { var suffixed := Suffixed(elem.t, NonEmpty(sep.t)); var elems' := SP(elems.t + [suffixed], sep.cs); // DISCUSS: Moving this down doubles the verification time - SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + + assert cs0.Bytes() == SuffixedElementsSpec(elems'.t) + sep.cs.Bytes() by { + assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + (ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix)) + sep.cs.Bytes() by { + assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix) + sep.cs.Bytes() by { + assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); + assert elems.cs.Bytes() == ElementSpec(suffixed.t) + elem.cs.Bytes(); + assert elem.cs.Bytes() == Spec.CommaSuffix(suffixed.suffix) + sep.cs.Bytes(); + Seq.Assoc'(SuffixedElementsSpec(elems.t), ElementSpec(suffixed.t), elem.cs.Bytes()); + Seq.Assoc'(SuffixedElementsSpec(elems.t) + ElementSpec(suffixed.t), Spec.CommaSuffix(suffixed.suffix), sep.cs.Bytes()); + } + Seq.Assoc(SuffixedElementsSpec(elems.t), ElementSpec(suffixed.t), Spec.CommaSuffix(suffixed.suffix)); + } + assert SuffixedElementsSpec(elems.t) + (ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix)) + sep.cs.Bytes() == SuffixedElementsSpec(elems'.t) + sep.cs.Bytes() by { + assert SuffixedElementsSpec(elems.t) + SuffixedElementSpec(suffixed) == SuffixedElementsSpec(elems.t + [suffixed]) by { + SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + assert Spec.ConcatBytes(elems.t, SuffixedElementSpec) + Spec.ConcatBytes([suffixed], SuffixedElementSpec) == Spec.ConcatBytes(elems.t + [suffixed], SuffixedElementSpec); + } + } + } + + assert elems'.StrictlySplitFrom?(cs0, SuffixedElementsSpec); + assert forall e | e in elems'.t :: e.suffix.NonEmpty?; elems' } @@ -211,7 +238,20 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { { var suffixed := Suffixed(elem.t, Empty()); var elems' := SP(elems.t + [suffixed], elem.cs); - SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + + assert cs0.Bytes() == SuffixedElementsSpec(elems'.t) + elem.cs.Bytes() by { + assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + ElementSpec(suffixed.t) + elem.cs.Bytes() by { + assert elem.t == suffixed.t; + } + assert SuffixedElementsSpec(elems.t) + ElementSpec(suffixed.t) + elem.cs.Bytes() == SuffixedElementsSpec(elems'.t) + elem.cs.Bytes() by { + assert SuffixedElementsSpec(elems.t) + SuffixedElementSpec(suffixed) == SuffixedElementsSpec(elems.t + [suffixed]) by { + SpecProperties.ConcatBytes_Linear(elems.t, [suffixed], SuffixedElementSpec); + assert Spec.ConcatBytes(elems.t, SuffixedElementSpec) + Spec.ConcatBytes([suffixed], SuffixedElementSpec) == Spec.ConcatBytes(elems.t + [suffixed], SuffixedElementSpec); + } + } + } + + assert elems'.StrictlySplitFrom?(cs0, SuffixedElementsSpec); elems' } @@ -235,13 +275,22 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { var sep := Core.TryStructural(elem.cs); var s0 := sep.t.t.Peek(); if s0 == SEPARATOR as opt_byte then + assert AppendWithSuffix.requires(open.cs, json, elems, elem, sep); var elems := AppendWithSuffix(open.cs, json, elems, elem, sep); Elements(cs0, json, open, elems) else if s0 == CLOSE as opt_byte then + assert AppendLast.requires(open.cs, json, elems, elem, sep) by { + assert sep.SplitFrom?(elem.cs, st => Spec.Structural(st, SpecView)); + assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); + } var elems := AppendLast(open.cs, json, elems, elem, sep); - Success(BracketedFromParts(cs0, open, elems, sep)) + assert BracketedFromParts.requires(cs0, open, elems, sep); + var bracketed := BracketedFromParts(cs0, open, elems, sep); + assert bracketed.StrictlySplitFrom?(cs0, BracketedSpec); + Success(bracketed) else - Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0)) + var pr := Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0)); + pr } function {:opaque} Bracketed(cs: FreshCursor, json: ValueParser) @@ -307,7 +356,10 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures jsr.Success? ==> v.Bytes() == Spec.JSON(jsr.value) { var SP(text, cs) :- JSON(Cursors.Cursor.OfView(v)); + assert Cursors.SP(text, cs).BytesSplitFrom?(Cursors.Cursor.OfView(v), Spec.JSON); + assert v.Bytes() == Spec.JSON(text) + cs.Bytes(); :- Need(cs.EOF?, Errors.ExpectingEOF); + assert cs.Bytes() == []; Success(text) } @@ -341,11 +393,23 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { { var c := cs.Peek(); if c == '{' as opt_byte then - var SP(obj, cs) :- Objects.Object(cs, ValueParser(cs)); - Success(SP(Grammar.Object(obj), cs)) + var SP(obj, cs') :- Objects.Object(cs, ValueParser(cs)); + var v := Grammar.Object(obj); + var sp := SP(v, cs'); + assert sp.StrictlySplitFrom?(cs, Spec.Value) by { + assert SP(obj, cs').StrictlySplitFrom?(cs, Spec.Object); + Spec.UnfoldValueObject(v); + } + Success(sp) else if c == '[' as opt_byte then - var SP(arr, cs) :- Arrays.Array(cs, ValueParser(cs)); - Success(SP(Grammar.Array(arr), cs)) + var SP(arr, cs') :- Arrays.Array(cs, ValueParser(cs)); + var v := Grammar.Array(arr); + var sp := SP(v, cs'); + assert sp.StrictlySplitFrom?(cs, Spec.Value) by { + assert SP(arr, cs').StrictlySplitFrom?(cs, Spec.Array); + Spec.UnfoldValueArray(v); + } + Success(sp) else if c == '\"' as opt_byte then var SP(str, cs) :- Strings.String(cs); Success(SP(Grammar.String(str), cs)) @@ -359,8 +423,14 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { var SP(cst, cs) :- Constants.Constant(cs, NULL); Success(SP(Grammar.Null(cst), cs)) else - var SP(num, cs) :- Numbers.Number(cs); - Success(SP(Grammar.Number(num), cs)) + var SP(num, cs') :- Numbers.Number(cs); + var v := Grammar.Number(num); + var sp := SP(v, cs'); + assert sp.StrictlySplitFrom?(cs, Spec.Value) by { + assert SP(num, cs').StrictlySplitFrom?(cs, Spec.Number); + Spec.UnfoldValueNumber(v); + } + Success(sp) } function {:opaque} ValueParser(cs: FreshCursor) : (p: ValueParser) @@ -531,14 +601,21 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures sp.StrictlySplitFrom?(cs, Spec.Number) { var sp := SP(Grammar.JNumber(minus.t, num.t, frac.t, exp.t), exp.cs); - calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: - cs.Bytes(); - Spec.View(minus.t) + minus.cs.Bytes(); - Spec.View(minus.t) + Spec.View(num.t) + num.cs.Bytes(); - Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac) + frac.cs.Bytes(); - Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac) + Spec.Maybe(exp.t, Spec.Exp) + exp.cs.Bytes(); - Spec.Number(sp.t) + exp.cs.Bytes(); + assert cs.Bytes() == Spec.Number(sp.t) + exp.cs.Bytes() by { + assert cs.Bytes() == Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac) + Spec.Maybe(exp.t, Spec.Exp) + exp.cs.Bytes() by { + assert cs.Bytes() == Spec.View(minus.t) + minus.cs.Bytes(); + assert minus.cs.Bytes() == Spec.View(num.t) + num.cs.Bytes(); + assert num.cs.Bytes() == Spec.Maybe(frac.t, Spec.Frac) + frac.cs.Bytes(); + assert frac.cs.Bytes() == Spec.Maybe(exp.t, Spec.Exp) + exp.cs.Bytes(); + Seq.Assoc'(Spec.View(minus.t), Spec.View(num.t), num.cs.Bytes()); + Seq.Assoc'(Spec.View(minus.t) + Spec.View(num.t), Spec.Maybe(frac.t, Spec.Frac), frac.cs.Bytes()); + Seq.Assoc'(Spec.View(minus.t) + Spec.View(num.t) + Spec.Maybe(frac.t, Spec.Frac), Spec.Maybe(exp.t, Spec.Exp), exp.cs.Bytes()); + } } + assert sp.StrictlySplitFrom?(cs, Spec.Number); + + + sp } @@ -574,14 +651,29 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { module Arrays refines Sequences { import opened Params = ArrayParams - function {:opaque} Array(cs: FreshCursor, json: ValueParser) + lemma BracketedToArray(arr: jarray) + ensures Spec.Bracketed(arr, SuffixedElementSpec) == Spec.Array(arr) + { + var rItem := (d: jitem) requires d < arr => Spec.Item(d); + assert Spec.Bracketed(arr, SuffixedElementSpec) == Spec.Bracketed(arr, rItem) by { + SpecProperties.Bracketed_Morphism(arr); + assert forall d | d < arr :: SuffixedElementSpec(d) == rItem(d); + } + calc { + Spec.Bracketed(arr, SuffixedElementSpec); + Spec.Bracketed(arr, rItem); + Spec.Array(arr); + } + } + + function {:opaque} Array(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) requires cs.SplitFrom?(json.cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Array) { var sp :- Bracketed(cs, json); - SpecProperties.Bracketed_Morphism(sp.t); - assert Spec.Bracketed(sp.t, SuffixedElementSpec) == Spec.Array(sp.t); + assert sp.StrictlySplitFrom?(cs, BracketedSpec); + BracketedToArray(sp.t); Success(sp) } } @@ -608,16 +700,19 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { requires k.StrictlySplitFrom?(cs, Spec.String) requires colon.StrictlySplitFrom?(k.cs, c => Spec.Structural(c, SpecView)) requires v.StrictlySplitFrom?(colon.cs, Spec.Value) - ensures sp.StrictlySplitFrom?(cs, Spec.KeyValue) + ensures sp.StrictlySplitFrom?(cs, ElementSpec) { var sp := SP(Grammar.KeyValue(k.t, colon.t, v.t), v.cs); - calc { // Dafny/Z3 has a lot of trouble with associativity, so do the steps one by one: - cs.Bytes(); - Spec.String(k.t) + k.cs.Bytes(); - Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + colon.cs.Bytes(); - Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + Spec.Value(v.t) + v.cs.Bytes(); - Spec.KeyValue(sp.t) + v.cs.Bytes(); + assert cs.Bytes() == Spec.KeyValue(sp.t) + v.cs.Bytes() by { + assert cs.Bytes() == Spec.String(k.t) + Spec.Structural(colon.t, SpecView) + Spec.Value(v.t) + v.cs.Bytes() by { + assert cs.Bytes() == Spec.String(k.t) + k.cs.Bytes(); + assert k.cs.Bytes() == Spec.Structural(colon.t, SpecView) + colon.cs.Bytes(); + assert colon.cs.Bytes() == Spec.Value(v.t) + v.cs.Bytes(); + Seq.Assoc'(Spec.String(k.t), Spec.Structural(colon.t, SpecView), colon.cs.Bytes()); + Seq.Assoc'(Spec.String(k.t) + Spec.Structural(colon.t, SpecView), Spec.Value(v.t), v.cs.Bytes()); + } } + assert sp.StrictlySplitFrom?(cs, ElementSpec); sp } @@ -628,23 +723,46 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { : (pr: ParseResult) { var k :- Strings.String(cs); - var colon :- Core.Structural(k.cs, Parsers.Parser(Colon, SpecView)); + assert k.cs.StrictlySplitFrom?(json.cs); + + var p := Parsers.Parser(Colon, SpecView); + assert p.Valid?(); + var colon :- Core.Structural(k.cs, p); + assert colon.StrictlySplitFrom?(k.cs, st => Spec.Structural(st, p.spec)); + assert colon.cs.StrictlySplitFrom?(json.cs); + var v :- json.fn(colon.cs); - Success(KeyValueFromParts(cs, k, colon, v)) + var kv := KeyValueFromParts(cs, k, colon, v); + Success(kv) } } module Objects refines Sequences { import opened Params = ObjectParams + lemma {:vcs_split_on_every_assert} BracketedToObject(obj: jobject) + ensures Spec.Bracketed(obj, SuffixedElementSpec) == Spec.Object(obj) + { + var rMember := (d: jmember) requires d < obj => Spec.Member(d); + assert Spec.Bracketed(obj, SuffixedElementSpec) == Spec.Bracketed(obj, rMember) by { + SpecProperties.Bracketed_Morphism(obj); + assert forall d | d < obj :: SuffixedElementSpec(d) == rMember(d); + } + calc { + Spec.Bracketed(obj, SuffixedElementSpec); + Spec.Bracketed(obj, rMember); + Spec.Object(obj); + } + } + function {:opaque} Object(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) requires cs.SplitFrom?(json.cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Object) { var sp :- Bracketed(cs, json); - SpecProperties.Bracketed_Morphism(sp.t); - assert Spec.Bracketed(sp.t, SuffixedElementSpec) == Spec.Object(sp.t); // DISCUSS + assert sp.StrictlySplitFrom?(cs, BracketedSpec); + BracketedToObject(sp.t); Success(sp) } } diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index 84ea62fd..11030e2d 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -58,13 +58,6 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { .Append(js.after) } - lemma UnfoldValueNumber(v: Value) - requires v.Number? - ensures Spec.Value(v) == Spec.Number(v.num) - { - assert Spec.Value(v) == match v { case Number(num) => Spec.Number(num) case _ => []}; - } - function {:opaque} {:vcs_split_on_every_assert} Value(v: Value, writer: Writer) : (wr: Writer) decreases v, 4 ensures wr.Bytes() == writer.Bytes() + Spec.Value(v) @@ -73,7 +66,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Serializer { case Null(n) => writer.Append(n) case Bool(b) => var wr := writer.Append(b); wr case String(str) => var wr := String(str, writer); wr - case Number(num) => assert Grammar.Number(num) == v by { UnfoldValueNumber(v); } var wr := Number(num, writer); wr + case Number(num) => assert Grammar.Number(num) == v by { Spec.UnfoldValueNumber(v); } var wr := Number(num, writer); wr case Object(obj) => assert Grammar.Object(obj) == v; assert Spec.Value(v) == Spec.Object(obj); var wr := Object(obj, writer); wr case Array(arr) => assert Grammar.Array(arr) == v; assert Spec.Value(v) == Spec.Array(arr); var wr := Array(arr, writer); assert wr.Bytes() == writer.Bytes() + Spec.Value(v); wr } From 942be81d480436adcfafb69281edadd96eb02b41 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 19 Apr 2023 15:41:23 -0700 Subject: [PATCH 50/84] Generalize JSON code to work with either `--unicode-char` mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Optimize UnicodeEncodingForm.EncodeScalarSequence * Add “polymorphic” (w.r.t. —unicode-char) string modules * Optimize Seq.Map * Rename for consistency * (Mostly) replaced JSON’s Unicode module * Fixed verification * Verifies with —unicode-char and UnicodeStringsWithUnicodeChar.dfy too * Unused file * Revert unneeded changes * Docstrings, attempting to fix CharIsUnicodeScalarValue * TODO --------- Co-authored-by: Fabio Madge --- src/Collections/Sequences/Seq.dfy | 48 +++++- src/JSON/Deserializer.dfy | 82 +++++---- src/JSON/Errors.dfy | 4 + src/JSON/Serializer.dfy | 29 +--- src/JSON/Spec.dfy | 126 +++++++++----- src/JSON/Tests.dfy | 12 +- src/JSON/Tutorial.dfy | 26 +-- src/JSON/Utils/Unicode.dfy | 160 ------------------ src/Unicode/UnicodeEncodingForm.dfy | 17 ++ src/Unicode/UnicodeStrings.dfy | 62 +++++++ src/Unicode/UnicodeStringsWithUnicodeChar.dfy | 85 ++++++++++ .../UnicodeStringsWithoutUnicodeChar.dfy | 52 ++++++ src/Wrappers.dfy | 6 + 13 files changed, 426 insertions(+), 283 deletions(-) delete mode 100644 src/JSON/Utils/Unicode.dfy create mode 100644 src/Unicode/UnicodeStrings.dfy create mode 100644 src/Unicode/UnicodeStringsWithUnicodeChar.dfy create mode 100644 src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy diff --git a/src/Collections/Sequences/Seq.dfy b/src/Collections/Sequences/Seq.dfy index 815970b9..0a04b969 100644 --- a/src/Collections/Sequences/Seq.dfy +++ b/src/Collections/Sequences/Seq.dfy @@ -90,6 +90,26 @@ module {:options "-functionSyntax:4"} Seq { * ***********************************************************/ + /* If a predicate is true at every index of a sequence, + it is true for every member of the sequence as a collection. + Useful for converting quantifiers between the two forms + to satisfy a precondition in the latter form. */ + lemma IndexingImpliesMembership(p: T -> bool, xs: seq) + requires forall i | 0 <= i < |xs| :: p(xs[i]) + ensures forall t | t in xs :: p(t) + { + } + + /* If a predicate is true for every member of a sequence as a collection, + it is true at every index of the sequence. + Useful for converting quantifiers between the two forms + to satisfy a precondition in the latter form. */ + lemma MembershipImpliesIndexing(p: T -> bool, xs: seq) + requires forall t | t in xs :: p(t) + ensures forall i | 0 <= i < |xs| :: p(xs[i]) + { + } + /* Is true if the sequence xs is a prefix of the sequence ys. */ ghost predicate IsPrefix(xs: seq, ys: seq) ensures IsPrefix(xs, ys) ==> (|xs| <= |ys| && xs == ys[..|xs|]) @@ -603,8 +623,12 @@ module {:options "-functionSyntax:4"} Seq { ensures forall i {:trigger result[i]} :: 0 <= i < |xs| ==> result[i] == f(xs[i]); reads set i, o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o { - if |xs| == 0 then [] - else [f(xs[0])] + Map(f, xs[1..]) + // This uses a sequence comprehension because it will usually be + // more efficient when compiled, allocating the storage for |xs| elements + // once instead of creating a chain of |xs| single element concatenations. + seq(|xs|, i requires 0 <= i < |xs| && f.requires(xs[i]) + reads set i,o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o + => f(xs[i])) } /* Applies a function to every element of a sequence, returning a Result value (which is a @@ -803,6 +827,26 @@ module {:options "-functionSyntax:4"} Seq { } } + /* Optimized implementation of Flatten(Map(f, xs)). */ + function {:opaque} FlatMap(f: (T ~> seq), xs: seq): (result: seq) + requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) + ensures result == Flatten(Map(f, xs)); + reads set i, o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o + { + Flatten(Map(f, xs)) + } + by method { + result := []; + ghost var unflattened: seq> := []; + for i := |xs| downto 0 + invariant unflattened == Map(f, xs[i..]) + invariant result == Flatten(unflattened) + { + var next := f(xs[i]); + unflattened := [next] + unflattened; + result := next + result; + } + } /********************************************************** * diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 07b359a6..32155a4b 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -11,7 +11,6 @@ include "../BoundedInts.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" -include "Utils/Unicode.dfy" include "Errors.dfy" include "AST.dfy" include "Grammar.dfy" @@ -25,7 +24,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Logarithm import opened Power import opened Utils.Str - import Utils.Unicode + import opened UnicodeStrings import AST import Spec @@ -39,60 +38,81 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { js.At(0) == 't' as byte } + function UnsupportedEscape16(code: seq): DeserializationError { + UnsupportedEscape(FromUTF16Checked(code).UnwrapOr("Couldn't decode UTF-16")) + } + + module Uint16StrConversion refines Str.ParametricConversion { + import opened BoundedInts + + type Char = uint16 + } + + const HEX_TABLE_16: map := + map[ + '0' as uint16 := 0, '1' as uint16 := 1, '2' as uint16 := 2, '3' as uint16 := 3, '4' as uint16 := 4, + '5' as uint16 := 5, '6' as uint16 := 6, '7' as uint16 := 7, '8' as uint16 := 8, '9' as uint16 := 9, + 'a' as uint16 := 0xA, 'b' as uint16 := 0xB, 'c' as uint16 := 0xC, 'd' as uint16 := 0xD, 'e' as uint16 := 0xE, 'f' as uint16 := 0xF, + 'A' as uint16 := 0xA, 'B' as uint16 := 0xB, 'C' as uint16 := 0xC, 'D' as uint16 := 0xD, 'E' as uint16 := 0xE, 'F' as uint16 := 0xF + ] + + function ToNat16(str: Uint16StrConversion.String): uint16 + requires |str| <= 4 + requires forall c | c in str :: c in HEX_TABLE_16 + { + Uint16StrConversion.ToNat_bound(str, 16, HEX_TABLE_16); + var hd := Uint16StrConversion.ToNat_any(str, 16, HEX_TABLE_16); + assert hd < 0x1_0000 by { reveal Pow(); } + hd as uint16 + } + // TODO: Verify this function - function Unescape(str: string, start: nat := 0): DeserializationResult + function Unescape(str: seq, start: nat := 0): DeserializationResult> decreases |str| - start { // Assumes UTF-16 strings if start >= |str| then Success([]) - else if str[start] == '\\' then + else if str[start] == '\\' as uint16 then if |str| == start + 1 then Failure(EscapeAtEOS) else var c := str[start + 1]; - if c == 'u' then + if c == 'u' as uint16 then if |str| <= start + 6 then Failure(EscapeAtEOS) else var code := str[start + 2..start + 6]; - if exists c | c in code :: c !in Str.HEX_TABLE then - Failure(UnsupportedEscape(code)) + if exists c | c in code :: c !in HEX_TABLE_16 then + Failure(UnsupportedEscape16(code)) else var tl :- Unescape(str, start + 6); - var hd := Str.ToNat(code, 16); - assert hd < 0x10000 by { reveal Pow(); } - if 0xD7FF < hd then - Failure(UnsupportedEscape(code)) - else - Success([hd as char]) - + var hd := ToNat16(code); + Success([hd]) else var unescaped: uint16 := match c - case '\"' => 0x22 as uint16 // quotation mark - case '\\' => 0x5C as uint16 // reverse solidus - case 'b' => 0x08 as uint16 // backspace - case 'f' => 0x0C as uint16 // form feed - case 'n' => 0x0A as uint16 // line feed - case 'r' => 0x0D as uint16 // carriage return - case 't' => 0x09 as uint16 // tab + case 0x22 => 0x22 as uint16 // \" => quotation mark + case 0x5C => 0x5C as uint16 // \\ => reverse solidus + case 0x62 => 0x08 as uint16 // \b => backspace + case 0x66 => 0x0C as uint16 // \f => form feed + case 0x6E => 0x0A as uint16 // \n => line feed + case 0x72 => 0x0D as uint16 // \r => carriage return + case 0x74 => 0x09 as uint16 // \t => tab case _ => 0 as uint16; - if unescaped == 0 as uint16 then - Failure(UnsupportedEscape(str[start..start+2])) + if unescaped as int == 0 then + Failure(UnsupportedEscape16(str[start..start+2])) else var tl :- Unescape(str, start + 2); - Success([unescaped as char] + tl) + Success([unescaped] + tl) else var tl :- Unescape(str, start + 1); Success([str[start]] + tl) } - function Transcode8To16Unescaped(str: seq): DeserializationResult - // TODO Optimize with a function by method - { - Unescape(Unicode.Transcode8To16(str)) - } - function String(js: Grammar.jstring): DeserializationResult { - Transcode8To16Unescaped(js.contents.Bytes()) + // TODO Optimize with a function by method + var asUtf32 :- FromUTF8Checked(js.contents.Bytes()).ToResult'(DeserializationError.InvalidUnicode); + var asUint16 :- ToUTF16Checked(asUtf32).ToResult'(DeserializationError.InvalidUnicode); + var unescaped :- Unescape(asUint16); + FromUTF16Checked(unescaped).ToResult'(DeserializationError.InvalidUnicode) } module ByteStrConversion refines Str.ParametricConversion { diff --git a/src/JSON/Errors.dfy b/src/JSON/Errors.dfy index d8aae9db..891cffc3 100644 --- a/src/JSON/Errors.dfy +++ b/src/JSON/Errors.dfy @@ -19,6 +19,7 @@ module {:options "-functionSyntax:4"} JSON.Errors { | ReachedEOF | ExpectingByte(expected: byte, b: opt_byte) | ExpectingAnyByte(expected_sq: seq, b: opt_byte) + | InvalidUnicode { function ToString() : string { match this @@ -36,6 +37,7 @@ module {:options "-functionSyntax:4"} JSON.Errors { var c := if b > 0 then "'" + [b as char] + "'" else "EOF"; var c0s := seq(|bs0|, idx requires 0 <= idx < |bs0| => bs0[idx] as char); "Expecting one of '" + c0s + "', read " + c + case InvalidUnicode => "Invalid Unicode sequence" } } @@ -43,12 +45,14 @@ module {:options "-functionSyntax:4"} JSON.Errors { | OutOfMemory | IntTooLarge(i: int) | StringTooLong(s: string) + | InvalidUnicode { function ToString() : string { match this case OutOfMemory => "Out of memory" case IntTooLarge(i: int) => "Integer too large: " + Str.OfInt(i) case StringTooLong(s: string) => "String too long: " + s + case InvalidUnicode => "Invalid Unicode sequence" } } diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index b0d65933..c1f364ec 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -12,7 +12,6 @@ include "../Math.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" -include "Utils/Unicode.dfy" include "Errors.dfy" include "AST.dfy" include "Grammar.dfy" @@ -24,7 +23,6 @@ module {:options "-functionSyntax:4"} JSON.Serializer { import opened Wrappers import opened BoundedInts import opened Utils.Str - import Utils.Unicode import AST import Spec @@ -43,37 +41,12 @@ module {:options "-functionSyntax:4"} JSON.Serializer { View.OfBytes(if b then TRUE else FALSE) } - function Transcode16To8Escaped(str: string, start: uint32 := 0): bytes { - Unicode.Transcode16To8(Spec.Escape(str)) - } // FIXME speed up using a `by method` - // by method { - // var len := |str| as uint32; - // if len == 0 { - // return []; - // } - // var st := new Vectors.Vector(0, len); - // var c0: uint16 := 0; - // var c1: uint16 := str[0] as uint16; - // var idx: uint32 := 0; - // while idx < len { - // var c0 := c1; - // var c1 := str[idx + 1] as uint16; - // if c0 < 0xD800 || c0 > 0xDBFF { - // Utf8Encode(st, Unicode.Utf16Decode1(c0)); - // idx := idx +1; - // } else { - // Utf8Encode(st, Unicode.Utf16Decode2(c0, c1)); - // idx := idx + 2; - // } - // } - // } - function CheckLength(s: seq, err: SerializationError): Outcome { Need(|s| < TWO_TO_THE_32, err) } function String(str: string): Result { - var bs := Transcode16To8Escaped(str); + var bs :- Spec.EscapeToUTF8(str); :- CheckLength(bs, StringTooLong(str)); Success(Grammar.JString(Grammar.DOUBLEQUOTE, View.OfBytes(bs), Grammar.DOUBLEQUOTE)) } diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 33cd3ff8..f4b06087 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -9,9 +9,12 @@ include "../BoundedInts.dfy" include "../NonlinearArithmetic/Logarithm.dfy" +include "../Collections/Sequences/Seq.dfy" +// TODO: Remove and follow one of the options documented in UnicodeStrings.dfy +include "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" include "AST.dfy" -include "Utils/Unicode.dfy" +include "Errors.dfy" include "Utils/Str.dfy" module {:options "-functionSyntax:4"} JSON.Spec { @@ -19,13 +22,20 @@ module {:options "-functionSyntax:4"} JSON.Spec { import opened Utils.Str import opened AST - import opened Utils.Unicode + import opened Wrappers + import opened Errors + import opened UnicodeStrings import opened Logarithm + import Seq + type bytes = seq + type Result<+T> = SerializationResult - function EscapeUnicode(c: uint16): string { - var s := Str.OfNat(c as nat, 16); + function EscapeUnicode(c: uint16): seq { + var sStr := Str.OfNat(c as nat, 16); + Seq.MembershipImpliesIndexing(c => 0 <= c as int < 128, sStr); + var s := ASCIIToUTF16(sStr); assert |s| <= 4 by { assert c as nat <= 0xFFFF; assert Log(16, c as nat) <= Log(16, 0xFFFF) by { @@ -33,71 +43,101 @@ module {:options "-functionSyntax:4"} JSON.Spec { } assert Log(16, 0xFFFF) == 3 by { reveal Log(); } } - s + seq(4 - |s|, _ => ' ') + s + seq(4 - |s|, _ => ' ' as uint16) } - function Escape(str: string, start: nat := 0): string + function Escape(str: seq, start: nat := 0): seq decreases |str| - start { if start >= |str| then [] else - (match str[start] as uint16 - case 0x22 => "\\\"" // quotation mark - case 0x5C => "\\\\" // reverse solidus - case 0x08 => "\\b" // backspace - case 0x0C => "\\f" // form feed - case 0x0A => "\\n" // line feed - case 0x0D => "\\r" // carriage return - case 0x09 => "\\t" // tab + (match str[start] + case 0x22 => ASCIIToUTF16("\\\"") // quotation mark + case 0x5C => ASCIIToUTF16("\\\\") // reverse solidus + case 0x08 => ASCIIToUTF16("\\b") // backspace + case 0x0C => ASCIIToUTF16("\\f") // form feed + case 0x0A => ASCIIToUTF16("\\n") // line feed + case 0x0D => ASCIIToUTF16("\\r") // carriage return + case 0x09 => ASCIIToUTF16("\\t") // tab case c => - if c < 0x001F then "\\u" + EscapeUnicode(c) - else [str[start]]) - + Escape(str, start + 1) + if c < 0x001F then ASCIIToUTF16("\\u") + EscapeUnicode(c) + else [str[start]]) + + Escape(str, start + 1) + } + + function EscapeToUTF8(str: string, start: nat := 0): Result { + var utf16 :- ToUTF16Checked(str).ToResult'(SerializationError.InvalidUnicode); + var escaped := Escape(utf16); + var utf32 :- FromUTF16Checked(escaped).ToResult'(SerializationError.InvalidUnicode); + ToUTF8Checked(utf32).ToResult'(SerializationError.InvalidUnicode) } - function ToBytes(s: string) : seq - requires forall c: char | c in s :: c as int < 256 + // Can fail due to invalid UTF-16 sequences in a string when --unicode-char is off + function String(str: string): Result { + var inBytes :- EscapeToUTF8(str); + Success(ASCIIToUTF8("\"") + inBytes + ASCIIToUTF8("\"")) + } + + lemma OfIntOnlyASCII(n: int) + ensures + && var s := Str.OfInt(n); + && forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 { - seq(|s|, i requires 0 <= i < |s| => - assert s[i] in s; s[i] as byte) + var s := Str.OfInt(n); + forall i | 0 <= i < |s| ensures 0 <= s[i] as int < 128 { + if i == 0 { + } else { + var isHexDigit := c => c in HEX_DIGITS; + assert CharStrConversion.NumberStr(s, '-', isHexDigit); + assert isHexDigit(s[i]); + } + } } - function String(str: string): bytes { - ToBytes("\"") + Transcode16To8(Escape(str)) + ToBytes("\"") + function IntToBytes(n: int): bytes { + var s := Str.OfInt(n); + OfIntOnlyASCII(n); + ASCIIToUTF8(s) } - function Number(dec: Decimal): bytes { - Transcode16To8(Str.OfInt(dec.n)) + + function Number(dec: Decimal): Result { + Success(IntToBytes(dec.n) + (if dec.e10 == 0 then [] - else ToBytes("e") + Transcode16To8(Str.OfInt(dec.e10))) + else ASCIIToUTF8("e") + IntToBytes(dec.e10))) } - function KeyValue(kv: (string, JSON)): bytes { - String(kv.0) + ToBytes(":") + JSON(kv.1) + function KeyValue(kv: (string, JSON)): Result { + var key :- String(kv.0); + var value :- JSON(kv.1); + Success(key + ASCIIToUTF8(":") + value) } - function Join(sep: bytes, items: seq): bytes { - if |items| == 0 then [] - else if |items| == 1 then items[0] - else items[0] + sep + Join(sep, items[1..]) + function Join(sep: bytes, items: seq>): Result { + if |items| == 0 then + Success([]) + else + var first :- items[0]; + if |items| == 1 then + Success(first) + else + var rest :- Join(sep, items[1..]); + Success(first + sep + rest) } - function Object(obj: seq<(string, JSON)>): bytes { - ToBytes("{") + - Join(ToBytes(","), seq(|obj|, i requires 0 <= i < |obj| => KeyValue(obj[i]))) + - ToBytes("}") + function Object(obj: seq<(string, JSON)>): Result { + var middle :- Join(ASCIIToUTF8(","), seq(|obj|, i requires 0 <= i < |obj| => KeyValue(obj[i]))); + Success(ASCIIToUTF8("{") + middle + ASCIIToUTF8("}")) } - function Array(arr: seq): bytes { - ToBytes("[") + - Join(ToBytes(","), seq(|arr|, i requires 0 <= i < |arr| => JSON(arr[i]))) + - ToBytes("]") + function Array(arr: seq): Result { + var middle :- Join(ASCIIToUTF8(","), seq(|arr|, i requires 0 <= i < |arr| => JSON(arr[i]))); + Success(ASCIIToUTF8("[") + middle + ASCIIToUTF8("]")) } - function JSON(js: JSON): bytes { + function JSON(js: JSON): Result { match js - case Null => ToBytes("null") - case Bool(b) => if b then ToBytes("true") else ToBytes("false") + case Null => Success(ASCIIToUTF8("null")) + case Bool(b) => Success(if b then ASCIIToUTF8("true") else ASCIIToUTF8("false")) case String(str) => String(str) case Number(dec) => Number(dec) case Object(obj) => Object(obj) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 255cfd9c..32044edc 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,6 +1,5 @@ // RUN: %run "%s" -include "Utils/Unicode.dfy" include "Errors.dfy" include "API.dfy" include "ZeroCopy/API.dfy" @@ -8,9 +7,8 @@ include "../Collections/Sequences/Seq.dfy" abstract module JSON.Tests.Wrapper { import Utils.Str - import Utils.Unicode import opened BoundedInts - + import opened UnicodeStrings import opened Errors type JSON @@ -23,15 +21,15 @@ abstract module JSON.Tests.Wrapper { var js :- expect Deserialize(bs); // print indent, "=> ", js, "\n"; var bs' :- expect Serialize(js); - print indent, "=> " + Unicode.Transcode8To16(bs') + "\n"; + print indent, "=> ", FromUTF8Checked(bs'), "\n"; var sbs' :- expect SpecSerialize(js); - print indent, "=> " + Unicode.Transcode8To16(sbs') + "\n"; + print indent, "=> ", FromUTF8Checked(sbs'), "\n"; var js' :- expect Deserialize(bs'); Check(bs, js, bs', sbs', js'); } method TestString(str: string, indent: string) { - var bs := Unicode.Transcode16To8(str); + var bs :- expect ToUTF8Checked(str); TestBytestring(bs, indent); } @@ -92,7 +90,7 @@ module JSON.Tests.AbstractSyntaxWrapper refines Wrapper { } method SpecSerialize(js: JSON) returns (bs: SerializationResult) { - bs := Success(Spec.JSON(js)); + bs := Spec.JSON(js); } method Check(bs: bytes, js: JSON, bs': bytes, sbs': bytes, js': JSON) { diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index a87384ce..e2ea3e97 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -12,9 +12,9 @@ include "ZeroCopy/API.dfy" module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import API - import Utils.Unicode import opened AST import opened Wrappers + import opened UnicodeStrings /// The high-level API works with fairly simple ASTs that contain native Dafny /// strings: @@ -28,14 +28,14 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { /// have syntax for byte strings; in a real application, we would be reading and /// writing raw bytes directly from disk or from the network instead). - var SIMPLE_JS := Unicode.Transcode16To8("[true]"); + var SIMPLE_JS :- expect ToUTF8Checked("[true]"); var SIMPLE_AST := Array([Bool(true)]); expect API.Deserialize(SIMPLE_JS) == Success(SIMPLE_AST); /// Here is a larger object, written using a verbatim string (with `@"`). In /// verbatim strings `""` represents a single double-quote character): - var CITIES_JS := Unicode.Transcode16To8(@"{ + var CITIES_JS :- expect ToUTF8Checked(@"{ ""Cities"": [ { ""Name"": ""Boston"", @@ -87,7 +87,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { /// For more complex object, the generated layout may not be exactly the same; note in particular how the representation of numbers and the whitespace have changed. - var EXPECTED := Unicode.Transcode16To8( + var EXPECTED :- expect ToUTF8Checked( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1},{""Name"":""Rome"",""Founded"":-753,""Population"":2873e3,""Area (km2)"":1285},{""Name"":""Paris"",""Founded"":null,""Population"":2161e3,""Area (km2)"":23835e-1}]}" ); @@ -97,7 +97,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { /// existing buffer or into an array. Below is the smaller example from the /// README, as a sanity check: - var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ + var CITY_JS :- expect ToUTF8Checked(@"{""Cities"": [{ ""Name"": ""Boston"", ""Founded"": 1630, ""Population"": 689386, @@ -113,7 +113,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { expect API.Deserialize(CITY_JS) == Success(CITY_AST); - var EXPECTED' := Unicode.Transcode16To8( + var EXPECTED' :- expect ToUTF8Checked( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" ); @@ -128,7 +128,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { import ZeroCopy.API - import Utils.Unicode + import opened UnicodeStrings import opened Grammar import opened Wrappers @@ -145,7 +145,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// all formatting information, re-serializing an object produces the original /// value: - var CITIES := Unicode.Transcode16To8(@"{ + var CITIES :- expect ToUTF8Checked(@"{ ""Cities"": [ { ""Name"": ""Boston"", @@ -176,8 +176,8 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// First, we construct a JSON value for the string `"Unknown"`; this could be /// done by hand using `View.OfBytes()`, but using `API.Deserialize` is even /// simpler: - - var UNKNOWN :- expect API.Deserialize(Unicode.Transcode16To8(@"""Unknown""")); + var UNKNOWN_JS :- expect ToUTF8Checked(@"""Unknown"""); + var UNKNOWN :- expect API.Deserialize(UNKNOWN_JS); /// `UNKNOWN` is of type `Grammar.JSON`, which contains optional whitespace and /// a `Grammar.Value` under the name `UNKNOWN.t`, which we can use in the @@ -188,7 +188,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// Then, if we reserialize, we see that all formatting (and, in fact, all of /// the serialization work) has been reused: - expect API.Serialize(without_null) == Success(Unicode.Transcode16To8(@"{ + var expected_js :- expect ToUTF8Checked(@"{ ""Cities"": [ { ""Name"": ""Boston"", @@ -207,7 +207,9 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { ""Area (km2)"": 2383.5 } ] - }")); + }"); + var actual_js :- expect API.Serialize(without_null); + expect actual_js == expected_js; } /// All that remains is to write the recursive traversal: diff --git a/src/JSON/Utils/Unicode.dfy b/src/JSON/Utils/Unicode.dfy deleted file mode 100644 index 872d85e3..00000000 --- a/src/JSON/Utils/Unicode.dfy +++ /dev/null @@ -1,160 +0,0 @@ -// RUN: %verify "%s" - -include "../../BoundedInts.dfy" - -// TODO: This module was written before Dafny got a Unicode library. It would -// be better to combine the two, especially given that the Unicode library has -// proofs! - -module {:options "-functionSyntax:4"} JSON.Utils.Unicode { - import opened BoundedInts - - function Utf16Decode1(c: uint16): uint32 - requires c < 0xD800 || 0xDBFF < c - { - c as uint32 - } - - function Utf16Decode2(c0: uint16, c1: uint16): uint32 - requires 0xD800 <= c0 as int <= 0xDBFF - { - (0x10000 - + ((((c0 as bv32) & 0x03FF) << 10) - | ((c1 as bv32) & 0x03FF))) - as uint32 - } - - newtype opt_uint16 = c: int | -1 <= c as int < TWO_TO_THE_16 - - function Utf16DecodeChars(c0: uint16, c1: opt_uint16): (r: (uint32, uint8)) - ensures r.1 in {1, 2} - ensures c1 == -1 ==> r.1 == 1 - { - if c0 < 0xD800 || 0xDBFF < c0 then - (Utf16Decode1(c0), 1) - else if c1 >= 0 then - (Utf16Decode2(c0, c1 as uint16), 2) - else - (0xFFFD, 1) // Replacement character - } - - function Utf16Decode(str: string, start: nat := 0): seq - decreases |str| - start - { - if start >= |str| then [] - else - var c0 := str[start] as uint16; - var c1 := if |str| > start + 1 then str[start + 1] as opt_uint16 else -1; - var (cp, delta) := Utf16DecodeChars(c0, c1); - [cp] + Utf16Decode(str, start + delta as nat) - } - - function Utf16Encode2(cp: uint32): seq - requires cp < 0x100000 - { - var bv := cp as bv20; - [(0xD800 | (bv >> 10)) as char, - (0xDC00 | (bv & 0x03FF)) as char] - } - - function Utf16Encode1(cp: uint32): seq { - if cp < 0xD800 || 0xDBFF < cp < 0x10000 then - [cp as char] - else if 0x10000 <= cp < 0x110000 then - Utf16Encode2(cp - 0x10000) - else - [] // Invalid: drop // TODO - } - - function Utf16Encode(codepoints: seq, start: nat := 0): seq - decreases |codepoints| - start - { - if start >= |codepoints| then [] - else Utf16Encode1(codepoints[start]) + Utf16Encode(codepoints, start + 1) - } - - function Utf8Encode1(cp: uint32): seq { - var bv := cp as bv32; - if cp < 0x80 then - [cp as uint8] - else if cp < 0x0800 then - [(((bv >> 6) & 0x1F) | 0xC0) as uint8, - ( (bv & 0x3F) | 0x80) as uint8] - else if cp < 0x10000 then - [(((bv >> 12) & 0x0F) | 0xE0) as uint8, - (((bv >> 6) & 0x3F) | 0x80) as uint8, - ( (bv & 0x3F) | 0x80) as uint8] - else if cp < 0x110000 then - [(((bv >> 18) & 0x07) | 0xF0) as uint8, - (((bv >> 12) & 0x3F) | 0x80) as uint8, - (((bv >> 6) & 0x3F) | 0x80) as uint8, - ( (bv & 0x3F) | 0x80) as uint8] - else - [] // Invalid: drop // TODO - } - - newtype opt_uint8 = c: int | -1 <= c as int < TWO_TO_THE_8 - - function Utf8DecodeChars(c0: uint8, c1: opt_uint8, c2: opt_uint8, c3: opt_uint8): (r: (uint32, uint8)) - ensures r.1 in {1, 2, 3, 4} - ensures c1 == -1 ==> r.1 <= 1 - ensures c2 == -1 ==> r.1 <= 2 - ensures c3 == -1 ==> r.1 <= 3 - { - if (c0 as bv32 & 0x80) == 0 then - (c0 as uint32, 1) - else if (c0 as bv32 & 0xE0) == 0xC0 && c1 > -1 then - (( (((c0 as bv32) & 0x1F) << 6) - | ((c1 as bv32) & 0x3F )) as uint32, - 2) - else if (c0 as bv32 & 0xF0) == 0xE0 && c1 > -1 && c2 > -1 then - (( (((c0 as bv32) & 0x0F) << 12) - | (((c1 as bv32) & 0x3F) << 6) - | ( (c2 as bv32) & 0x3F )) as uint32, - 3) - else if (c0 as bv32 & 0xF8) == 0xF0 && c1 > -1 && c2 > -1 && c3 > -1 then - (( (((c0 as bv32) & 0x07) << 18) - | (((c1 as bv32) & 0x3F) << 12) - | (((c2 as bv32) & 0x3F) << 6) - | ( (c3 as bv32) & 0x3F )) as uint32, - 4) - else - (0xFFFD, 1) // Replacement character - } - - function Utf8Decode(str: seq, start: nat := 0): seq - decreases |str| - start - { - if start >= |str| then [] - else - var c0 := str[start] as uint8; - var c1 := if |str| > start + 1 then str[start + 1] as opt_uint8 else -1; - var c2 := if |str| > start + 2 then str[start + 2] as opt_uint8 else -1; - var c3 := if |str| > start + 3 then str[start + 3] as opt_uint8 else -1; - var (cp, delta) := Utf8DecodeChars(c0, c1, c2, c3); - [cp] + Utf8Decode(str, start + delta as nat) - } - - function Utf8Encode(codepoints: seq, start: nat := 0): seq - decreases |codepoints| - start - { - if start >= |codepoints| then [] - else Utf8Encode1(codepoints[start]) + Utf8Encode(codepoints, start + 1) - } - - function Transcode16To8(s: string): seq { - Utf8Encode(Utf16Decode(s)) - } - - function Transcode8To16(s: seq): string { - Utf16Encode(Utf8Decode(s)) - } - - function ASCIIToBytes(s: string): seq - // Keep ASCII characters in `s` and discard all other characters - { - seq(|s|, idx requires 0 <= idx < |s| => - if s[idx] as uint16 < 128 then s[idx] as uint8 - else 0 as uint8) - } -} diff --git a/src/Unicode/UnicodeEncodingForm.dfy b/src/Unicode/UnicodeEncodingForm.dfy index d7f8a463..854ed1fb 100644 --- a/src/Unicode/UnicodeEncodingForm.dfy +++ b/src/Unicode/UnicodeEncodingForm.dfy @@ -239,6 +239,23 @@ abstract module {:options "-functionSyntax:4"} UnicodeEncodingForm { LemmaFlattenMinimalWellFormedCodeUnitSubsequences(ms); Seq.Flatten(ms) } + by method { + // Optimize to to avoid allocating the intermediate unflattened sequence. + // We can't quite use Seq.FlatMap easily because we need to prove the result + // is not just a seq but a WellFormedCodeUnitSeq. + // TODO: We can be even more efficient by using a JSON.Utils.Vectors.Vector instead. + s := []; + ghost var unflattened: seq := []; + for i := |vs| downto 0 + invariant unflattened == Seq.Map(EncodeScalarValue, vs[i..]) + invariant s == Seq.Flatten(unflattened) + { + var next: MinimalWellFormedCodeUnitSeq := EncodeScalarValue(vs[i]); + unflattened := [next] + unflattened; + LemmaPrependMinimalWellFormedCodeUnitSubsequence(next, s); + s := next + s; + } + } /** * Returns the scalar value sequence encoded by the given well-formed Unicode string. diff --git a/src/Unicode/UnicodeStrings.dfy b/src/Unicode/UnicodeStrings.dfy new file mode 100644 index 00000000..c2e2cf5d --- /dev/null +++ b/src/Unicode/UnicodeStrings.dfy @@ -0,0 +1,62 @@ +// RUN: %verify %s + +/// Converting between strings and UTF-8/UTF-16 +/// ============================================= +/// +/// This abstract module defines an interface for converting +/// between the Dafny `string` type and sequences of UTF-8 and UTF-16 +/// codepoints, where codepoints are represents as bounded ints +/// (`uint8` and `uint16`). +/// +/// Because the `--unicode-char` option changes the meaning of the `char` +/// type and hence the `string` type, there are two different concrete +/// implementations in separate files. +/// Only include the one that matches your value of that option! +/// +/// If you also want to maintain code that works for either mode, +/// you have two options: +/// +/// 1. Implement your logic in an abstract module as well that +/// imports this one, and define two different refining modules +/// that import the appropriate UnicodeStrings module. +/// See (TODO example) for an example. +/// 2. Do not `include` any of these three files in your source code, +/// instead passing the appropriate file to Dafny when verifying and building, +/// so that references to `UnicodeStrings` can be resolved. +/// +/// Option 2. avoids needing to write boilerplate refining modules, +/// but is less IDE-friendly until we have better project configuration support. + +include "../BoundedInts.dfy" +include "../Wrappers.dfy" +include "../Collections/Sequences/Seq.dfy" +include "Utf8EncodingForm.dfy" +include "Utf16EncodingForm.dfy" + +abstract module {:options "-functionSyntax:4"} AbstractUnicodeStrings { + + import Seq + + import opened Wrappers + import opened BoundedInts + + function ToUTF8Checked(s: string): Option> + + function ASCIIToUTF8(s: string): seq + requires forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 + { + Seq.Map(c requires 0 <= c as int < 128 => c as uint8, s) + } + + function FromUTF8Checked(bs: seq): Option + + function ToUTF16Checked(s: string): Option> + + function ASCIIToUTF16(s: string): seq + requires forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 + { + Seq.Map(c requires 0 <= c as int < 128 => c as uint16, s) + } + + function FromUTF16Checked(bs: seq): Option +} \ No newline at end of file diff --git a/src/Unicode/UnicodeStringsWithUnicodeChar.dfy b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy new file mode 100644 index 00000000..3f9d1259 --- /dev/null +++ b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy @@ -0,0 +1,85 @@ +// RUN: %verify --unicode-char:true %s + +/// Converting between strings and UTF-8/UTF-16 +/// ============================================= +/// +/// Implementation of `AbstractUnicodeStrings` for `--unicode-char:true`. +/// See `UnicodeStrings.dfy` for details. + +include "UnicodeStrings.dfy" +include "../Wrappers.dfy" +include "../Collections/Sequences/Seq.dfy" + +module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStrings { + + import Unicode + import Utf8EncodingForm + import Utf16EncodingForm + + lemma {:vcs_split_on_every_assert} CharIsUnicodeScalarValue(c: char) + ensures + && var asBits := c as bv24; + && asBits <= 0x10_FFFF + && (0 <= asBits < Unicode.HIGH_SURROGATE_MIN || Unicode.LOW_SURROGATE_MAX < asBits) + { + assert c as int < 0x11_0000; + // TODO: Doesn't verify and not sure what else to try + assert c as int as bv24 < 0x11_0000 as bv24; + var asBits := c as int as bv24; + assert (asBits < Unicode.HIGH_SURROGATE_MIN || asBits > Unicode.LOW_SURROGATE_MAX); + assert asBits <= 0x10_FFFF; + } + + lemma UnicodeScalarValueIsChar(sv: Unicode.ScalarValue) + ensures + && var asInt := sv as int; + && (0 <= asInt < 0xD800 || 0xE000 <= asInt < 0x11_0000) + { + var asInt := sv as int; + assert (asInt < 0xD800 || asInt > 0xDFFF); + assert (asInt < 0xDBFF || asInt > 0xDC00); + } + + function CharAsUnicodeScalarValue(c: char): Unicode.ScalarValue { + CharIsUnicodeScalarValue(c); + c as Unicode.ScalarValue + } + + function CharFromUnicodeScalarValue(sv: Unicode.ScalarValue): char { + UnicodeScalarValueIsChar(sv); + // TODO: Can we avoid the extra cast to int? + sv as int as char + } + + function ToUTF8Checked(s: string): Option> + ensures ToUTF8Checked(s).Some? + { + var asCodeUnits := Seq.Map(CharAsUnicodeScalarValue, s); + var asUtf8CodeUnits := Utf8EncodingForm.EncodeScalarSequence(asCodeUnits); + var asBytes := Seq.Map(cu => cu as uint8, asUtf8CodeUnits); + Some(asBytes) + } + + function FromUTF8Checked(bs: seq): Option { + var asCodeUnits := Seq.Map(c => c as Utf8EncodingForm.CodeUnit, bs); + var utf32 :- Utf8EncodingForm.DecodeCodeUnitSequenceChecked(asCodeUnits); + var asChars := Seq.Map(CharFromUnicodeScalarValue, utf32); + Some(asChars) + } + + function ToUTF16Checked(s: string): Option> + ensures ToUTF16Checked(s).Some? + { + var asCodeUnits := Seq.Map(CharAsUnicodeScalarValue, s); + var asUtf16CodeUnits := Utf16EncodingForm.EncodeScalarSequence(asCodeUnits); + var asBytes := Seq.Map(cu => cu as uint16, asUtf16CodeUnits); + Some(asBytes) + } + + function FromUTF16Checked(bs: seq): Option { + var asCodeUnits := Seq.Map(c => c as Utf16EncodingForm.CodeUnit, bs); + var utf32 :- Utf16EncodingForm.DecodeCodeUnitSequenceChecked(asCodeUnits); + var asChars := Seq.Map(CharFromUnicodeScalarValue, utf32); + Some(asChars) + } +} \ No newline at end of file diff --git a/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy b/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy new file mode 100644 index 00000000..2a095d25 --- /dev/null +++ b/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy @@ -0,0 +1,52 @@ +// RUN: %verify --unicode-char:false %s + +/// Converting between strings and UTF-8/UTF-16 +/// ============================================= +/// +/// Implementation of `AbstractUnicodeStrings` for `--unicode-char:false`. +/// See `UnicodeStrings.dfy` for details. + +include "UnicodeStrings.dfy" +include "../Wrappers.dfy" +include "../Collections/Sequences/Seq.dfy" + +module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStrings { + + import Unicode + import Utf8EncodingForm + import Utf16EncodingForm + + predicate IsWellFormedString(s: string) + ensures |s| == 0 ==> IsWellFormedString(s) + { + Utf16EncodingForm.IsWellFormedCodeUnitSequence(Seq.Map(c => c as Utf16EncodingForm.CodeUnit, s)) + } + + function ToUTF8Checked(s: string): Option> { + var asCodeUnits := Seq.Map(c => c as Utf16EncodingForm.CodeUnit, s); + var utf32 :- Utf16EncodingForm.DecodeCodeUnitSequenceChecked(asCodeUnits); + var asUtf8CodeUnits := Utf8EncodingForm.EncodeScalarSequence(utf32); + Some(Seq.Map(c => c as byte, asUtf8CodeUnits)) + } + + function {:vcs_split_on_every_assert} FromUTF8Checked(bs: seq): Option { + var asCodeUnits := Seq.Map(c => c as Utf8EncodingForm.CodeUnit, bs); + var utf32 :- Utf8EncodingForm.DecodeCodeUnitSequenceChecked(asCodeUnits); + var asUtf16CodeUnits := Utf16EncodingForm.EncodeScalarSequence(utf32); + Some(Seq.Map(cu => cu as char, asUtf16CodeUnits)) + } + + function ToUTF16Checked(s: string): Option> { + if Utf16EncodingForm.IsWellFormedCodeUnitSequence(Seq.Map(c => c as Utf16EncodingForm.CodeUnit, s)) then + Some(Seq.Map(c => c as uint16, s)) + else + None + } + + function FromUTF16Checked(bs: seq): Option { + if Utf16EncodingForm.IsWellFormedCodeUnitSequence(Seq.Map(c => c as Utf16EncodingForm.CodeUnit, bs)) then + Some(Seq.Map(c => c as char, bs)) + else + None + } +} \ No newline at end of file diff --git a/src/Wrappers.dfy b/src/Wrappers.dfy index 852d9a8c..167cb574 100644 --- a/src/Wrappers.dfy +++ b/src/Wrappers.dfy @@ -14,6 +14,12 @@ module {:options "-functionSyntax:4"} Wrappers { case None() => Failure("Option is None") } + function ToResult'(error: R): Result { + match this + case Some(v) => Success(v) + case None() => Failure(error) + } + function UnwrapOr(default: T): T { match this case Some(v) => v From 2029a96ee049225060fb677117bdf07b9b1bd7d3 Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 20 Apr 2023 00:47:43 +0200 Subject: [PATCH 51/84] format --- src/Collections/Sequences/Seq.dfy | 6 ++--- src/JSON/Deserializer.dfy | 2 +- src/JSON/Spec.dfy | 22 +++++++++---------- src/Unicode/UnicodeEncodingForm.dfy | 2 +- src/Unicode/UnicodeStrings.dfy | 8 +++---- src/Unicode/UnicodeStringsWithUnicodeChar.dfy | 6 ++--- .../UnicodeStringsWithoutUnicodeChar.dfy | 10 ++++----- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/Collections/Sequences/Seq.dfy b/src/Collections/Sequences/Seq.dfy index 0a04b969..645864ec 100644 --- a/src/Collections/Sequences/Seq.dfy +++ b/src/Collections/Sequences/Seq.dfy @@ -627,8 +627,8 @@ module {:options "-functionSyntax:4"} Seq { // more efficient when compiled, allocating the storage for |xs| elements // once instead of creating a chain of |xs| single element concatenations. seq(|xs|, i requires 0 <= i < |xs| && f.requires(xs[i]) - reads set i,o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o - => f(xs[i])) + reads set i,o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o + => f(xs[i])) } /* Applies a function to every element of a sequence, returning a Result value (which is a @@ -838,7 +838,7 @@ module {:options "-functionSyntax:4"} Seq { by method { result := []; ghost var unflattened: seq> := []; - for i := |xs| downto 0 + for i := |xs| downto 0 invariant unflattened == Map(f, xs[i..]) invariant result == Flatten(unflattened) { diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 32155a4b..77798779 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -51,7 +51,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { const HEX_TABLE_16: map := map[ '0' as uint16 := 0, '1' as uint16 := 1, '2' as uint16 := 2, '3' as uint16 := 3, '4' as uint16 := 4, - '5' as uint16 := 5, '6' as uint16 := 6, '7' as uint16 := 7, '8' as uint16 := 8, '9' as uint16 := 9, + '5' as uint16 := 5, '6' as uint16 := 6, '7' as uint16 := 7, '8' as uint16 := 8, '9' as uint16 := 9, 'a' as uint16 := 0xA, 'b' as uint16 := 0xB, 'c' as uint16 := 0xC, 'd' as uint16 := 0xD, 'e' as uint16 := 0xE, 'f' as uint16 := 0xF, 'A' as uint16 := 0xA, 'B' as uint16 := 0xB, 'C' as uint16 := 0xC, 'D' as uint16 := 0xD, 'E' as uint16 := 0xE, 'F' as uint16 := 0xF ] diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index f4b06087..fc03b59d 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -10,7 +10,7 @@ include "../BoundedInts.dfy" include "../NonlinearArithmetic/Logarithm.dfy" include "../Collections/Sequences/Seq.dfy" -// TODO: Remove and follow one of the options documented in UnicodeStrings.dfy + // TODO: Remove and follow one of the options documented in UnicodeStrings.dfy include "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" include "AST.dfy" @@ -60,9 +60,9 @@ module {:options "-functionSyntax:4"} JSON.Spec { case 0x0D => ASCIIToUTF16("\\r") // carriage return case 0x09 => ASCIIToUTF16("\\t") // tab case c => - if c < 0x001F then ASCIIToUTF16("\\u") + EscapeUnicode(c) - else [str[start]]) - + Escape(str, start + 1) + if c < 0x001F then ASCIIToUTF16("\\u") + EscapeUnicode(c) + else [str[start]]) + + Escape(str, start + 1) } function EscapeToUTF8(str: string, start: nat := 0): Result { @@ -79,7 +79,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { } lemma OfIntOnlyASCII(n: int) - ensures + ensures && var s := Str.OfInt(n); && forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 { @@ -102,8 +102,8 @@ module {:options "-functionSyntax:4"} JSON.Spec { function Number(dec: Decimal): Result { Success(IntToBytes(dec.n) + - (if dec.e10 == 0 then [] - else ASCIIToUTF8("e") + IntToBytes(dec.e10))) + (if dec.e10 == 0 then [] + else ASCIIToUTF8("e") + IntToBytes(dec.e10))) } function KeyValue(kv: (string, JSON)): Result { @@ -113,13 +113,13 @@ module {:options "-functionSyntax:4"} JSON.Spec { } function Join(sep: bytes, items: seq>): Result { - if |items| == 0 then + if |items| == 0 then Success([]) - else + else var first :- items[0]; - if |items| == 1 then + if |items| == 1 then Success(first) - else + else var rest :- Join(sep, items[1..]); Success(first + sep + rest) } diff --git a/src/Unicode/UnicodeEncodingForm.dfy b/src/Unicode/UnicodeEncodingForm.dfy index 854ed1fb..8119b0e0 100644 --- a/src/Unicode/UnicodeEncodingForm.dfy +++ b/src/Unicode/UnicodeEncodingForm.dfy @@ -246,7 +246,7 @@ abstract module {:options "-functionSyntax:4"} UnicodeEncodingForm { // TODO: We can be even more efficient by using a JSON.Utils.Vectors.Vector instead. s := []; ghost var unflattened: seq := []; - for i := |vs| downto 0 + for i := |vs| downto 0 invariant unflattened == Seq.Map(EncodeScalarValue, vs[i..]) invariant s == Seq.Flatten(unflattened) { diff --git a/src/Unicode/UnicodeStrings.dfy b/src/Unicode/UnicodeStrings.dfy index c2e2cf5d..524b4ce7 100644 --- a/src/Unicode/UnicodeStrings.dfy +++ b/src/Unicode/UnicodeStrings.dfy @@ -25,7 +25,7 @@ /// so that references to `UnicodeStrings` can be resolved. /// /// Option 2. avoids needing to write boilerplate refining modules, -/// but is less IDE-friendly until we have better project configuration support. +/// but is less IDE-friendly until we have better project configuration support. include "../BoundedInts.dfy" include "../Wrappers.dfy" @@ -42,7 +42,7 @@ abstract module {:options "-functionSyntax:4"} AbstractUnicodeStrings { function ToUTF8Checked(s: string): Option> - function ASCIIToUTF8(s: string): seq + function ASCIIToUTF8(s: string): seq requires forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 { Seq.Map(c requires 0 <= c as int < 128 => c as uint8, s) @@ -52,11 +52,11 @@ abstract module {:options "-functionSyntax:4"} AbstractUnicodeStrings { function ToUTF16Checked(s: string): Option> - function ASCIIToUTF16(s: string): seq + function ASCIIToUTF16(s: string): seq requires forall i | 0 <= i < |s| :: 0 <= s[i] as int < 128 { Seq.Map(c requires 0 <= c as int < 128 => c as uint16, s) } function FromUTF16Checked(bs: seq): Option -} \ No newline at end of file +} diff --git a/src/Unicode/UnicodeStringsWithUnicodeChar.dfy b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy index 3f9d1259..7744839d 100644 --- a/src/Unicode/UnicodeStringsWithUnicodeChar.dfy +++ b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy @@ -14,7 +14,7 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri import Unicode import Utf8EncodingForm - import Utf16EncodingForm + import Utf16EncodingForm lemma {:vcs_split_on_every_assert} CharIsUnicodeScalarValue(c: char) ensures @@ -31,7 +31,7 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri } lemma UnicodeScalarValueIsChar(sv: Unicode.ScalarValue) - ensures + ensures && var asInt := sv as int; && (0 <= asInt < 0xD800 || 0xE000 <= asInt < 0x11_0000) { @@ -82,4 +82,4 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri var asChars := Seq.Map(CharFromUnicodeScalarValue, utf32); Some(asChars) } -} \ No newline at end of file +} diff --git a/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy b/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy index 2a095d25..9c5b5f59 100644 --- a/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy +++ b/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy @@ -14,12 +14,12 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri import Unicode import Utf8EncodingForm - import Utf16EncodingForm - + import Utf16EncodingForm + predicate IsWellFormedString(s: string) - ensures |s| == 0 ==> IsWellFormedString(s) + ensures |s| == 0 ==> IsWellFormedString(s) { - Utf16EncodingForm.IsWellFormedCodeUnitSequence(Seq.Map(c => c as Utf16EncodingForm.CodeUnit, s)) + Utf16EncodingForm.IsWellFormedCodeUnitSequence(Seq.Map(c => c as Utf16EncodingForm.CodeUnit, s)) } function ToUTF8Checked(s: string): Option> { @@ -49,4 +49,4 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri else None } -} \ No newline at end of file +} From c984b22dab7830243c8209ff9e61e26701836fbb Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Thu, 20 Apr 2023 00:58:28 +0200 Subject: [PATCH 52/84] docs --- src/JSON/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/JSON/README.md b/src/JSON/README.md index bedd0d7d..390cae37 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -19,12 +19,12 @@ The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, include "src/JSON/API.dfy" import JSON.API -import JSON.Utils.Unicode +import opened UnicodeStrings import opened JSON.AST import opened Wrappers method Test(){ - var CITY_JS := Unicode.Transcode16To8(@"{""Cities"": [{ + var CITY_JS :- expect ToUTF8Checked(@"{""Cities"": [{ ""Name"": ""Boston"", ""Founded"": 1630, ""Population"": 689386, @@ -39,9 +39,11 @@ method Test(){ expect API.Deserialize(CITY_JS) == Success(CITY_AST); - expect API.Serialize(CITY_AST) == Success(Unicode.Transcode16To8( + var EXPECTED :- expect ToUTF8Checked( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" - )); + ); + + expect API.Serialize(CITY_AST) == Success(EXPECTED); } ``` From 99d189b7498670d83a8a5911fdea96c17e0eeb67 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 20 Apr 2023 09:55:03 -0700 Subject: [PATCH 53/84] Axiomatize trivially true but expensive assertion --- src/Unicode/UnicodeStringsWithUnicodeChar.dfy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Unicode/UnicodeStringsWithUnicodeChar.dfy b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy index 7744839d..31a418e2 100644 --- a/src/Unicode/UnicodeStringsWithUnicodeChar.dfy +++ b/src/Unicode/UnicodeStringsWithUnicodeChar.dfy @@ -23,8 +23,9 @@ module {:options "-functionSyntax:4"} UnicodeStrings refines AbstractUnicodeStri && (0 <= asBits < Unicode.HIGH_SURROGATE_MIN || Unicode.LOW_SURROGATE_MAX < asBits) { assert c as int < 0x11_0000; - // TODO: Doesn't verify and not sure what else to try - assert c as int as bv24 < 0x11_0000 as bv24; + // This seems to be just too expensive to verify for such a wide bit-vector type, + // but is clearly true given the above. + assume {:axiom} c as bv24 < 0x11_0000 as bv24; var asBits := c as int as bv24; assert (asBits < Unicode.HIGH_SURROGATE_MIN || asBits > Unicode.LOW_SURROGATE_MAX); assert asBits <= 0x10_FFFF; From dc50674c5a7c65e891157df449f257812da33025 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 20 Apr 2023 10:37:34 -0700 Subject: [PATCH 54/84] =?UTF-8?q?Refactor=20-=20resolves=20but=20doesn?= =?UTF-8?q?=E2=80=99t=20verify=20yet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JSON/Utils/Parsers.dfy | 10 +++++----- src/JSON/ZeroCopy/Deserializer.dfy | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 8ae42ae2..c195fc62 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -13,9 +13,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> - type Parser = p: Parser_ | p.Valid?() - // BUG(https://github.com/dafny-lang/dafny/issues/2103) - witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + // type Parser = p: Parser_ | p.Valid?() + // // BUG(https://github.com/dafny-lang/dafny/issues/2103) + // witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, ghost spec: T -> bytes) { ghost predicate Valid?() { @@ -48,8 +48,8 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { && (forall cs': FreshCursor | pre(cs') :: fn(cs').Success? ==> fn(cs').value.StrictlySplitFrom?(cs', spec)) } } - type SubParser = p: SubParser_ | p.Valid?() - witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + // type SubParser = p: SubParser_ | p.Valid?() + // witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) function {:opaque} SubParserWitness(): (subp: SubParser_) ensures subp.Valid?() diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 42e2b39c..958a7f35 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -24,8 +24,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type JSONError = Errors.DeserializationError type Error = CursorError type ParseResult<+T> = SplitResult - type Parser = Parsers.Parser - type SubParser = Parsers.SubParser + type Parser_ = Parsers.Parser_ + type SubParser_ = Parsers.SubParser_ // BUG(https://github.com/dafny-lang/dafny/issues/2179) const SpecView := (v: Vs.View) => Spec.View(v); @@ -54,8 +54,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { return Cursor(cs.s, cs.beg, point', cs.end).Split(); } - function {:opaque} Structural(cs: FreshCursor, parser: Parser) + function {:opaque} Structural(cs: FreshCursor, parser: Parser_) : (pr: ParseResult>) + requires parser.Valid?() requires forall cs :: parser.fn.requires(cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) { @@ -77,8 +78,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { SP(Grammar.Structural(before, val, after), cs) } - type ValueParser = sp: SubParser | - forall t :: sp.spec(t) == Spec.Value(t) + type ValueParser = sp: SubParser_ | + sp.Valid?() && forall t :: sp.spec(t) == Spec.Value(t) witness * } type Error = Core.Error @@ -151,7 +152,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Success(cs.Split()) } - function {:opaque} BracketedFromParts(ghost cs: Cursor, + function {:opaque} {:vcs_split_on_every_assertion} BracketedFromParts(ghost cs: Cursor, open: Split>, elems: Split>, close: Split>) From 0a8acf78530d267fb90b627ff8d8a408869c427e Mon Sep 17 00:00:00 2001 From: Fabio Madge Date: Sat, 22 Apr 2023 01:09:19 +0200 Subject: [PATCH 55/84] really fix the deserializer --- src/JSON/ZeroCopy/Deserializer.dfy | 31 +++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 42e2b39c..8b48099b 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -196,7 +196,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { var elems' := SP(elems.t + [suffixed], sep.cs); // DISCUSS: Moving this down doubles the verification time assert cs0.Bytes() == SuffixedElementsSpec(elems'.t) + sep.cs.Bytes() by { - assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + (ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix)) + sep.cs.Bytes() by { + assert {:focus} cs0.Bytes() == SuffixedElementsSpec(elems.t) + (ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix)) + sep.cs.Bytes() by { assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + ElementSpec(suffixed.t) + Spec.CommaSuffix(suffixed.suffix) + sep.cs.Bytes() by { assert cs0.Bytes() == SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); assert elems.cs.Bytes() == ElementSpec(suffixed.t) + elem.cs.Bytes(); @@ -213,9 +213,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } } } - assert elems'.StrictlySplitFrom?(cs0, SuffixedElementsSpec); - assert forall e | e in elems'.t :: e.suffix.NonEmpty?; + assert forall e | e in elems'.t :: e.suffix.NonEmpty? by { assert elems'.t == elems.t + [suffixed]; } + assert {:split_here} elems'.cs.Length() < elems.cs.Length(); elems' } @@ -275,21 +275,33 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { var sep := Core.TryStructural(elem.cs); var s0 := sep.t.t.Peek(); if s0 == SEPARATOR as opt_byte then - assert AppendWithSuffix.requires(open.cs, json, elems, elem, sep); + assert AppendWithSuffix.requires(open.cs, json, elems, elem, sep) by { + assert {:focus} elems.cs.StrictlySplitFrom?(json.cs); + assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); + assert elem.StrictlySplitFrom?(elems.cs, ElementSpec); + assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); + assert forall e | e in elems.t :: e.suffix.NonEmpty?; + assert {:split_here} true; + } var elems := AppendWithSuffix(open.cs, json, elems, elem, sep); Elements(cs0, json, open, elems) else if s0 == CLOSE as opt_byte then assert AppendLast.requires(open.cs, json, elems, elem, sep) by { - assert sep.SplitFrom?(elem.cs, st => Spec.Structural(st, SpecView)); assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); + assert elems.cs.StrictlySplitFrom?(json.cs); + assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); + assert elem.StrictlySplitFrom?(elems.cs, ElementSpec); + } + var elems' := AppendLast(open.cs, json, elems, elem, sep); + assert elems'.SplitFrom?(open.cs, SuffixedElementsSpec) by { + assert elems'.StrictlySplitFrom?(open.cs, SuffixedElementsSpec); } - var elems := AppendLast(open.cs, json, elems, elem, sep); - assert BracketedFromParts.requires(cs0, open, elems, sep); - var bracketed := BracketedFromParts(cs0, open, elems, sep); + var bracketed := BracketedFromParts(cs0, open, elems', sep); assert bracketed.StrictlySplitFrom?(cs0, BracketedSpec); Success(bracketed) else - var pr := Failure(ExpectingAnyByte([CLOSE, SEPARATOR], s0)); + var separator := SEPARATOR; + var pr := Failure(ExpectingAnyByte([CLOSE, separator], s0)); pr } @@ -430,6 +442,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { assert SP(num, cs').StrictlySplitFrom?(cs, Spec.Number); Spec.UnfoldValueNumber(v); } + assert sp.StrictlySplitFrom?(cs, Spec.Value); Success(sp) } From d36231a7f8c9afeb4b8987d9fb3be5f4bd0b16a8 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 25 Apr 2023 20:20:23 -0700 Subject: [PATCH 56/84] =?UTF-8?q?Attempt=20at=20function=20by=20method,=20?= =?UTF-8?q?doesn=E2=80=99t=20verify=20yet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JSON/Deserializer.dfy | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 77798779..0a478144 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -67,11 +67,14 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { } // TODO: Verify this function - function Unescape(str: seq, start: nat := 0): DeserializationResult> + function {:vcs_split_on_every_assert} UnescapeOne(str: seq, start: nat := 0): (res: DeserializationResult<(uint16, nat)>) + requires start < |str| decreases |str| - start + ensures res.Success? ==> + var (_, next) := res.value; + start < next <= |str| { // Assumes UTF-16 strings - if start >= |str| then Success([]) - else if str[start] == '\\' as uint16 then + if str[start] == '\\' as uint16 then if |str| == start + 1 then Failure(EscapeAtEOS) else @@ -84,9 +87,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if exists c | c in code :: c !in HEX_TABLE_16 then Failure(UnsupportedEscape16(code)) else - var tl :- Unescape(str, start + 6); var hd := ToNat16(code); - Success([hd]) + Success((hd, start + 6)) else var unescaped: uint16 := match c case 0x22 => 0x22 as uint16 // \" => quotation mark @@ -100,11 +102,34 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if unescaped as int == 0 then Failure(UnsupportedEscape16(str[start..start+2])) else - var tl :- Unescape(str, start + 2); - Success([unescaped] + tl) + Success((unescaped, start + 2)) else - var tl :- Unescape(str, start + 1); - Success([str[start]] + tl) + Success((str[start], start + 1)) + } + + function {:vcs_split_on_every_assert} Unescape(str: seq, start: nat := 0): (res: DeserializationResult>) + decreases |str| - start + { + if start >= |str| then + Success([]) + else + var (hd, next) :- UnescapeOne(str, start); + var tl :- Unescape(str, next); + Success([hd] + tl) + } + by method { + var result := []; + var next := start; + while next < |str| + decreases |str| - next + { + var nextPair :- UnescapeOne(str, next); + var (chunk, newNext) := nextPair; + + result := result + [chunk]; + next := newNext; + } + return Success(result); } function String(js: Grammar.jstring): DeserializationResult { From 43e51b149495d5491380540cd8aac4fd9793caaa Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 25 Apr 2023 20:20:50 -0700 Subject: [PATCH 57/84] =?UTF-8?q?Revert=20"Attempt=20at=20function=20by=20?= =?UTF-8?q?method,=20doesn=E2=80=99t=20verify=20yet"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d36231a7f8c9afeb4b8987d9fb3be5f4bd0b16a8. --- src/JSON/Deserializer.dfy | 43 ++++++++------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 0a478144..77798779 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -67,14 +67,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { } // TODO: Verify this function - function {:vcs_split_on_every_assert} UnescapeOne(str: seq, start: nat := 0): (res: DeserializationResult<(uint16, nat)>) - requires start < |str| + function Unescape(str: seq, start: nat := 0): DeserializationResult> decreases |str| - start - ensures res.Success? ==> - var (_, next) := res.value; - start < next <= |str| { // Assumes UTF-16 strings - if str[start] == '\\' as uint16 then + if start >= |str| then Success([]) + else if str[start] == '\\' as uint16 then if |str| == start + 1 then Failure(EscapeAtEOS) else @@ -87,8 +84,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if exists c | c in code :: c !in HEX_TABLE_16 then Failure(UnsupportedEscape16(code)) else + var tl :- Unescape(str, start + 6); var hd := ToNat16(code); - Success((hd, start + 6)) + Success([hd]) else var unescaped: uint16 := match c case 0x22 => 0x22 as uint16 // \" => quotation mark @@ -102,34 +100,11 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if unescaped as int == 0 then Failure(UnsupportedEscape16(str[start..start+2])) else - Success((unescaped, start + 2)) + var tl :- Unescape(str, start + 2); + Success([unescaped] + tl) else - Success((str[start], start + 1)) - } - - function {:vcs_split_on_every_assert} Unescape(str: seq, start: nat := 0): (res: DeserializationResult>) - decreases |str| - start - { - if start >= |str| then - Success([]) - else - var (hd, next) :- UnescapeOne(str, start); - var tl :- Unescape(str, next); - Success([hd] + tl) - } - by method { - var result := []; - var next := start; - while next < |str| - decreases |str| - next - { - var nextPair :- UnescapeOne(str, next); - var (chunk, newNext) := nextPair; - - result := result + [chunk]; - next := newNext; - } - return Success(result); + var tl :- Unescape(str, start + 1); + Success([str[start]] + tl) } function String(js: Grammar.jstring): DeserializationResult { From 989c6d98a08a5e205c36ffaad9b5c6342d069d9e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 25 Apr 2023 20:26:39 -0700 Subject: [PATCH 58/84] Simple extra-argument tail recursion --- src/JSON/Deserializer.dfy | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 77798779..0fd1eb86 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -67,7 +67,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { } // TODO: Verify this function - function Unescape(str: seq, start: nat := 0): DeserializationResult> + function {:tailrecursion} {:vcs_split_on_every_assert} Unescape(str: seq, start: nat := 0, prefix: seq := []): DeserializationResult> decreases |str| - start { // Assumes UTF-16 strings if start >= |str| then Success([]) @@ -84,9 +84,8 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if exists c | c in code :: c !in HEX_TABLE_16 then Failure(UnsupportedEscape16(code)) else - var tl :- Unescape(str, start + 6); var hd := ToNat16(code); - Success([hd]) + Unescape(str, start + 6, prefix + [hd]) else var unescaped: uint16 := match c case 0x22 => 0x22 as uint16 // \" => quotation mark @@ -100,11 +99,9 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { if unescaped as int == 0 then Failure(UnsupportedEscape16(str[start..start+2])) else - var tl :- Unescape(str, start + 2); - Success([unescaped] + tl) + Unescape(str, start + 2, prefix + [unescaped]) else - var tl :- Unescape(str, start + 1); - Success([str[start]] + tl) + Unescape(str, start + 1, prefix + [str[start]]) } function String(js: Grammar.jstring): DeserializationResult { From 407432f318ad7b1d4da6ce465a85154d1a41a231 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 26 Apr 2023 14:30:00 -0700 Subject: [PATCH 59/84] Have you even ever coded before Robin --- src/JSON/Deserializer.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 0fd1eb86..5fe25373 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -70,7 +70,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { function {:tailrecursion} {:vcs_split_on_every_assert} Unescape(str: seq, start: nat := 0, prefix: seq := []): DeserializationResult> decreases |str| - start { // Assumes UTF-16 strings - if start >= |str| then Success([]) + if start >= |str| then Success(prefix) else if str[start] == '\\' as uint16 then if |str| == start + 1 then Failure(EscapeAtEOS) From ff5ce0b59f4cab4281184d66a03630de9fbfb2d4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 27 Apr 2023 12:49:25 -0700 Subject: [PATCH 60/84] Bound resource count instead of time Actually a tighter bound anyway --- .github/workflows/reusable-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 37d489b0..7c78aef0 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -44,7 +44,7 @@ jobs: lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv' . - name: Generate Report - run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-duration-seconds 10 + run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-resource-count 40000000 - uses: actions/upload-artifact@v2 # upload test results with: From 0ea991dcc1518d8d3f7398ee82500de99bf011e2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 27 Apr 2023 12:49:25 -0700 Subject: [PATCH 61/84] Bound resource count instead of time Actually a tighter bound anyway --- .github/workflows/reusable-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 37d489b0..7c78aef0 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -44,7 +44,7 @@ jobs: lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv' . - name: Generate Report - run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-duration-seconds 10 + run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-resource-count 40000000 - uses: actions/upload-artifact@v2 # upload test results with: From 99f0662950b77cfa1eb7dbf8f0bc6108ab46a159 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 27 Apr 2023 14:30:44 -0700 Subject: [PATCH 62/84] =?UTF-8?q?Don=E2=80=99t=20include=20UnicodeStrings?= =?UTF-8?q?=20directly,=20augment=20CI=20to=20test=20both=20ways?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/nightly.yml | 2 ++ .github/workflows/reusable-tests.yml | 13 +++++++++++-- .github/workflows/tests.yml | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ff144cfa..72da8bb4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,7 +15,9 @@ jobs: strategy: matrix: version: [ nightly-latest ] + unicode-char: [ true, false ] uses: ./.github/workflows/reusable-tests.yml with: dafny-version: ${{ matrix.version }} + unicode-char: ${{ matrix.unicode-char }} diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 7c78aef0..6ff38421 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -7,6 +7,9 @@ on: dafny-version: required: true type: string + unicode-char: + required: true + type: boolean jobs: reusable_verification: @@ -39,9 +42,15 @@ jobs: - name: Set up JS dependencies run: npm install bignumber.js - - name: Verify Code and Examples + - name: Verify Code and Examples (--unicode-char:false) + if: ${{ !inputs.dafny-version }} + run: | + lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv --unicode-char:true ${{ github.workspace }}/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy' . + + - name: Verify Code and Examples (--unicode-char:false) + if: ${{ inputs.dafny-version }} run: | - lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv' . + lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv --unicode-char:true ${{ github.workspace }}/src/Unicode/UnicodeStringsWithUnicodeChar.dfy' . - name: Generate Report run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-resource-count 40000000 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b42bc958..ea5b8b8f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,9 @@ jobs: strategy: matrix: version: [ 3.13.1, 4.0.0 ] + unicode-char: [ true, false ] uses: ./.github/workflows/reusable-tests.yml with: dafny-version: ${{ matrix.version }} + unicode-char: ${{ matrix.unicode-char }} From fb26d7fec731845be6bb19ddcba4eafecbe720cd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 27 Apr 2023 14:46:39 -0700 Subject: [PATCH 63/84] =?UTF-8?q?Revert=20"Don=E2=80=99t=20include=20Unico?= =?UTF-8?q?deStrings=20directly,=20augment=20CI=20to=20test=20both=20ways"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 99f0662950b77cfa1eb7dbf8f0bc6108ab46a159. --- .github/workflows/nightly.yml | 2 -- .github/workflows/reusable-tests.yml | 13 ++----------- .github/workflows/tests.yml | 2 -- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 72da8bb4..ff144cfa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,9 +15,7 @@ jobs: strategy: matrix: version: [ nightly-latest ] - unicode-char: [ true, false ] uses: ./.github/workflows/reusable-tests.yml with: dafny-version: ${{ matrix.version }} - unicode-char: ${{ matrix.unicode-char }} diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 6ff38421..7c78aef0 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -7,9 +7,6 @@ on: dafny-version: required: true type: string - unicode-char: - required: true - type: boolean jobs: reusable_verification: @@ -42,15 +39,9 @@ jobs: - name: Set up JS dependencies run: npm install bignumber.js - - name: Verify Code and Examples (--unicode-char:false) - if: ${{ !inputs.dafny-version }} - run: | - lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv --unicode-char:true ${{ github.workspace }}/src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy' . - - - name: Verify Code and Examples (--unicode-char:false) - if: ${{ inputs.dafny-version }} + - name: Verify Code and Examples run: | - lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv --unicode-char:true ${{ github.workspace }}/src/Unicode/UnicodeStringsWithUnicodeChar.dfy' . + lit --time-tests -v --param 'dafny_params=--log-format trx --log-format csv' . - name: Generate Report run: find . -name '*.csv' -print0 | xargs -0 --verbose dafny-reportgenerator summarize-csv-results --max-resource-count 40000000 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ea5b8b8f..b42bc958 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,9 +14,7 @@ jobs: strategy: matrix: version: [ 3.13.1, 4.0.0 ] - unicode-char: [ true, false ] uses: ./.github/workflows/reusable-tests.yml with: dafny-version: ${{ matrix.version }} - unicode-char: ${{ matrix.unicode-char }} From 10545c8cd0c16e974ee13d71e3afcde02522b10a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 27 Apr 2023 14:56:36 -0700 Subject: [PATCH 64/84] Different approach with multiple lit commands instead --- src/JSON/API.dfy | 3 ++- src/JSON/ConcreteSyntax.SpecProperties.dfy | 3 ++- src/JSON/Deserializer.dfy | 3 ++- src/JSON/Serializer.dfy | 3 ++- src/JSON/Spec.dfy | 8 +++++--- src/JSON/Tests.dfy | 3 ++- src/JSON/Tutorial.dfy | 4 +++- src/JSON/ZeroCopy/API.dfy | 3 ++- src/JSON/ZeroCopy/Deserializer.dfy | 3 ++- src/JSON/ZeroCopy/Serializer.dfy | 3 ++- 10 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index b4924b20..6077bb3d 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy include "Serializer.dfy" include "Deserializer.dfy" diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 8613de04..4e5889f8 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy include "ConcreteSyntax.Spec.dfy" diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 77798779..8b735ed2 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy /// =============================================== /// Deserialization from JSON.Grammar to JSON.AST diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index c1f364ec..8a64bf92 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy /// ============================================= /// Serialization from JSON.AST to JSON.Grammar diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index fc03b59d..dde7a139 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy /// ============================================= /// Serialization from AST.JSON to bytes (Spec) @@ -10,8 +11,9 @@ include "../BoundedInts.dfy" include "../NonlinearArithmetic/Logarithm.dfy" include "../Collections/Sequences/Seq.dfy" - // TODO: Remove and follow one of the options documented in UnicodeStrings.dfy -include "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" +// include one of these two files externally as well: +// "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" +// "../Unicode/UnicodeStringsWithUnicodeChar.dfy" include "AST.dfy" include "Errors.dfy" diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 32044edc..04e57a08 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,4 +1,5 @@ -// RUN: %run "%s" +// RUN: %run "%s" --unicode-char:false --input ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %run "%s" --unicode-char:true --input ../Unicode/UnicodeStringsWithUnicodeChar.dfy include "Errors.dfy" include "API.dfy" diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index e2ea3e97..d5e3286f 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -1,4 +1,6 @@ -// RUN: %test "%s" +// RUN: %run "%s" --unicode-char:false --input ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %run "%s" --unicode-char:true --input ../Unicode/UnicodeStringsWithUnicodeChar.dfy + /// # Using the JSON library include "API.dfy" diff --git a/src/JSON/ZeroCopy/API.dfy b/src/JSON/ZeroCopy/API.dfy index bc1ef17d..21b1cc84 100644 --- a/src/JSON/ZeroCopy/API.dfy +++ b/src/JSON/ZeroCopy/API.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../../Unicode/UnicodeStringsWithUnicodeChar.dfy include "../Grammar.dfy" include "../ConcreteSyntax.Spec.dfy" diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 8b48099b..b20fab44 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../../Unicode/UnicodeStringsWithUnicodeChar.dfy include "../Utils/Seq.dfy" include "../Errors.dfy" diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index 11030e2d..19dfc58f 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,4 +1,5 @@ -// RUN: %verify "%s" +// RUN: %verify "%s" --unicode-char:false ../../Unicode/UnicodeStringsWithoutUnicodeChar.dfy +// RUN: %verify "%s" --unicode-char:true ../../Unicode/UnicodeStringsWithUnicodeChar.dfy include "../Utils/Seq.dfy" include "../Errors.dfy" From e84b10ac6944d04fb259d669f510af842112bcfb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 13:44:01 -0700 Subject: [PATCH 65/84] Tweak comment --- src/Unicode/UnicodeStrings.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Unicode/UnicodeStrings.dfy b/src/Unicode/UnicodeStrings.dfy index 524b4ce7..879edcc8 100644 --- a/src/Unicode/UnicodeStrings.dfy +++ b/src/Unicode/UnicodeStrings.dfy @@ -19,10 +19,10 @@ /// 1. Implement your logic in an abstract module as well that /// imports this one, and define two different refining modules /// that import the appropriate UnicodeStrings module. -/// See (TODO example) for an example. /// 2. Do not `include` any of these three files in your source code, /// instead passing the appropriate file to Dafny when verifying and building, /// so that references to `UnicodeStrings` can be resolved. +/// See the JSON modules as an example. /// /// Option 2. avoids needing to write boilerplate refining modules, /// but is less IDE-friendly until we have better project configuration support. From 53ce3dd436f4a26bde5327d02b1e990f84bac879 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 15:44:54 -0700 Subject: [PATCH 66/84] =?UTF-8?q?ZeroCopy=20code=20doesn=E2=80=99t=20depen?= =?UTF-8?q?d=20on=20UnicodeStrings=20module(s)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JSON/ZeroCopy/Deserializer.dfy | 3 +-- src/JSON/ZeroCopy/Serializer.dfy | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index b20fab44..8b48099b 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -1,5 +1,4 @@ -// RUN: %verify "%s" --unicode-char:false ../../Unicode/UnicodeStringsWithoutUnicodeChar.dfy -// RUN: %verify "%s" --unicode-char:true ../../Unicode/UnicodeStringsWithUnicodeChar.dfy +// RUN: %verify "%s" include "../Utils/Seq.dfy" include "../Errors.dfy" diff --git a/src/JSON/ZeroCopy/Serializer.dfy b/src/JSON/ZeroCopy/Serializer.dfy index 19dfc58f..11030e2d 100644 --- a/src/JSON/ZeroCopy/Serializer.dfy +++ b/src/JSON/ZeroCopy/Serializer.dfy @@ -1,5 +1,4 @@ -// RUN: %verify "%s" --unicode-char:false ../../Unicode/UnicodeStringsWithoutUnicodeChar.dfy -// RUN: %verify "%s" --unicode-char:true ../../Unicode/UnicodeStringsWithUnicodeChar.dfy +// RUN: %verify "%s" include "../Utils/Seq.dfy" include "../Errors.dfy" From 6136cda54c0957ea5782cfdbd20d991343578a01 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 16:29:45 -0700 Subject: [PATCH 67/84] Formatting --- src/JSON/Spec.dfy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index dde7a139..6f71414f 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -11,9 +11,9 @@ include "../BoundedInts.dfy" include "../NonlinearArithmetic/Logarithm.dfy" include "../Collections/Sequences/Seq.dfy" -// include one of these two files externally as well: -// "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" -// "../Unicode/UnicodeStringsWithUnicodeChar.dfy" +/// include one of these two files externally as well: +/// "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" +/// "../Unicode/UnicodeStringsWithUnicodeChar.dfy" include "AST.dfy" include "Errors.dfy" From 4aacbd628e6918bb878174d560ffde93e230cdd4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 16:38:44 -0700 Subject: [PATCH 68/84] Fix README.md --- src/JSON/README.md | 1 + src/JSON/Tutorial.dfy | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/JSON/README.md b/src/JSON/README.md index 390cae37..7bf261e6 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -17,6 +17,7 @@ The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, ```dafny include "src/JSON/API.dfy" +include "src/Unicode/UnicodeStringsWithUnicodeChar.dfy" import JSON.API import opened UnicodeStrings diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index d5e3286f..9ee445e3 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -16,6 +16,11 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import API import opened AST import opened Wrappers + +/// Note that you will need to include one of the two files that defines UnicodeStrings +/// according to whether you are using --unicode-char:false or --unicode-char:true. +/// See ../../Unicode/UnicodeStrings.dfy for more details. + import opened UnicodeStrings /// The high-level API works with fairly simple ASTs that contain native Dafny From 7d626c5690d819ae22aad188cd4f190276c27813 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 16:47:43 -0700 Subject: [PATCH 69/84] =?UTF-8?q?Actually=20check-examples=20uses=20?= =?UTF-8?q?=E2=80=94unicode-char:false?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JSON/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/JSON/README.md b/src/JSON/README.md index 7bf261e6..1e3355a0 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -17,7 +17,8 @@ The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, ```dafny include "src/JSON/API.dfy" -include "src/Unicode/UnicodeStringsWithUnicodeChar.dfy" +// Or "..WithUnicodeChar.dfy" if you are using --unicode-char:true +include "src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy" import JSON.API import opened UnicodeStrings From c35669c3af639489d5ddd65d1e993946077eee66 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 17:16:22 -0700 Subject: [PATCH 70/84] Rename JSON.AST to JSON.Values --- src/JSON/API.dfy | 10 ++++----- src/JSON/Deserializer.dfy | 38 ++++++++++++++++---------------- src/JSON/Grammar.dfy | 2 +- src/JSON/README.md | 16 ++++++++------ src/JSON/Serializer.dfy | 22 +++++++++--------- src/JSON/Spec.dfy | 10 ++++----- src/JSON/Tests.dfy | 4 ++-- src/JSON/Tutorial.dfy | 28 +++++++++++------------ src/JSON/{AST.dfy => Values.dfy} | 2 +- 9 files changed, 67 insertions(+), 65 deletions(-) rename src/JSON/{AST.dfy => Values.dfy} (87%) diff --git a/src/JSON/API.dfy b/src/JSON/API.dfy index 6077bb3d..74f9c89e 100644 --- a/src/JSON/API.dfy +++ b/src/JSON/API.dfy @@ -8,31 +8,31 @@ include "ZeroCopy/API.dfy" module {:options "-functionSyntax:4"} JSON.API { import opened BoundedInts import opened Errors - import AST + import Values import Serializer import Deserializer import ZeroCopy = ZeroCopy.API - function {:opaque} Serialize(js: AST.JSON) : (bs: SerializationResult>) + function {:opaque} Serialize(js: Values.JSON) : (bs: SerializationResult>) { var js :- Serializer.JSON(js); ZeroCopy.Serialize(js) } - method SerializeAlloc(js: AST.JSON) returns (bs: SerializationResult>) + method SerializeAlloc(js: Values.JSON) returns (bs: SerializationResult>) { var js :- Serializer.JSON(js); bs := ZeroCopy.SerializeAlloc(js); } - method SerializeInto(js: AST.JSON, bs: array) returns (len: SerializationResult) + method SerializeInto(js: Values.JSON, bs: array) returns (len: SerializationResult) modifies bs { var js :- Serializer.JSON(js); len := ZeroCopy.SerializeInto(js, bs); } - function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) + function {:opaque} Deserialize(bs: seq) : (js: DeserializationResult) { var js :- ZeroCopy.Deserialize(bs); Deserializer.JSON(js) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 8b735ed2..3b413592 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -1,9 +1,9 @@ // RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy // RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy -/// =============================================== -/// Deserialization from JSON.Grammar to JSON.AST -/// =============================================== +/// ================================================== +/// Deserialization from JSON.Grammar to JSON.Values +/// ================================================== /// /// For the spec, see ``JSON.Spec.dfy``. @@ -13,7 +13,7 @@ include "../BoundedInts.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" include "Errors.dfy" -include "AST.dfy" +include "Values.dfy" include "Grammar.dfy" include "Spec.dfy" @@ -27,7 +27,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { import opened Utils.Str import opened UnicodeStrings - import AST + import Values import Spec import opened Errors import opened Utils.Vectors @@ -135,7 +135,7 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { Success(if sign.Char?('-') then -n else n) } - function Number(js: Grammar.jnumber): DeserializationResult { + function Number(js: Grammar.jnumber): DeserializationResult { var JNumber(minus, num, frac, exp) := js; var n :- ToInt(minus, num); @@ -143,38 +143,38 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { case Empty => Success(0) case NonEmpty(JExp(_, sign, num)) => ToInt(sign, num); match frac - case Empty => Success(AST.Decimal(n, e10)) + case Empty => Success(Values.Decimal(n, e10)) case NonEmpty(JFrac(_, num)) => var pow10 := num.Length() as int; var frac :- ToInt(minus, num); - Success(AST.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) + Success(Values.Decimal(n * Pow(10, pow10) + frac, e10 - pow10)) } - function KeyValue(js: Grammar.jKeyValue): DeserializationResult<(string, AST.JSON)> { + function KeyValue(js: Grammar.jKeyValue): DeserializationResult<(string, Values.JSON)> { var k :- String(js.k); var v :- Value(js.v); Success((k, v)) } - function Object(js: Grammar.jobject): DeserializationResult> { + function Object(js: Grammar.jobject): DeserializationResult> { Seq.MapWithResult(d requires d in js.data => KeyValue(d.t), js.data) } - function Array(js: Grammar.jarray): DeserializationResult> { + function Array(js: Grammar.jarray): DeserializationResult> { Seq.MapWithResult(d requires d in js.data => Value(d.t), js.data) } - function Value(js: Grammar.Value): DeserializationResult { + function Value(js: Grammar.Value): DeserializationResult { match js - case Null(_) => Success(AST.Null()) - case Bool(b) => Success(AST.Bool(Bool(b))) - case String(str) => var s :- String(str); Success(AST.String(s)) - case Number(dec) => var n :- Number(dec); Success(AST.Number(n)) - case Object(obj) => var o :- Object(obj); Success(AST.Object(o)) - case Array(arr) => var a :- Array(arr); Success(AST.Array(a)) + case Null(_) => Success(Values.Null()) + case Bool(b) => Success(Values.Bool(Bool(b))) + case String(str) => var s :- String(str); Success(Values.String(s)) + case Number(dec) => var n :- Number(dec); Success(Values.Number(n)) + case Object(obj) => var o :- Object(obj); Success(Values.Object(o)) + case Array(arr) => var a :- Array(arr); Success(Values.Array(a)) } - function JSON(js: Grammar.JSON): DeserializationResult { + function JSON(js: Grammar.JSON): DeserializationResult { Value(js.t) } } diff --git a/src/JSON/Grammar.dfy b/src/JSON/Grammar.dfy index 7f6013d9..c32b8b3c 100644 --- a/src/JSON/Grammar.dfy +++ b/src/JSON/Grammar.dfy @@ -4,7 +4,7 @@ /// Low-level JSON grammar (Concrete syntax) /// ========================================== /// -/// See ``JSON.AST`` for the high-level interface. +/// See ``JSON.Values`` for the high-level interface. include "../BoundedInts.dfy" include "Utils/Views.dfy" diff --git a/src/JSON/README.md b/src/JSON/README.md index 1e3355a0..e81467da 100644 --- a/src/JSON/README.md +++ b/src/JSON/README.md @@ -6,13 +6,15 @@ This library provides two APIs: - A low-level (zero-copy) API that is efficient, verified (see [What is verified?](#what-is-verified) below for details) and allows incremental changes (re-serialization is much faster for unchanged objects), but is more cumbersome to use. This API operates on concrete syntax trees that capture details of punctuation and blanks and represent strings using unescaped, undecoded utf-8 byte sequences. -- A high-level API built on top of the previous one. This API is more convenient to use, but it is unverified and less efficient. It produces abstract syntax trees that represent strings using Dafny's built-in `string` type. +- A high-level API built on top of the previous one. This API is more convenient to use, but it is unverified and less efficient. It produces abstract datatype value trees that represent strings using Dafny's built-in `string` type. + +Both APIs provides functions for serialization (JSON values to utf-8 bytes) and deserialization (utf-8 bytes to JSON values). +See the Unicode module in `../Unicode` if you need to read or produce JSON text in other encodings. -Both APIs provides functions for serialization (utf-8 bytes to AST) and deserialization (AST to utf-8 bytes). Unverified transcoding functions are provided in `Utils/Unicode.dfy` if you need to read or produce JSON text in other encodings. ## Library usage -The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, call the high-level API, and use the low-level API to make localized modifications to a partial parse of a JSON AST. The main entry points are `API.Serialize` (to go from utf-8 bytes to a JSON AST), and `API.Deserialize` (for the reverse operation): +The tutorial in [`Tutorial.dfy`](Tutorial.dfy) shows how to import the library, call the high-level API, and use the low-level API to make localized modifications to a partial parse of a JSON AST. The main entry points are `API.Serialize` (to go from a JSON value to utf-8 bytes), and `API.Deserialize` (for the reverse operation): ```dafny @@ -22,7 +24,7 @@ include "src/Unicode/UnicodeStringsWithoutUnicodeChar.dfy" import JSON.API import opened UnicodeStrings -import opened JSON.AST +import opened JSON.Values import opened Wrappers method Test(){ @@ -32,20 +34,20 @@ method Test(){ ""Population"": 689386, ""Area (km2)"": 4584.2}]}"); - var CITY_AST := Object([("Cities", Array([ + var CITY_VALUE := Object([("Cities", Array([ Object([ ("Name", String("Boston")), ("Founded", Number(Int(1630))), ("Population", Number(Int(689386))), ("Area (km2)", Number(Decimal(45842, -1)))])]))]); - expect API.Deserialize(CITY_JS) == Success(CITY_AST); + expect API.Deserialize(CITY_JS) == Success(CITY_VALUE); var EXPECTED :- expect ToUTF8Checked( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" ); - expect API.Serialize(CITY_AST) == Success(EXPECTED); + expect API.Serialize(CITY_VALUE) == Success(EXPECTED); } ``` diff --git a/src/JSON/Serializer.dfy b/src/JSON/Serializer.dfy index 8a64bf92..1f936deb 100644 --- a/src/JSON/Serializer.dfy +++ b/src/JSON/Serializer.dfy @@ -1,9 +1,9 @@ // RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy // RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy -/// ============================================= -/// Serialization from JSON.AST to JSON.Grammar -/// ============================================= +/// ================================================ +/// Serialization from JSON.Values to JSON.Grammar +/// ================================================ /// /// For the spec, see ``JSON.Spec.dfy``. @@ -14,7 +14,7 @@ include "../Math.dfy" include "Utils/Views.dfy" include "Utils/Vectors.dfy" include "Errors.dfy" -include "AST.dfy" +include "Values.dfy" include "Grammar.dfy" include "Spec.dfy" @@ -25,7 +25,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { import opened BoundedInts import opened Utils.Str - import AST + import Values import Spec import opened Errors import opened Utils.Vectors @@ -81,7 +81,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { Success(View.OfBytes(bs)) } - function Number(dec: AST.Decimal): Result { + function Number(dec: Values.Decimal): Result { var minus: jminus := Sign(dec.n); var num: jnum :- Int(Math.Abs(dec.n)); var frac: Maybe := Empty(); @@ -103,7 +103,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { const COLON: Structural := MkStructural(Grammar.COLON) - function KeyValue(kv: (string, AST.JSON)): Result { + function KeyValue(kv: (string, Values.JSON)): Result { var k :- String(kv.0); var v :- Value(kv.1); Success(Grammar.KeyValue(k, COLON, v)) @@ -121,7 +121,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { const COMMA: Structural := MkStructural(Grammar.COMMA) - function Object(obj: seq<(string, AST.JSON)>): Result { + function Object(obj: seq<(string, Values.JSON)>): Result { var items :- Seq.MapWithResult(v requires v in obj => KeyValue(v), obj); Success(Bracketed(MkStructural(LBRACE), MkSuffixedSequence(items, COMMA), @@ -129,14 +129,14 @@ module {:options "-functionSyntax:4"} JSON.Serializer { } - function Array(arr: seq): Result { + function Array(arr: seq): Result { var items :- Seq.MapWithResult(v requires v in arr => Value(v), arr); Success(Bracketed(MkStructural(LBRACKET), MkSuffixedSequence(items, COMMA), MkStructural(RBRACKET))) } - function Value(js: AST.JSON): Result { + function Value(js: Values.JSON): Result { match js case Null => Success(Grammar.Null(View.OfBytes(NULL))) case Bool(b) => Success(Grammar.Bool(Bool(b))) @@ -146,7 +146,7 @@ module {:options "-functionSyntax:4"} JSON.Serializer { case Array(arr) => var a :- Array(arr); Success(Grammar.Array(a)) } - function JSON(js: AST.JSON): Result { + function JSON(js: Values.JSON): Result { var val :- Value(js); Success(MkStructural(val)) } diff --git a/src/JSON/Spec.dfy b/src/JSON/Spec.dfy index 6f71414f..8a76ea1e 100644 --- a/src/JSON/Spec.dfy +++ b/src/JSON/Spec.dfy @@ -1,9 +1,9 @@ // RUN: %verify "%s" --unicode-char:false ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy // RUN: %verify "%s" --unicode-char:true ../Unicode/UnicodeStringsWithUnicodeChar.dfy -/// ============================================= -/// Serialization from AST.JSON to bytes (Spec) -/// ============================================= +/// ================================================ +/// Serialization from Values.JSON to bytes (Spec) +/// ================================================ /// /// This is the high-level spec. For the implementation, see /// ``JSON.Serializer.dfy``. @@ -15,7 +15,7 @@ include "../Collections/Sequences/Seq.dfy" /// "../Unicode/UnicodeStringsWithoutUnicodeChar.dfy" /// "../Unicode/UnicodeStringsWithUnicodeChar.dfy" -include "AST.dfy" +include "Values.dfy" include "Errors.dfy" include "Utils/Str.dfy" @@ -23,7 +23,7 @@ module {:options "-functionSyntax:4"} JSON.Spec { import opened BoundedInts import opened Utils.Str - import opened AST + import opened Values import opened Wrappers import opened Errors import opened UnicodeStrings diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 04e57a08..2a04681c 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -77,10 +77,10 @@ module JSON.Tests.AbstractSyntaxWrapper refines Wrapper { import opened Wrappers import Grammar import API - import AST + import Values import Spec - type JSON = AST.JSON + type JSON = Values.JSON method Deserialize(bs: bytes) returns (js: DeserializationResult) { js := API.Deserialize(bs); diff --git a/src/JSON/Tutorial.dfy b/src/JSON/Tutorial.dfy index 9ee445e3..04800fd5 100644 --- a/src/JSON/Tutorial.dfy +++ b/src/JSON/Tutorial.dfy @@ -6,15 +6,15 @@ include "API.dfy" include "ZeroCopy/API.dfy" -/// This library offers two APIs: a high-level one (giving abstract syntax trees +/// This library offers two APIs: a high-level one (giving abstract value trees /// with no concrete syntactic details) and a low-level one (including all /// information about blanks, separator positions, character escapes, etc.). /// -/// ## High-level API (Abstract syntax) +/// ## High-level API (JSON values) module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import API - import opened AST + import opened Values import opened Wrappers /// Note that you will need to include one of the two files that defines UnicodeStrings @@ -23,7 +23,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { import opened UnicodeStrings -/// The high-level API works with fairly simple ASTs that contain native Dafny +/// The high-level API works with fairly simple datatype values that contain native Dafny /// strings: method {:test} Test() { @@ -36,8 +36,8 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { /// writing raw bytes directly from disk or from the network instead). var SIMPLE_JS :- expect ToUTF8Checked("[true]"); - var SIMPLE_AST := Array([Bool(true)]); - expect API.Deserialize(SIMPLE_JS) == Success(SIMPLE_AST); + var SIMPLE_VALUE := Array([Bool(true)]); + expect API.Deserialize(SIMPLE_JS) == Success(SIMPLE_VALUE); /// Here is a larger object, written using a verbatim string (with `@"`). In /// verbatim strings `""` represents a single double-quote character): @@ -62,7 +62,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { } ] }"); - var CITIES_AST := + var CITIES_VALUE := Object([ ("Cities", Array([ Object([ @@ -85,12 +85,12 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { ]) ])) ]); - expect API.Deserialize(CITIES_JS) == Success(CITIES_AST); + expect API.Deserialize(CITIES_JS) == Success(CITIES_VALUE); /// Serialization works similarly, with `API.Serialize`. For this first example /// the generated string matches what we started with exactly: - expect API.Serialize(SIMPLE_AST) == Success(SIMPLE_JS); + expect API.Serialize(SIMPLE_VALUE) == Success(SIMPLE_JS); /// For more complex object, the generated layout may not be exactly the same; note in particular how the representation of numbers and the whitespace have changed. @@ -98,7 +98,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1},{""Name"":""Rome"",""Founded"":-753,""Population"":2873e3,""Area (km2)"":1285},{""Name"":""Paris"",""Founded"":null,""Population"":2161e3,""Area (km2)"":23835e-1}]}" ); - expect API.Serialize(CITIES_AST) == Success(EXPECTED); + expect API.Serialize(CITIES_VALUE) == Success(EXPECTED); /// Additional methods are defined in `API.dfy` to serialize an object into an /// existing buffer or into an array. Below is the smaller example from the @@ -110,7 +110,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { ""Population"": 689386, ""Area (km2)"": 4584.2}]}"); - var CITY_AST := + var CITY_VALUE := Object([("Cities", Array([ Object([ ("Name", String("Boston")), @@ -118,13 +118,13 @@ module {:options "-functionSyntax:4"} JSON.Examples.AbstractSyntax { ("Population", Number(Int(689386))), ("Area (km2)", Number(Decimal(45842, -1)))])]))]); - expect API.Deserialize(CITY_JS) == Success(CITY_AST); + expect API.Deserialize(CITY_JS) == Success(CITY_VALUE); var EXPECTED' :- expect ToUTF8Checked( @"{""Cities"":[{""Name"":""Boston"",""Founded"":1630,""Population"":689386,""Area (km2)"":45842e-1}]}" ); - expect API.Serialize(CITY_AST) == Success(EXPECTED'); + expect API.Serialize(CITY_VALUE) == Success(EXPECTED'); } } @@ -148,7 +148,7 @@ module {:options "-functionSyntax:4"} JSON.Examples.ConcreteSyntax { /// The low-level API exposes the same functions and methods as the high-level /// one, but the type that they consume and produce is `Grammar.JSON` (defined /// in `Grammar.dfy` as a `Grammar.Value` surrounded by optional whitespace) -/// instead of `AST.JSON` (defined in `AST.dfy`). Since `Grammar.JSON` contains +/// instead of `Values.JSON` (defined in `Values.dfy`). Since `Grammar.JSON` contains /// all formatting information, re-serializing an object produces the original /// value: diff --git a/src/JSON/AST.dfy b/src/JSON/Values.dfy similarity index 87% rename from src/JSON/AST.dfy rename to src/JSON/Values.dfy index 17d5629f..76952ec2 100644 --- a/src/JSON/AST.dfy +++ b/src/JSON/Values.dfy @@ -1,6 +1,6 @@ // RUN: %verify "%s" -module {:options "-functionSyntax:4"} JSON.AST { +module {:options "-functionSyntax:4"} JSON.Values { datatype Decimal = Decimal(n: int, e10: int) // (n) * 10^(e10) From 4287c0bb16c9955f1f1992f9d0190e33663b6dfe Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 3 May 2023 19:26:39 -0700 Subject: [PATCH 71/84] Add Bug note, test on Java too --- src/JSON/Tests.dfy | 1 + src/JSON/Utils/Parsers.dfy | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index 32044edc..06d71f8c 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,4 +1,5 @@ // RUN: %run "%s" +// RUN: %run -t:java "%s" include "Errors.dfy" include "API.dfy" diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index c195fc62..66fef57c 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -13,6 +13,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> + // BUG(https://github.com/dafny-lang/dafny/issues/3883) // type Parser = p: Parser_ | p.Valid?() // // BUG(https://github.com/dafny-lang/dafny/issues/2103) // witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) @@ -48,6 +49,7 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { && (forall cs': FreshCursor | pre(cs') :: fn(cs').Success? ==> fn(cs').value.StrictlySplitFrom?(cs', spec)) } } + // BUG(https://github.com/dafny-lang/dafny/issues/3883) // type SubParser = p: SubParser_ | p.Valid?() // witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) From 89926086e26611d1b2b4e4121b77b92698e37fac Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 10:38:42 -0700 Subject: [PATCH 72/84] =?UTF-8?q?Remove=20unused=20function=20that=20can?= =?UTF-8?q?=E2=80=99t=20be=20compiled=20for=20Java?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/dafny-lang/dafny/issues/3951 --- src/JSON/Utils/Str.dfy | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/JSON/Utils/Str.dfy b/src/JSON/Utils/Str.dfy index 554ca937..9f099419 100644 --- a/src/JSON/Utils/Str.dfy +++ b/src/JSON/Utils/Str.dfy @@ -87,12 +87,6 @@ module {:options "-functionSyntax:4"} JSON.Utils.Str { else [minus] + OfNat_any(-n, chars) } - function DigitsTable(digits: seq): map - requires forall i, j | 0 <= i < j < |digits| :: digits[i] != digits[j] - { - map i: nat | 0 <= i < |digits| :: digits[i] := i - } - function ToNat_any(str: String, base: nat, digits: map) : (n: nat) requires base > 0 requires forall c | c in str :: c in digits From f277ffec3da090f8b1329f051d762f1129d5fe02 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 10:50:21 -0700 Subject: [PATCH 73/84] Formatting --- src/JSON/ZeroCopy/Deserializer.dfy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 7b9ea0ea..1482fbe6 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -153,9 +153,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} {:vcs_split_on_every_assertion} BracketedFromParts(ghost cs: Cursor, - open: Split>, - elems: Split>, - close: Split>) + open: Split>, + elems: Split>, + close: Split>) : (sp: Split) requires Grammar.NoTrailingSuffix(elems.t) requires open.StrictlySplitFrom?(cs, c => Spec.Structural(c, SpecView)) From d2fa10f96d0481b86bb237952a7f30646accb59e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 11:57:39 -0700 Subject: [PATCH 74/84] Double checking ZeroCopy/Deserializer.dfy verifies with the subset types --- src/JSON/Utils/Parsers.dfy | 10 +++++----- src/JSON/ZeroCopy/Deserializer.dfy | 13 ++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 66fef57c..cb8b3bc0 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -14,9 +14,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> // BUG(https://github.com/dafny-lang/dafny/issues/3883) - // type Parser = p: Parser_ | p.Valid?() - // // BUG(https://github.com/dafny-lang/dafny/issues/2103) - // witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + type Parser = p: Parser_ | p.Valid?() + // BUG(https://github.com/dafny-lang/dafny/issues/2103) + witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, ghost spec: T -> bytes) { ghost predicate Valid?() { @@ -50,8 +50,8 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { } } // BUG(https://github.com/dafny-lang/dafny/issues/3883) - // type SubParser = p: SubParser_ | p.Valid?() - // witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + type SubParser = p: SubParser_ | p.Valid?() + witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) function {:opaque} SubParserWitness(): (subp: SubParser_) ensures subp.Valid?() diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 1482fbe6..b4df13ef 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -24,8 +24,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type JSONError = Errors.DeserializationError type Error = CursorError type ParseResult<+T> = SplitResult - type Parser_ = Parsers.Parser_ - type SubParser_ = Parsers.SubParser_ + type Parser = Parsers.Parser + type SubParser = Parsers.SubParser // BUG(https://github.com/dafny-lang/dafny/issues/2179) const SpecView := (v: Vs.View) => Spec.View(v); @@ -54,9 +54,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { return Cursor(cs.s, cs.beg, point', cs.end).Split(); } - function {:opaque} Structural(cs: FreshCursor, parser: Parser_) + function {:opaque} Structural(cs: FreshCursor, parser: Parser) : (pr: ParseResult>) - requires parser.Valid?() requires forall cs :: parser.fn.requires(cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) { @@ -78,8 +77,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { SP(Grammar.Structural(before, val, after), cs) } - type ValueParser = sp: SubParser_ | - sp.Valid?() && forall t :: sp.spec(t) == Spec.Value(t) + type ValueParser = sp: SubParser | + forall t :: sp.spec(t) == Spec.Value(t) witness * } type Error = Core.Error @@ -152,7 +151,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Success(cs.Split()) } - function {:opaque} {:vcs_split_on_every_assertion} BracketedFromParts(ghost cs: Cursor, + function {:opaque} BracketedFromParts(ghost cs: Cursor, open: Split>, elems: Split>, close: Split>) From acdb6ec36b00c04a2675064664a395e5a5f8c14c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 11:57:52 -0700 Subject: [PATCH 75/84] Revert "Double checking ZeroCopy/Deserializer.dfy verifies with the subset types" This reverts commit d2fa10f96d0481b86bb237952a7f30646accb59e. --- src/JSON/Utils/Parsers.dfy | 10 +++++----- src/JSON/ZeroCopy/Deserializer.dfy | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index cb8b3bc0..66fef57c 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -14,9 +14,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> // BUG(https://github.com/dafny-lang/dafny/issues/3883) - type Parser = p: Parser_ | p.Valid?() - // BUG(https://github.com/dafny-lang/dafny/issues/2103) - witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + // type Parser = p: Parser_ | p.Valid?() + // // BUG(https://github.com/dafny-lang/dafny/issues/2103) + // witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, ghost spec: T -> bytes) { ghost predicate Valid?() { @@ -50,8 +50,8 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { } } // BUG(https://github.com/dafny-lang/dafny/issues/3883) - type SubParser = p: SubParser_ | p.Valid?() - witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + // type SubParser = p: SubParser_ | p.Valid?() + // witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) function {:opaque} SubParserWitness(): (subp: SubParser_) ensures subp.Valid?() diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index b4df13ef..1482fbe6 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -24,8 +24,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type JSONError = Errors.DeserializationError type Error = CursorError type ParseResult<+T> = SplitResult - type Parser = Parsers.Parser - type SubParser = Parsers.SubParser + type Parser_ = Parsers.Parser_ + type SubParser_ = Parsers.SubParser_ // BUG(https://github.com/dafny-lang/dafny/issues/2179) const SpecView := (v: Vs.View) => Spec.View(v); @@ -54,8 +54,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { return Cursor(cs.s, cs.beg, point', cs.end).Split(); } - function {:opaque} Structural(cs: FreshCursor, parser: Parser) + function {:opaque} Structural(cs: FreshCursor, parser: Parser_) : (pr: ParseResult>) + requires parser.Valid?() requires forall cs :: parser.fn.requires(cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) { @@ -77,8 +78,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { SP(Grammar.Structural(before, val, after), cs) } - type ValueParser = sp: SubParser | - forall t :: sp.spec(t) == Spec.Value(t) + type ValueParser = sp: SubParser_ | + sp.Valid?() && forall t :: sp.spec(t) == Spec.Value(t) witness * } type Error = Core.Error @@ -151,7 +152,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Success(cs.Split()) } - function {:opaque} BracketedFromParts(ghost cs: Cursor, + function {:opaque} {:vcs_split_on_every_assertion} BracketedFromParts(ghost cs: Cursor, open: Split>, elems: Split>, close: Split>) From ceed95194b2210e0d6cec98dd4c1b338c1561ea1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 14:10:22 -0700 Subject: [PATCH 76/84] Progress on verifying again --- src/JSON/ZeroCopy/Deserializer.dfy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 1482fbe6..e2d012c9 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -258,7 +258,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { // The implementation and proof of this function is more painful than // expected due to the tail recursion. - function {:opaque} {:tailrecursion} Elements( + function {:opaque} {:tailrecursion} {:vcs_split_on_every_assert} Elements( ghost cs0: FreshCursor, json: ValueParser, open: Split>, @@ -280,6 +280,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { assert {:focus} elems.cs.StrictlySplitFrom?(json.cs); assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); assert elem.StrictlySplitFrom?(elems.cs, ElementSpec); + assert json.Valid?(); assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); assert forall e | e in elems.t :: e.suffix.NonEmpty?; assert {:split_here} true; @@ -288,6 +289,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Elements(cs0, json, open, elems) else if s0 == CLOSE as opt_byte then assert AppendLast.requires(open.cs, json, elems, elem, sep) by { + assert json.Valid?(); assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); assert elems.cs.StrictlySplitFrom?(json.cs); assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); @@ -359,7 +361,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { case OtherError(err) => err } - function {:opaque} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) + function {:opaque} {:vcs_split_on_every_assert} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.JSON) { Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)).MapFailure(LiftCursorError) @@ -400,7 +402,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import ConcreteSyntax.SpecProperties - function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} {:vcs_split_on_every_assert} Value(cs: FreshCursor) : (pr: ParseResult) decreases cs.Length(), 1 ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Value) { @@ -733,7 +735,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { function ElementSpec(t: TElement) : bytes { Spec.KeyValue(t) } - function {:opaque} Element(cs: FreshCursor, json: ValueParser) + function {:opaque} {:vcs_split_on_every_assert} Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { var k :- Strings.String(cs); @@ -745,6 +747,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { assert colon.StrictlySplitFrom?(k.cs, st => Spec.Structural(st, p.spec)); assert colon.cs.StrictlySplitFrom?(json.cs); + assert json.Valid?(); var v :- json.fn(colon.cs); var kv := KeyValueFromParts(cs, k, colon, v); Success(kv) From fc2ee8cd5ba8af318d5a6367bc432df1419faf6b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 15:53:31 -0700 Subject: [PATCH 77/84] Revert "Progress on verifying again" This reverts commit ceed95194b2210e0d6cec98dd4c1b338c1561ea1. --- src/JSON/ZeroCopy/Deserializer.dfy | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index e2d012c9..1482fbe6 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -258,7 +258,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { // The implementation and proof of this function is more painful than // expected due to the tail recursion. - function {:opaque} {:tailrecursion} {:vcs_split_on_every_assert} Elements( + function {:opaque} {:tailrecursion} Elements( ghost cs0: FreshCursor, json: ValueParser, open: Split>, @@ -280,7 +280,6 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { assert {:focus} elems.cs.StrictlySplitFrom?(json.cs); assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); assert elem.StrictlySplitFrom?(elems.cs, ElementSpec); - assert json.Valid?(); assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); assert forall e | e in elems.t :: e.suffix.NonEmpty?; assert {:split_here} true; @@ -289,7 +288,6 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Elements(cs0, json, open, elems) else if s0 == CLOSE as opt_byte then assert AppendLast.requires(open.cs, json, elems, elem, sep) by { - assert json.Valid?(); assert sep.StrictlySplitFrom?(elem.cs, c => Spec.Structural(c, SpecView)); assert elems.cs.StrictlySplitFrom?(json.cs); assert elems.SplitFrom?(open.cs, SuffixedElementsSpec); @@ -361,7 +359,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { case OtherError(err) => err } - function {:opaque} {:vcs_split_on_every_assert} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) + function {:opaque} JSON(cs: Cursors.FreshCursor) : (pr: DeserializationResult>) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.JSON) { Core.Structural(cs, Parsers.Parser(Values.Value, Spec.Value)).MapFailure(LiftCursorError) @@ -402,7 +400,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { import ConcreteSyntax.SpecProperties - function {:opaque} {:vcs_split_on_every_assert} Value(cs: FreshCursor) : (pr: ParseResult) + function {:opaque} Value(cs: FreshCursor) : (pr: ParseResult) decreases cs.Length(), 1 ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, Spec.Value) { @@ -735,7 +733,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { function ElementSpec(t: TElement) : bytes { Spec.KeyValue(t) } - function {:opaque} {:vcs_split_on_every_assert} Element(cs: FreshCursor, json: ValueParser) + function {:opaque} Element(cs: FreshCursor, json: ValueParser) : (pr: ParseResult) { var k :- Strings.String(cs); @@ -747,7 +745,6 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { assert colon.StrictlySplitFrom?(k.cs, st => Spec.Structural(st, p.spec)); assert colon.cs.StrictlySplitFrom?(json.cs); - assert json.Valid?(); var v :- json.fn(colon.cs); var kv := KeyValueFromParts(cs, k, colon, v); Success(kv) From 53b7abf3e3dff22b9f6b4370582e6fefe0d3d38e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 15:58:28 -0700 Subject: [PATCH 78/84] Undoing subset type workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Can’t get it to verify with Dafny 4.0. Will leave getting the code to compile for follow up after merging. --- src/JSON/Tests.dfy | 3 +-- src/JSON/Utils/Parsers.dfy | 13 ++++++------- src/JSON/ZeroCopy/Deserializer.dfy | 13 ++++++------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/JSON/Tests.dfy b/src/JSON/Tests.dfy index de95b5f0..8051e75f 100644 --- a/src/JSON/Tests.dfy +++ b/src/JSON/Tests.dfy @@ -1,8 +1,7 @@ // RUN: %run "%s" --unicode-char:false --input ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy // RUN: %run "%s" --unicode-char:true --input ../Unicode/UnicodeStringsWithUnicodeChar.dfy -// RUN: %run -t:java "%s" --unicode-char:false --input ../Unicode/UnicodeStringsWithoutUnicodeChar.dfy -// RUN: %run -t:java "%s" --unicode-char:true --input ../Unicode/UnicodeStringsWithUnicodeChar.dfy +// TODO: Test for Java and other target languages too include "Errors.dfy" include "API.dfy" diff --git a/src/JSON/Utils/Parsers.dfy b/src/JSON/Utils/Parsers.dfy index 66fef57c..637a24bd 100644 --- a/src/JSON/Utils/Parsers.dfy +++ b/src/JSON/Utils/Parsers.dfy @@ -13,10 +13,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { type SplitResult<+T, +R> = Result, CursorError> - // BUG(https://github.com/dafny-lang/dafny/issues/3883) - // type Parser = p: Parser_ | p.Valid?() - // // BUG(https://github.com/dafny-lang/dafny/issues/2103) - // witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + type Parser = p: Parser_ | p.Valid?() + // BUG(https://github.com/dafny-lang/dafny/issues/2103) + witness ParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) datatype Parser_ = Parser(fn: FreshCursor -> SplitResult, ghost spec: T -> bytes) { ghost predicate Valid?() { @@ -49,9 +48,9 @@ module {:options "-functionSyntax:4"} JSON.Utils.Parsers { && (forall cs': FreshCursor | pre(cs') :: fn(cs').Success? ==> fn(cs').value.StrictlySplitFrom?(cs', spec)) } } - // BUG(https://github.com/dafny-lang/dafny/issues/3883) - // type SubParser = p: SubParser_ | p.Valid?() - // witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) + + type SubParser = p: SubParser_ | p.Valid?() + witness SubParserWitness() // BUG(https://github.com/dafny-lang/dafny/issues/2175) function {:opaque} SubParserWitness(): (subp: SubParser_) ensures subp.Valid?() diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 1482fbe6..b4df13ef 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -24,8 +24,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { type JSONError = Errors.DeserializationError type Error = CursorError type ParseResult<+T> = SplitResult - type Parser_ = Parsers.Parser_ - type SubParser_ = Parsers.SubParser_ + type Parser = Parsers.Parser + type SubParser = Parsers.SubParser // BUG(https://github.com/dafny-lang/dafny/issues/2179) const SpecView := (v: Vs.View) => Spec.View(v); @@ -54,9 +54,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { return Cursor(cs.s, cs.beg, point', cs.end).Split(); } - function {:opaque} Structural(cs: FreshCursor, parser: Parser_) + function {:opaque} Structural(cs: FreshCursor, parser: Parser) : (pr: ParseResult>) - requires parser.Valid?() requires forall cs :: parser.fn.requires(cs) ensures pr.Success? ==> pr.value.StrictlySplitFrom?(cs, st => Spec.Structural(st, parser.spec)) { @@ -78,8 +77,8 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { SP(Grammar.Structural(before, val, after), cs) } - type ValueParser = sp: SubParser_ | - sp.Valid?() && forall t :: sp.spec(t) == Spec.Value(t) + type ValueParser = sp: SubParser | + forall t :: sp.spec(t) == Spec.Value(t) witness * } type Error = Core.Error @@ -152,7 +151,7 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { Success(cs.Split()) } - function {:opaque} {:vcs_split_on_every_assertion} BracketedFromParts(ghost cs: Cursor, + function {:opaque} BracketedFromParts(ghost cs: Cursor, open: Split>, elems: Split>, close: Split>) From 2753e76a1dbaee4fb09eeb47f16beacf7fea6d11 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 May 2023 16:06:40 -0700 Subject: [PATCH 79/84] Formatting --- src/JSON/ZeroCopy/Deserializer.dfy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index b4df13ef..8b48099b 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -152,9 +152,9 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { } function {:opaque} BracketedFromParts(ghost cs: Cursor, - open: Split>, - elems: Split>, - close: Split>) + open: Split>, + elems: Split>, + close: Split>) : (sp: Split) requires Grammar.NoTrailingSuffix(elems.t) requires open.StrictlySplitFrom?(cs, c => Spec.Structural(c, SpecView)) From 35cca47485f39f545daa7cd550c5f468d5bb3da2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 May 2023 13:36:08 -0700 Subject: [PATCH 80/84] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mikaël Mayer --- src/Collections/Sequences/Seq.dfy | 3 +-- src/JSON/ConcreteSyntax.SpecProperties.dfy | 4 ++-- src/JSON/Utils/Seq.dfy | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Collections/Sequences/Seq.dfy b/src/Collections/Sequences/Seq.dfy index 645864ec..a71a487c 100644 --- a/src/Collections/Sequences/Seq.dfy +++ b/src/Collections/Sequences/Seq.dfy @@ -828,9 +828,8 @@ module {:options "-functionSyntax:4"} Seq { } /* Optimized implementation of Flatten(Map(f, xs)). */ - function {:opaque} FlatMap(f: (T ~> seq), xs: seq): (result: seq) + function FlatMap(f: (T ~> seq), xs: seq): (result: seq) requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) - ensures result == Flatten(Map(f, xs)); reads set i, o | 0 <= i < |xs| && o in f.reads(xs[i]) :: o { Flatten(Map(f, xs)) diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 4e5889f8..48297e49 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -21,8 +21,8 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties { forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d < bracketed :: pd0.requires(d)) - && (forall d | d < bracketed :: pd1.requires(d)) - && (forall d | d < bracketed :: pd0(d) == pd1(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) { calc { Spec.Bracketed(bracketed, pd0); diff --git a/src/JSON/Utils/Seq.dfy b/src/JSON/Utils/Seq.dfy index 18d336f4..b051d62c 100644 --- a/src/JSON/Utils/Seq.dfy +++ b/src/JSON/Utils/Seq.dfy @@ -6,15 +6,18 @@ module {:options "-functionSyntax:4"} JSON.Utils.Seq { {} lemma Assoc(a: seq, b: seq, c: seq) + // `a + b + c` is parsed as `(a + b) + c` ensures a + b + c == a + (b + c) {} lemma Assoc'(a: seq, b: seq, c: seq) + // `a + b + c` is parsed as `(a + b) + c` ensures a + (b + c) == a + b + c {} lemma Assoc2(a: seq, b: seq, c: seq, d: seq) + // `a + b + c + d` is parsed as `((a + b) + c) + d` ensures a + b + c + d == a + (b + c + d) {} } From 62d5c89a0c40441e292b16627170f3dea60c13da Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 May 2023 13:36:36 -0700 Subject: [PATCH 81/84] Removing stale TODOs --- src/JSON/Deserializer.dfy | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/JSON/Deserializer.dfy b/src/JSON/Deserializer.dfy index 545612cc..bc158985 100644 --- a/src/JSON/Deserializer.dfy +++ b/src/JSON/Deserializer.dfy @@ -67,7 +67,6 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { hd as uint16 } - // TODO: Verify this function function {:tailrecursion} {:vcs_split_on_every_assert} Unescape(str: seq, start: nat := 0, prefix: seq := []): DeserializationResult> decreases |str| - start { // Assumes UTF-16 strings @@ -106,7 +105,6 @@ module {:options "-functionSyntax:4"} JSON.Deserializer { } function String(js: Grammar.jstring): DeserializationResult { - // TODO Optimize with a function by method var asUtf32 :- FromUTF8Checked(js.contents.Bytes()).ToResult'(DeserializationError.InvalidUnicode); var asUint16 :- ToUTF16Checked(asUtf32).ToResult'(DeserializationError.InvalidUnicode); var unescaped :- Unescape(asUint16); From da1b5ac93ba1cec8819861707c7fd8ff982a6a68 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 May 2023 13:39:29 -0700 Subject: [PATCH 82/84] =?UTF-8?q?Mikael=E2=80=99s=20calc=20rewrite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/JSON/ZeroCopy/Deserializer.dfy | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/JSON/ZeroCopy/Deserializer.dfy b/src/JSON/ZeroCopy/Deserializer.dfy index 8b48099b..1f4a0adb 100644 --- a/src/JSON/ZeroCopy/Deserializer.dfy +++ b/src/JSON/ZeroCopy/Deserializer.dfy @@ -163,14 +163,18 @@ module {:options "-functionSyntax:4"} JSON.ZeroCopy.Deserializer { ensures sp.StrictlySplitFrom?(cs, BracketedSpec) { var sp := SP(Grammar.Bracketed(open.t, elems.t, close.t), close.cs); - assert cs.Bytes() == Spec.Bracketed(sp.t, SuffixedElementSpec) + close.cs.Bytes() by { - assert cs.Bytes() == Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + Spec.Structural(close.t, SpecView) + close.cs.Bytes() by { - assert cs.Bytes() == Spec.Structural(open.t, SpecView) + open.cs.Bytes(); - assert open.cs.Bytes() == SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); - assert elems.cs.Bytes() == Spec.Structural(close.t, SpecView) + close.cs.Bytes(); - Seq.Assoc'(Spec.Structural(open.t, SpecView), SuffixedElementsSpec(elems.t), elems.cs.Bytes()); - Seq.Assoc'(Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t), Spec.Structural(close.t, SpecView), close.cs.Bytes()); - } + calc { + cs.Bytes(); + Spec.Structural(open.t, SpecView) + open.cs.Bytes(); + { assert open.cs.Bytes() == SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); } + Spec.Structural(open.t, SpecView) + (SuffixedElementsSpec(elems.t) + elems.cs.Bytes()); + { Seq.Assoc'(Spec.Structural(open.t, SpecView), SuffixedElementsSpec(elems.t), elems.cs.Bytes()); } + Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + elems.cs.Bytes(); + { assert elems.cs.Bytes() == Spec.Structural(close.t, SpecView) + close.cs.Bytes(); } + Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + (Spec.Structural(close.t, SpecView) + close.cs.Bytes()); + { Seq.Assoc'(Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t), Spec.Structural(close.t, SpecView), close.cs.Bytes()); } + Spec.Structural(open.t, SpecView) + SuffixedElementsSpec(elems.t) + Spec.Structural(close.t, SpecView) + close.cs.Bytes(); + Spec.Bracketed(sp.t, SuffixedElementSpec) + close.cs.Bytes(); } assert sp.StrictlySplitFrom?(cs, BracketedSpec); sp From 45055e160f324c3c4afb2dad099278aab4d1af13 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 May 2023 13:46:35 -0700 Subject: [PATCH 83/84] Formatting --- src/JSON/ConcreteSyntax.SpecProperties.dfy | 6 +++--- src/JSON/Utils/Seq.dfy | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 48297e49..81ecf4e2 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -4,7 +4,7 @@ include "ConcreteSyntax.Spec.dfy" module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties -// Some useful properties about the functions used in `ConcreteSyntax.Spec`. + // Some useful properties about the functions used in `ConcreteSyntax.Spec`. { import opened BoundedInts @@ -21,8 +21,8 @@ module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties { forall pd0: Suffixed --> bytes, pd1: Suffixed --> bytes | && (forall d | d < bracketed :: pd0.requires(d)) - && (forall d | d < bracketed :: pd1.requires(d)) - && (forall d | d < bracketed :: pd0(d) == pd1(d)) + && (forall d | d < bracketed :: pd1.requires(d)) + && (forall d | d < bracketed :: pd0(d) == pd1(d)) { calc { Spec.Bracketed(bracketed, pd0); diff --git a/src/JSON/Utils/Seq.dfy b/src/JSON/Utils/Seq.dfy index b051d62c..3293db29 100644 --- a/src/JSON/Utils/Seq.dfy +++ b/src/JSON/Utils/Seq.dfy @@ -6,18 +6,18 @@ module {:options "-functionSyntax:4"} JSON.Utils.Seq { {} lemma Assoc(a: seq, b: seq, c: seq) - // `a + b + c` is parsed as `(a + b) + c` + // `a + b + c` is parsed as `(a + b) + c` ensures a + b + c == a + (b + c) {} lemma Assoc'(a: seq, b: seq, c: seq) - // `a + b + c` is parsed as `(a + b) + c` + // `a + b + c` is parsed as `(a + b) + c` ensures a + (b + c) == a + b + c {} lemma Assoc2(a: seq, b: seq, c: seq, d: seq) - // `a + b + c + d` is parsed as `((a + b) + c) + d` + // `a + b + c + d` is parsed as `((a + b) + c) + d` ensures a + b + c + d == a + (b + c + d) {} } From a58e01d0c2da7a55ab561618061a8562bbe1f425 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 May 2023 14:18:15 -0700 Subject: [PATCH 84/84] I said, FORMATTING --- src/JSON/ConcreteSyntax.SpecProperties.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/JSON/ConcreteSyntax.SpecProperties.dfy b/src/JSON/ConcreteSyntax.SpecProperties.dfy index 81ecf4e2..4e5889f8 100644 --- a/src/JSON/ConcreteSyntax.SpecProperties.dfy +++ b/src/JSON/ConcreteSyntax.SpecProperties.dfy @@ -4,7 +4,7 @@ include "ConcreteSyntax.Spec.dfy" module {:options "-functionSyntax:4"} JSON.ConcreteSyntax.SpecProperties - // Some useful properties about the functions used in `ConcreteSyntax.Spec`. +// Some useful properties about the functions used in `ConcreteSyntax.Spec`. { import opened BoundedInts