Special token for empty string.

Manifest.toml

@@ -155,9 +155,9 @@ version = "0.17.2"
 
 [[deps.HDF5_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"]
-git-tree-sha1 = "6384c847ff5056c5624e30e75b3ca48902cae0ac"
+git-tree-sha1 = "82a471768b513dc39e471540fdadc84ff80ff997"
 uuid = "0234f1f7-429e-5d53-9886-15a909be8d59"
-version = "1.14.3+2"
+version = "1.14.3+3"
 
 [[deps.Hwloc_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]

docs/v0.1.0/.documenter-siteinfo.json

@@ -1 +1 @@
-{"documenter":{"julia_version":"1.10.2","generation_timestamp":"2024-04-10T08:11:56","documenter_version":"1.3.0"}}
+{"documenter":{"julia_version":"1.10.2","generation_timestamp":"2024-04-10T20:22:24","documenter_version":"1.3.0"}}

docs/v0.1.0/tokens.html

@@ -727,7 +727,11 @@ <h2 id="Tokenization">
 <a href="tokens.html#Daf.Tokens.Token">
 <code>Token
 </code>
-</a>. Anything else is reported as an invalid character.
+</a>. As a special case, 
+<code>&#39;&#39;
+</code> is converted to an empty string, which is otherwise impossible to represent (write 
+<code>\&#39;\&#39;
+</code> to prevent this). Anything else is reported as an invalid character.
 </p>
 <div class="admonition is-info">
 <header class="admonition-header">Note

src/tokens.jl

@@ -170,8 +170,9 @@ into a single line (discarding line breaks and comments), and the squashed expre
 is reasonable for dealing with `Daf` queries which are expected to be "relatively simple".
 
 When tokenizing, we discard the spaces. Anything that matches the [`VALUE_REGEX`](@ref) is considered to be a value
-[`Token`](@ref). Anything that matches the `operators` is considered to be an operator [`Token`](@ref). Anything else is
-reported as an invalid character.
+[`Token`](@ref). Anything that matches the `operators` is considered to be an operator [`Token`](@ref). As a special
+case, `''` is converted to an empty string, which is otherwise impossible to represent (write `\\'\\'` to prevent this).
+Anything else is reported as an invalid character.
 
 !!! note
 
@@ -195,6 +196,13 @@ function tokenize(string::AbstractString, operators::Regex)::Vector{Token}
             continue
         end
 
+        if startswith(rest_of_string, "''")
+            push!(tokens, Token(false, "", token_index, first_index, first_index + 2, "''"))
+            first_index += 2
+            rest_of_string = rest_of_string[3:end]
+            continue
+        end
+
         value = match(VALUE_REGEX, rest_of_string)
         if value !== nothing
             @assert value.offset == 1

test/tokens.jl

@@ -117,6 +117,8 @@ nested_test("tokens") do
         nested_test("single") do
             @test token_strings("1") == String["1"]
             @test token_strings("x") == String["x"]
+            @test token_strings("''") == String[""]
+            @test token_strings("\\'\\'") == String["''"]
         end
 
         nested_test("multiple") do