From ab01585af0518579f97d2cc5aff045a07b7a2aee Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Sun, 5 Jul 2020 17:52:03 +0800 Subject: [PATCH 1/2] split tests into multiple files This only organizes codes without any modification on the test cases --- test/equality_metrics.jl | 0 test/fileio.jl | 0 test/references/{ => dataframe}/dataframe.csv | 0 test/references/{ => image}/camera.png | Bin test/references/{ => image}/camera.sha256 | 0 test/references/{ => image}/camera.txt | 0 test/references/{ => image}/lena.sha256 | 0 test/references/{ => image}/lena.txt | 0 test/references/{ => string}/ansii.txt | 0 test/references/{ => string}/number1.sha256 | 0 test/references/{ => string}/string1.nottxt | 0 test/references/{ => string}/string1.sha256 | 0 test/references/{ => string}/string1.txt | 0 test/references/{ => string}/string2.sha256 | 0 test/references/{ => string}/string2.txt | 0 test/references/{ => string}/string3.txt | 0 test/references/{ => string}/string4.txt | 0 test/references/{ => string}/string5.txt | 0 test/references/{ => string}/string6.txt | 0 test/render.jl | 0 test/runtests.jl | 152 +++--------------- test/test_reference.jl | 41 +++++ test/testutils.jl | 1 + test/types/dataframe.jl | 7 + test/types/image.jl | 29 ++++ test/types/number_array.jl | 1 + test/types/string.jl | 55 +++++++ test/utils.jl | 16 ++ 28 files changed, 170 insertions(+), 132 deletions(-) create mode 100644 test/equality_metrics.jl create mode 100644 test/fileio.jl rename test/references/{ => dataframe}/dataframe.csv (100%) rename test/references/{ => image}/camera.png (100%) rename test/references/{ => image}/camera.sha256 (100%) rename test/references/{ => image}/camera.txt (100%) rename test/references/{ => image}/lena.sha256 (100%) rename test/references/{ => image}/lena.txt (100%) rename test/references/{ => string}/ansii.txt (100%) rename test/references/{ => string}/number1.sha256 (100%) rename test/references/{ => string}/string1.nottxt (100%) rename test/references/{ => string}/string1.sha256 (100%) rename test/references/{ => string}/string1.txt (100%) rename test/references/{ => string}/string2.sha256 (100%) rename test/references/{ => string}/string2.txt (100%) rename test/references/{ => string}/string3.txt (100%) rename test/references/{ => string}/string4.txt (100%) rename test/references/{ => string}/string5.txt (100%) rename test/references/{ => string}/string6.txt (100%) create mode 100644 test/render.jl create mode 100644 test/test_reference.jl create mode 100644 test/testutils.jl create mode 100644 test/types/dataframe.jl create mode 100644 test/types/image.jl create mode 100644 test/types/number_array.jl create mode 100644 test/types/string.jl create mode 100644 test/utils.jl diff --git a/test/equality_metrics.jl b/test/equality_metrics.jl new file mode 100644 index 0000000..e69de29 diff --git a/test/fileio.jl b/test/fileio.jl new file mode 100644 index 0000000..e69de29 diff --git a/test/references/dataframe.csv b/test/references/dataframe/dataframe.csv similarity index 100% rename from test/references/dataframe.csv rename to test/references/dataframe/dataframe.csv diff --git a/test/references/camera.png b/test/references/image/camera.png similarity index 100% rename from test/references/camera.png rename to test/references/image/camera.png diff --git a/test/references/camera.sha256 b/test/references/image/camera.sha256 similarity index 100% rename from test/references/camera.sha256 rename to test/references/image/camera.sha256 diff --git a/test/references/camera.txt b/test/references/image/camera.txt similarity index 100% rename from test/references/camera.txt rename to test/references/image/camera.txt diff --git a/test/references/lena.sha256 b/test/references/image/lena.sha256 similarity index 100% rename from test/references/lena.sha256 rename to test/references/image/lena.sha256 diff --git a/test/references/lena.txt b/test/references/image/lena.txt similarity index 100% rename from test/references/lena.txt rename to test/references/image/lena.txt diff --git a/test/references/ansii.txt b/test/references/string/ansii.txt similarity index 100% rename from test/references/ansii.txt rename to test/references/string/ansii.txt diff --git a/test/references/number1.sha256 b/test/references/string/number1.sha256 similarity index 100% rename from test/references/number1.sha256 rename to test/references/string/number1.sha256 diff --git a/test/references/string1.nottxt b/test/references/string/string1.nottxt similarity index 100% rename from test/references/string1.nottxt rename to test/references/string/string1.nottxt diff --git a/test/references/string1.sha256 b/test/references/string/string1.sha256 similarity index 100% rename from test/references/string1.sha256 rename to test/references/string/string1.sha256 diff --git a/test/references/string1.txt b/test/references/string/string1.txt similarity index 100% rename from test/references/string1.txt rename to test/references/string/string1.txt diff --git a/test/references/string2.sha256 b/test/references/string/string2.sha256 similarity index 100% rename from test/references/string2.sha256 rename to test/references/string/string2.sha256 diff --git a/test/references/string2.txt b/test/references/string/string2.txt similarity index 100% rename from test/references/string2.txt rename to test/references/string/string2.txt diff --git a/test/references/string3.txt b/test/references/string/string3.txt similarity index 100% rename from test/references/string3.txt rename to test/references/string/string3.txt diff --git a/test/references/string4.txt b/test/references/string/string4.txt similarity index 100% rename from test/references/string4.txt rename to test/references/string/string4.txt diff --git a/test/references/string5.txt b/test/references/string/string5.txt similarity index 100% rename from test/references/string5.txt rename to test/references/string/string5.txt diff --git a/test/references/string6.txt b/test/references/string/string6.txt similarity index 100% rename from test/references/string6.txt rename to test/references/string/string6.txt diff --git a/test/render.jl b/test/render.jl new file mode 100644 index 0000000..e69de29 diff --git a/test/runtests.jl b/test/runtests.jl index f53377f..7e1b27f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using Test using ImageInTerminal, TestImages, ImageCore, ImageTransformations +using DataFrames, CSVFiles using Random if isinteractive() @@ -9,145 +10,32 @@ else @info ("Ten tests should correctly report failure in the transcript" * " (but not the test summary).") end + # check for ambiguities refambs = detect_ambiguities(ImageInTerminal, Base, Core) using ReferenceTests ambs = detect_ambiguities(ReferenceTests, ImageInTerminal, Base, Core) -strip_summary(content::String) = join(split(content, "\n")[2:end], "\n") - -@testset "ReferenceTests" begin - -@test Set(setdiff(ambs, refambs)) == Set{Tuple{Method,Method}}() - -# load/create some example images -lena = testimage("lena_color_256") -camera = testimage("cameraman") -cameras = similar(camera, size(camera)..., 2) -copyto!(view(cameras,:,:,1), camera) -copyto!(view(cameras,:,:,2), camera) -square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9] -rgb_rect = rand(RGB{N0f8}, 2, 3) - -@testset "io2str" begin - @test_throws LoadError eval(@macroexpand @io2str(::IO)) - @test_throws ArgumentError @io2str(2) - @test_throws ArgumentError @io2str(string(2)) - @test @io2str(print(::IO, "foo")) == "foo" - @test @io2str(println(::IO, "foo")) == "foo\n" - @test @io2str(show(::IO, "foo")) == "\"foo\"" - A = ones(30,30) - @test @io2str(show(IOContext(::IO, :limit => true, :displaysize => (5,5)), A)) == "[1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0; … ; 1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0]" -end - -@testset "withcolor" begin - @test_throws ArgumentError @withcolor throw(ArgumentError("foo")) - @test @withcolor Base.have_color == true - @test @withcolor @io2str(printstyled(IOContext(::IO, :color => true), "test", color=:green)) == "\e[32mtest\e[39m" -end - -@testset "string as txt" begin - foo = "foo" - @test_reference "references/string1.txt" foo * "bar" - @test_reference "references/string1.txt" [foo * "bar"] - A = ones(30,30) - @test_reference "references/string2.txt" @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A) - @test_reference "references/string3.txt" 1337 - @test_reference "references/string3.txt" 1338 by=(ref, x)->isapprox(ref, x; atol=10) - @test_reference "references/string4.txt" strip_summary(@io2str show(::IO, MIME"text/plain"(), Int64.(collect(1:5)))) - - # ignore CRLF/LF differences - @test_reference "references/string5.txt" """ - This is a\r - multiline string that does not end with a new line.""" - @test_reference "references/string5.txt" """ - This is a - multiline string that does not end with a new line.""" +const refroot = joinpath(@__DIR__(), "references") - @test_reference "references/string6.txt" """ - This on the other hand is a - multiline string that does indeed end with a new line. - """ - - @test_throws ErrorException @test_reference "references/string1.txt" "intentionally wrong to check that this message prints" - @test_throws ErrorException @test_reference "references/string5.txt" """ - This is an incorrect - multiline string that does not end with a new line.""" -end +test_files = [ + "equality_metrics.jl", + "fileio.jl", + "utils.jl", + "render.jl", + "test_reference.jl", +] -@testset "string as unknown file type" begin - @test_reference "references/string1.nottxt" "This is not a .txt file, but it should be treated as such.\n" -end - -@testset "images as txt using ImageInTerminal" begin - #@test_throws MethodError @test_reference "references/fail.txt" rand(2,2) - - @test_reference "references/camera.txt" camera size=(5,10) - @test_reference "references/lena.txt" lena -end - -@testset "plain ansi string" begin - @test_reference( - "references/ansii.txt", - @io2str(printstyled(IOContext(::IO, :color=>true), "this should be blue", color=:blue)), - render = ReferenceTests.BeforeAfterFull() - ) - @test_throws ErrorException @test_reference( - "references/ansii.txt", - @io2str(printstyled(IOContext(::IO, :color=>true), "this should be red", color=:red)), - render = ReferenceTests.BeforeAfterFull() - ) -end - -@testset "string as SHA" begin - @test_reference "references/number1.sha256" 1337 - foo = "foo" - @test_reference "references/string1.sha256" foo * "bar" - A = ones(30,30) - @test_reference "references/string2.sha256" @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A) -end - -@testset "images as SHA" begin - @test_reference "references/camera.sha256" camera - @test_reference "references/lena.sha256" convert(Matrix{RGB{Float64}}, lena) -end - -@testset "images as PNG" begin - @test_reference "references/camera.png" imresize(camera, (64,64)) - @test_reference "references/camera.png" imresize(camera, (64,64)) by=psnr_equality(25) - @test_throws ErrorException @test_reference "references/camera.png" imresize(lena, (64,64)) - @test_throws Exception @test_reference "references/camera.png" camera # unequal size -end - -using DataFrames, CSVFiles -@testset "DataFrame as CSV" begin - @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["a","b","c"]) - @test_throws ErrorException @test_reference "references/dataframe.csv" DataFrame(v1=[1,2,3], v2=["c","b","c"]) - -end - -@testset "Create new $ext" for (ext, val) in ( - (".csv", DataFrame(v1=[1,2,3], v2=["c","b","c"])), - (".png", imresize(camera, (64,64))), - (".txt", "Lorem ipsum dolor sit amet, labore et dolore magna aliqua."), -) - newfilename = "references/newfilename.$ext" - @assert !isfile(newfilename) - @test_reference newfilename val # this should create it - @test isfile(newfilename) # Was created - @test_reference newfilename val # Matches expected content - rm(newfilename, force=true) -end - -@testset "Create new image as txt" begin - # This is a sperate testset as need to use the `size` argument to ``@test_reference` - newfilename = "references/new_camera.txt" - @assert !isfile(newfilename) - @test_reference newfilename camera size=(5,10) # this should create it - @test isfile(newfilename) # Was created - @test_reference newfilename camera size=(5,10) # Matches expected content - rm(newfilename, force=true) -end +include("testutils.jl") +@testset "ReferenceTests" begin + @test Set(setdiff(ambs, refambs)) == Set{Tuple{Method,Method}}() + + for file in test_files + filename = first(splitext(file)) + @testset "File: $filename" begin + include(file) + end + end end # top level testset diff --git a/test/test_reference.jl b/test/test_reference.jl new file mode 100644 index 0000000..66a89d3 --- /dev/null +++ b/test/test_reference.jl @@ -0,0 +1,41 @@ +type_files = [ + "string.jl", + "image.jl", + "number_array.jl", + "dataframe.jl" +] + +for file in type_files + type = first(splitext(file)) + @testset "Type: $type" begin + include(joinpath("types", file)) + end +end + +# TODO: split this testset into previous files +@testset "Reference regeneration" begin + camera = testimage("cameraman") + + @testset "Create new $ext" for (ext, val) in ( + (".csv", DataFrame(v1=[1,2,3], v2=["c","b","c"])), + (".png", imresize(camera, (64,64))), + (".txt", "Lorem ipsum dolor sit amet, labore et dolore magna aliqua."), + ) + newfilename = joinpath(refroot, "newfilename.$ext") + @assert !isfile(newfilename) + @test_reference newfilename val # this should create it + @test isfile(newfilename) # Was created + @test_reference newfilename val # Matches expected content + rm(newfilename, force=true) + end + + @testset "Create new image as txt" begin + # This is a sperate testset as need to use the `size` argument to ``@test_reference` + newfilename = joinpath(refroot, "new_camera.txt") + @assert !isfile(newfilename) + @test_reference newfilename camera size=(5,10) # this should create it + @test isfile(newfilename) # Was created + @test_reference newfilename camera size=(5,10) # Matches expected content + rm(newfilename, force=true) + end +end diff --git a/test/testutils.jl b/test/testutils.jl new file mode 100644 index 0000000..8208b14 --- /dev/null +++ b/test/testutils.jl @@ -0,0 +1 @@ +strip_summary(content::String) = join(split(content, "\n")[2:end], "\n") diff --git a/test/types/dataframe.jl b/test/types/dataframe.jl new file mode 100644 index 0000000..45472ba --- /dev/null +++ b/test/types/dataframe.jl @@ -0,0 +1,7 @@ +refdir = joinpath(refroot, "dataframe") + +@testset "DataFrame as CSV" begin + @test_reference joinpath(refdir, "dataframe.csv") DataFrame(v1=[1,2,3], v2=["a","b","c"]) + @test_throws ErrorException @test_reference joinpath(refdir, "dataframe.csv") DataFrame(v1=[1,2,3], v2=["c","b","c"]) + +end diff --git a/test/types/image.jl b/test/types/image.jl new file mode 100644 index 0000000..42e8940 --- /dev/null +++ b/test/types/image.jl @@ -0,0 +1,29 @@ +# load/create some example images +refdir = joinpath(refroot, "image") + +lena = testimage("lena_color_256") +camera = testimage("cameraman") +cameras = similar(camera, size(camera)..., 2) +copyto!(view(cameras,:,:,1), camera) +copyto!(view(cameras,:,:,2), camera) +square = Gray{N0f8}[0.1 0.2 0.3; 0.4 0.5 0.6; 0.7 0.6 0.9] +rgb_rect = rand(RGB{N0f8}, 2, 3) + +@testset "images as txt using ImageInTerminal" begin + #@test_throws MethodError @test_reference "references/fail.txt" rand(2,2) + + @test_reference joinpath(refdir, "camera.txt") camera size=(5,10) + @test_reference joinpath(refdir, "lena.txt") lena +end + +@testset "images as SHA" begin + @test_reference joinpath(refdir, "camera.sha256") camera + @test_reference joinpath(refdir, "lena.sha256") convert(Matrix{RGB{Float64}}, lena) +end + +@testset "images as PNG" begin + @test_reference joinpath(refdir, "camera.png") imresize(camera, (64,64)) + @test_reference joinpath(refdir, "camera.png") imresize(camera, (64,64)) by=psnr_equality(25) + @test_throws ErrorException @test_reference joinpath(refdir, "camera.png") imresize(lena, (64,64)) + @test_throws Exception @test_reference joinpath(refdir, "camera.png") camera # unequal size +end diff --git a/test/types/number_array.jl b/test/types/number_array.jl new file mode 100644 index 0000000..7c3481e --- /dev/null +++ b/test/types/number_array.jl @@ -0,0 +1 @@ +refdir = joinpath(refroot, "number_array") diff --git a/test/types/string.jl b/test/types/string.jl new file mode 100644 index 0000000..8f6d1e1 --- /dev/null +++ b/test/types/string.jl @@ -0,0 +1,55 @@ +refdir = joinpath(refroot, "string") + +@testset "string as unknown file type" begin + @test_reference joinpath(refdir, "string1.nottxt") "This is not a .txt file, but it should be treated as such.\n" +end + +@testset "string as txt" begin + foo = "foo" + @test_reference joinpath(refdir, "string1.txt") foo * "bar" + @test_reference joinpath(refdir, "string1.txt") [foo * "bar"] + A = ones(30,30) + @test_reference joinpath(refdir, "string2.txt") @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A) + @test_reference joinpath(refdir, "string3.txt") 1337 + @test_reference joinpath(refdir, "string3.txt") 1338 by=(ref, x)->isapprox(ref, x; atol=10) + @test_reference joinpath(refdir, "string4.txt") strip_summary(@io2str show(::IO, MIME"text/plain"(), Int64.(collect(1:5)))) + + # ignore CRLF/LF differences + @test_reference joinpath(refdir, "string5.txt") """ + This is a\r + multiline string that does not end with a new line.""" + @test_reference joinpath(refdir, "string5.txt") """ + This is a + multiline string that does not end with a new line.""" + + @test_reference joinpath(refdir, "string6.txt") """ + This on the other hand is a + multiline string that does indeed end with a new line. + """ + + @test_throws ErrorException @test_reference joinpath(refdir, "string1.txt") "intentionally wrong to check that this message prints" + @test_throws ErrorException @test_reference joinpath(refdir, "string5.txt") """ + This is an incorrect + multiline string that does not end with a new line.""" +end + +@testset "string as SHA" begin + @test_reference joinpath(refdir, "number1.sha256") 1337 + foo = "foo" + @test_reference joinpath(refdir, "string1.sha256") foo * "bar" + A = ones(30,30) + @test_reference joinpath(refdir, "string2.sha256") @io2str show(IOContext(::IO, :limit=>true, :displaysize=>(5,5)), A) +end + +@testset "plain ansi string" begin + @test_reference( + joinpath(refdir, "ansii.txt"), + @io2str(printstyled(IOContext(::IO, :color=>true), "this should be blue", color=:blue)), + render = ReferenceTests.BeforeAfterFull() + ) + @test_throws ErrorException @test_reference( + joinpath(refdir, "ansii.txt"), + @io2str(printstyled(IOContext(::IO, :color=>true), "this should be red", color=:red)), + render = ReferenceTests.BeforeAfterFull() + ) +end diff --git a/test/utils.jl b/test/utils.jl new file mode 100644 index 0000000..c759ada --- /dev/null +++ b/test/utils.jl @@ -0,0 +1,16 @@ +@testset "io2str" begin + @test_throws LoadError eval(@macroexpand @io2str(::IO)) + @test_throws ArgumentError @io2str(2) + @test_throws ArgumentError @io2str(string(2)) + @test @io2str(print(::IO, "foo")) == "foo" + @test @io2str(println(::IO, "foo")) == "foo\n" + @test @io2str(show(::IO, "foo")) == "\"foo\"" + A = ones(30,30) + @test @io2str(show(IOContext(::IO, :limit => true, :displaysize => (5,5)), A)) == "[1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0; … ; 1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0]" +end + +@testset "withcolor" begin + @test_throws ArgumentError @withcolor throw(ArgumentError("foo")) + @test @withcolor Base.have_color == true + @test @withcolor @io2str(printstyled(IOContext(::IO, :color => true), "test", color=:green)) == "\e[32mtest\e[39m" +end From afc31d4e4806fd03557c72b639192d1d54c88ac9 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Tue, 7 Jul 2020 12:16:15 +0800 Subject: [PATCH 2/2] rename _convert to maybe_encode (#64) * rename _convert to maybe_encode * add tests for maybe_encode * ignore CRLF/LF differences Co-authored-by: Lyndon White --- src/fileio.jl | 52 ++++-- src/test_reference.jl | 2 +- test/fileio.jl | 165 ++++++++++++++++++ .../references/fileio/dataframe_as_sha256.txt | 1 + test/references/fileio/gray_1d_as_sha256.txt | 1 + test/references/fileio/gray_2d_as_sha256.txt | 1 + test/references/fileio/gray_2d_as_txt.txt | 1 + test/references/fileio/gray_3d_as_sha256.txt | 1 + test/references/fileio/rgb_1d_as_sha256.txt | 1 + test/references/fileio/rgb_2d_as_sha256.txt | 1 + test/references/fileio/rgb_2d_as_txt.txt | 1 + test/references/fileio/rgb_3d_as_sha256.txt | 1 + 12 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 test/references/fileio/dataframe_as_sha256.txt create mode 100644 test/references/fileio/gray_1d_as_sha256.txt create mode 100644 test/references/fileio/gray_2d_as_sha256.txt create mode 100644 test/references/fileio/gray_2d_as_txt.txt create mode 100644 test/references/fileio/gray_3d_as_sha256.txt create mode 100644 test/references/fileio/rgb_1d_as_sha256.txt create mode 100644 test/references/fileio/rgb_2d_as_sha256.txt create mode 100644 test/references/fileio/rgb_2d_as_txt.txt create mode 100644 test/references/fileio/rgb_3d_as_sha256.txt diff --git a/src/fileio.jl b/src/fileio.jl index 75c0d09..f90b7a8 100644 --- a/src/fileio.jl +++ b/src/fileio.jl @@ -9,7 +9,7 @@ function loadfile(T, file::File) end function loadfile(T, file::TextFile) - replace(read(file.filename, String), "\r"=>"") # ignore CRLF/LF difference + _ignore_CR(read(file.filename, String)) end function loadfile(::Type{<:Number}, file::File{format"TXT"}) @@ -24,7 +24,7 @@ function savefile(file::TextFile, content) write(file.filename, string(content)) end -function query_extended(filename) +function query_extended(filename::AbstractString) file, ext = splitext(filename) # TODO: make this less hacky if uppercase(ext) == ".SHA256" @@ -38,20 +38,28 @@ function query_extended(filename) res end +# Some target formats are not supported by FileIO and thus require an encoding/compression process +# before saving. For other formats, we should trust IO backends and make as few changes as possible. +# Otherwise, reference becomes unfaithful. The encoding process helps making the actual data matches +# the reference data, which is loaded from reference file via IO backends. +# +# TODO: split `maybe_encode` to `maybe_preprocess` and `maybe_encode` """ - _convert(T::Type{<:DataFormat}, x; kw...) -> out + maybe_encode(T::Type{<:DataFormat}, x; kw...) -> out -Convert `x` to a validate content for file data format `T`. +If needed, encode `x` to a valid content that matches format `T`. + +If there is no known method to encode `x`, then it directly return `x` without warning. """ -_convert(::Type{<:DataFormat}, x; kw...) = x +maybe_encode(::Type{<:DataFormat}, x; kw...) = x # plain TXT -_convert(::Type{DataFormat{:TXT}}, x; kw...) = replace(string(x), "\r"=>"") # ignore CRLF/LF difference -_convert(::Type{DataFormat{:TXT}}, x::Number; kw...) = x -function _convert(::Type{DataFormat{:TXT}}, x::AbstractArray{<:AbstractString}; kw...) - return join(x, '\n') -end -function _convert( +maybe_encode(::Type{DataFormat{:TXT}}, x; kw...) = _ignore_CR(string(x)) +maybe_encode(::Type{DataFormat{:TXT}}, x::AbstractArray{<:AbstractString}; kw...) = _join(x) +maybe_encode(::Type{DataFormat{:TXT}}, x::AbstractString; kw...) = _ignore_CR(x) +maybe_encode(::Type{DataFormat{:TXT}}, x::Number; kw...) = x # TODO: Change this to string(x) ? + +function maybe_encode( ::Type{DataFormat{:TXT}}, img::AbstractArray{<:Colorant}; size = (20,40), kw...) @@ -65,11 +73,25 @@ function _convert( end # SHA256 -_convert(::Type{DataFormat{:SHA256}}, x; kw...) = bytes2hex(sha256(string(x))) -function _convert(::Type{DataFormat{:SHA256}}, img::AbstractArray{<:Colorant}; kw...) +maybe_encode(::Type{DataFormat{:SHA256}}, x; kw...) = _sha256(string(x)) +maybe_encode(::Type{DataFormat{:SHA256}}, x::AbstractString) = _sha256(_ignore_CR(x)) +maybe_encode(::Type{DataFormat{:SHA256}}, x::AbstractArray{<:AbstractString}) = _sha256(_join(x)) +function maybe_encode(::Type{DataFormat{:SHA256}}, img::AbstractArray{<:Colorant}; kw...) # encode image into SHA256 - size_str = bytes2hex(sha256(reinterpret(UInt8,[map(Int64,size(img))...]))) - img_str = bytes2hex(sha256(reinterpret(UInt8,vec(rawview(channelview(img)))))) + size_str = _sha256(reinterpret(UInt8,[map(Int64,size(img))...])) + img_str = _sha256(reinterpret(UInt8,vec(rawview(channelview(img))))) return size_str * img_str end + +# Helpers +_join(x::AbstractArray{<:AbstractString}) = _ignore_CR(join(x, "\n")) +_sha256(x) = bytes2hex(sha256(x)) +""" + _ignore_CR(x::AbstractString) + +Ignore the CRLF(`\\r\\n`) and LF(`\\n`) difference by removing `\\r` from the given string. + +CRLF format is widely used by Windows while LF format is mainly used by Linux. +""" +_ignore_CR(x::AbstractString) = replace(x, "\r\n"=>"\n") # issue #39 diff --git a/src/test_reference.jl b/src/test_reference.jl index 211f510..f70873c 100644 --- a/src/test_reference.jl +++ b/src/test_reference.jl @@ -107,7 +107,7 @@ function test_reference( rendermode = default_rendermode(F, raw_actual) end - actual = _convert(F, raw_actual; kw...) + actual = maybe_encode(F, raw_actual; kw...) # preprocessing when reference file doesn't exists if !isfile(path) @info("Reference file for \"$filename\" does not exist. It will be created") diff --git a/test/fileio.jl b/test/fileio.jl index e69de29..be68526 100644 --- a/test/fileio.jl +++ b/test/fileio.jl @@ -0,0 +1,165 @@ +refdir = joinpath(refroot, "fileio") + +@testset "query" begin + check_types = [ + # text types + ("textfile_with_no_extension", format"TXT"), + ("textfile.txt", format"TXT"), + ("textfile.unknown", format"TXT"), + ("textfile.sha256", format"SHA256"), + + # image types + ("imagefile.jpg", format"JPEG"), + ("imagefile.jpeg", format"JPEG"), + ("imagefile.png", format"PNG"), + ("imagefile.tif", format"TIFF"), + ("imagefile.tiff", format"TIFF"), + + # dataframe types + ("dataframe_file.csv", format"CSV") + ] + for (file, fmt) in check_types + @test ReferenceTests.query_extended(file) == File{fmt}(file) + @test ReferenceTests.query_extended(abspath(file)) == File{fmt}(abspath(file)) + end +end + +@testset "maybe_encode" begin + @testset "string" begin + str1 = "Hello world" + str1_sha256 = "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c" + str2 = "Hello\n world" + str2_sha256 = "60b65ab310480818c4289227f2ec68f1714743db8571b4cb190e100c0085be3d" # bytes2hex(SHA.sha256(str2)) + str2_crlf = "Hello\r\n world" + str3 = "Hello\nworld" + str3_sha256 = "46e0ea795802f17d0b340983ca7d7068c94d7d9172ee4daea37a1ab1168649ec" # bytes2hex(SHA.sha256(str3)) + str3_arr1 = ["Hello", "world"] + str3_arr2 = ["Hello" "world"] + str4 = "Hello\n world1\nHello\n world2" + str4_sha256 = "c7dc8b82c3a6fed4afa0c8790a0586b73df0e4f35524efe6810e5d78b6b6a611" # bytes2hex(SHA.sha256(str4)) + str4_arr = ["Hello\r\n world1", "Hello\n world2"] + + # string as plain text + fmt = format"TXT" + # convert should respect whitespaces + @test str1 == ReferenceTests.maybe_encode(fmt, str1) + @test str2 == ReferenceTests.maybe_encode(fmt, str2) + # but ignore CRLF/LF differences + @test str2 == ReferenceTests.maybe_encode(fmt, str2_crlf) + # string arrays are treated as multi-line strings, even for UNKNOWN format + @test str3 == ReferenceTests.maybe_encode(fmt, str3) + @test str3 == ReferenceTests.maybe_encode(fmt, str3_arr1) + @test str3 == ReferenceTests.maybe_encode(fmt, str3_arr2) + # string arrays should ignore CRLF/LF differences, too + @test str4 == ReferenceTests.maybe_encode(fmt, str4_arr) + + # string as SHA256 should also ignore CRLF/LF differences + fmt = format"SHA256" + @test str1_sha256 == ReferenceTests.maybe_encode(fmt, str1) + @test str2_sha256 == ReferenceTests.maybe_encode(fmt, str2) + # but ignore CRLF/LF differences + @test str2_sha256 == ReferenceTests.maybe_encode(fmt, str2_crlf) + # string arrays are treated as multi-line strings, even for UNKNOWN format + @test str3_sha256 == ReferenceTests.maybe_encode(fmt, str3) + @test str3_sha256 == ReferenceTests.maybe_encode(fmt, str3_arr1) + @test str3_sha256 == ReferenceTests.maybe_encode(fmt, str3_arr2) + # string arrays should ignore CRLF/LF differences, too + @test str4_sha256 == ReferenceTests.maybe_encode(fmt, str4_arr) + + # unknown formats + fmt = format"PNG" + for str in (str1, str2, str2_crlf, str3, str3_arr1, str3_arr2) + @test str === ReferenceTests.maybe_encode(fmt, str) + end + end + + @testset "numbers" begin + for num in (0x01, 1, 1.0f0, 1.0) + for fmt in (format"TXT", format"UNKNOWN") + @test num === ReferenceTests.maybe_encode(fmt, num) + end + fmt = format"SHA256" + @test ReferenceTests.maybe_encode(fmt, num) == ReferenceTests.maybe_encode(fmt, string(num)) + end + + + for (fmt, a, ref) in [ + # if target is TXT, convert it to string + (format"TXT", [1, 2], "[1, 2]"), + (format"TXT", [1,2], "[1, 2]"), + (format"TXT", [1;2], "[1, 2]"), + (format"TXT", [1 2], "[1 2]"), + (format"TXT", [1 2; 3 4], "[1 2; 3 4]"), + # if target is Unknown, make no change + (format"UNKNOWN", [1, 2], [1, 2]), + (format"UNKNOWN", [1,2], [1, 2]), + (format"UNKNOWN", [1;2], [1, 2]), + (format"UNKNOWN", [1 2], [1 2]), + (format"UNKNOWN", [1 2; 3 4], [1 2; 3 4]), + ] + @test ref == ReferenceTests.maybe_encode(fmt, a) + end + + for a in [[1, 2], [1 2], [1 2; 3 4]] + fmt = format"SHA256" + @test ReferenceTests.maybe_encode(fmt, a) == ReferenceTests.maybe_encode(fmt, string(a)) + end + + end + + @testset "image" begin + gray_1d = Gray{N0f8}.(0.0:0.1:0.9) + rgb_1d = RGB.(gray_1d) + gray_2d = Gray{N0f8}.(reshape(0.0:0.1:0.9, 2, 5)) + rgb_2d = RGB.(gray_2d) + gray_3d = Gray{N0f8}.(reshape(0.0:0.02:0.95, 2, 4, 6)) + rgb_3d = RGB.(gray_3d) + + # any common image types + for img in (gray_1d, gray_2d, gray_3d, rgb_1d, rgb_2d, rgb_3d) + for fmt in (format"JPEG", format"PNG", format"TIFF", format"UNKNOWN") + @test img === ReferenceTests.maybe_encode(fmt, img) + end + end + + # image as text file + fmt = format"TXT" + # TODO: support n-D image encoding + # @test_reference joinpath(refdir, "gray_1d_as_txt.txt") ReferenceTests.maybe_encode(fmt, gray_1d) + # @test_reference joinpath(refdir, "rgb_1d_as_txt.txt") ReferenceTests.maybe_encode(fmt, rgb_1d) + @test_reference joinpath(refdir, "gray_2d_as_txt.txt") ReferenceTests.maybe_encode(fmt, gray_2d) + @test_reference joinpath(refdir, "rgb_2d_as_txt.txt") ReferenceTests.maybe_encode(fmt, rgb_2d) + # @test_reference joinpath(refdir, "gray_3d_as_txt.txt") ReferenceTests.maybe_encode(fmt, gray_3d) + # @test_reference joinpath(refdir, "rgb_3d_as_txt.txt") ReferenceTests.maybe_encode(fmt, rgb_3d) + + # image as SHA256 + fmt = format"SHA256" + for (file, img) in [ + ("gray_1d", gray_1d), + ("gray_2d", gray_2d), + ("gray_3d", gray_3d), + ("rgb_1d", rgb_1d), + ("rgb_2d", rgb_2d), + ("rgb_3d", rgb_3d) + ] + reffile = joinpath(refdir, "$(file)_as_sha256.txt") + @test_reference reffile ReferenceTests.maybe_encode(fmt, img) + end + end + + # dataframe + @testset "dataframe" begin + df = DataFrame(v1=[1,2,3], v2=["a","b","c"]) + + @test string(df) == ReferenceTests.maybe_encode(format"TXT", df) + for fmt in (format"CSV", format"UNKNOWN") + @test df === ReferenceTests.maybe_encode(fmt, df) + end + + fmt = format"SHA256" + @test_reference joinpath(refdir, "dataframe_as_sha256.txt") ReferenceTests.maybe_encode(fmt, df) + + end +end + +# TODO: savefile & loadfile diff --git a/test/references/fileio/dataframe_as_sha256.txt b/test/references/fileio/dataframe_as_sha256.txt new file mode 100644 index 0000000..37fe296 --- /dev/null +++ b/test/references/fileio/dataframe_as_sha256.txt @@ -0,0 +1 @@ +2cf7c4edcafc27a5eb1b74fb0af704edc0d9bbef91a1b55d3b7350fa4b54cd18 \ No newline at end of file diff --git a/test/references/fileio/gray_1d_as_sha256.txt b/test/references/fileio/gray_1d_as_sha256.txt new file mode 100644 index 0000000..dd28753 --- /dev/null +++ b/test/references/fileio/gray_1d_as_sha256.txt @@ -0,0 +1 @@ +a111f275cc2e7588000001d300a31e76336d15b9d314cd1a1d8f3d3556975eed10ef43c7fcace84c4d0d54b8e92c0c9be2d14a6bf3dd7647254a3cc0c4a04297 \ No newline at end of file diff --git a/test/references/fileio/gray_2d_as_sha256.txt b/test/references/fileio/gray_2d_as_sha256.txt new file mode 100644 index 0000000..465a648 --- /dev/null +++ b/test/references/fileio/gray_2d_as_sha256.txt @@ -0,0 +1 @@ +26cfbb315c316a0b15516434f90284e5011dcb58503fe39eb036bf669bd8233d10ef43c7fcace84c4d0d54b8e92c0c9be2d14a6bf3dd7647254a3cc0c4a04297 \ No newline at end of file diff --git a/test/references/fileio/gray_2d_as_txt.txt b/test/references/fileio/gray_2d_as_txt.txt new file mode 100644 index 0000000..908bd18 --- /dev/null +++ b/test/references/fileio/gray_2d_as_txt.txt @@ -0,0 +1 @@ +▀▀▀▀▀ \ No newline at end of file diff --git a/test/references/fileio/gray_3d_as_sha256.txt b/test/references/fileio/gray_3d_as_sha256.txt new file mode 100644 index 0000000..21b6828 --- /dev/null +++ b/test/references/fileio/gray_3d_as_sha256.txt @@ -0,0 +1 @@ +72307e420b5460c03a1c167060ed336407c26ea74aabf8fab76dd8e9dbe8cbe4baf0f53196e8d5270c0b0b2da82bbbb4676edbb0ebf84ec0dcbd8c0bf4d9af68 \ No newline at end of file diff --git a/test/references/fileio/rgb_1d_as_sha256.txt b/test/references/fileio/rgb_1d_as_sha256.txt new file mode 100644 index 0000000..9708f12 --- /dev/null +++ b/test/references/fileio/rgb_1d_as_sha256.txt @@ -0,0 +1 @@ +a111f275cc2e7588000001d300a31e76336d15b9d314cd1a1d8f3d3556975eedebd6b0ad29dd5402ce5745bb5b48d4c59b7f8da0cdf8d2f287befd9094f6ac89 \ No newline at end of file diff --git a/test/references/fileio/rgb_2d_as_sha256.txt b/test/references/fileio/rgb_2d_as_sha256.txt new file mode 100644 index 0000000..0b6d3ff --- /dev/null +++ b/test/references/fileio/rgb_2d_as_sha256.txt @@ -0,0 +1 @@ +26cfbb315c316a0b15516434f90284e5011dcb58503fe39eb036bf669bd8233debd6b0ad29dd5402ce5745bb5b48d4c59b7f8da0cdf8d2f287befd9094f6ac89 \ No newline at end of file diff --git a/test/references/fileio/rgb_2d_as_txt.txt b/test/references/fileio/rgb_2d_as_txt.txt new file mode 100644 index 0000000..908bd18 --- /dev/null +++ b/test/references/fileio/rgb_2d_as_txt.txt @@ -0,0 +1 @@ +▀▀▀▀▀ \ No newline at end of file diff --git a/test/references/fileio/rgb_3d_as_sha256.txt b/test/references/fileio/rgb_3d_as_sha256.txt new file mode 100644 index 0000000..8f59425 --- /dev/null +++ b/test/references/fileio/rgb_3d_as_sha256.txt @@ -0,0 +1 @@ +72307e420b5460c03a1c167060ed336407c26ea74aabf8fab76dd8e9dbe8cbe45465bcbf50acdbe5600207e3266eedef6548bc4d244e55d7a1af0f1af09e019f \ No newline at end of file