Ordered table

README.md

@@ -19,15 +19,15 @@ Or you can download source manually and copy `src/*` into somewhere on your `pac
 	print(lunajson.encode(t)) -- prints {"Hello":["lunajson",1.5]}
 
 ## API
-### lunajson.decode(jsonstr, [pos, [nullv, [arraylen]]])
-Decode `jsonstr`. If `pos` is specified, it starts decoding from `pos` until the JSON definition ends, otherwise the entire input is parsed as JSON. `null` inside `jsonstr` will be decoded as the optional sentinel value `nullv` if specified, and discarded otherwise. If `arraylen` is true, the length of an array `ary` will be stored in `ary[0]`. This behavior is useful when empty arrays should not be confused with empty objects.
+### lunajson.decode(jsonstr, [pos, [nullv, [arraylen, [preserveorder]]]])
+Decode `jsonstr`. If `pos` is specified, it starts decoding from `pos` until the JSON definition ends, otherwise the entire input is parsed as JSON. `null` inside `jsonstr` will be decoded as the optional sentinel value `nullv` if specified, and discarded otherwise. If `arraylen` is true, the length of an array `ary` will be stored in `ary[0]`. This behavior is useful when empty arrays should not be confused with empty objects. If `preservedorder` is true, keys' orders of JSON objects are preserved by means of `__pairs` metamethod. The standard `pairs` function on Lua 5.2 or later respects this metamethod. On Lua 5.1 and LuaJIT not built with `-DLUAJIT_ENABLE_LUA52COMPAT`, you have to monkey-patch `pairs` function or call `__pairs` metamethod directly.
 
 This function returns the decoded value if `jsonstr` contains valid JSON,  otherwise an error will be raised. If `pos` is specified it also returns the position immediately after the end of decoded JSON.
 
 ### lunajson.encode(value, [nullv])
 Encode `value` into a JSON string and return it. If `nullv` is specified, values equal to `nullv` will be encoded as `null`.
 
-This function encodes a table `t` as a JSON array if a value `t[1]` is present or a number `t[0]` is present. If `t[0]` is present, its value is considered as the length of the array. Then the array may contain `nil` and those will be encoded as `null`. Otherwise, this function scans non `nil` values starting from index 1, up to the first `nil` it finds. When the table `t` is not an array, it is an object and all of its keys must be strings.
+This function encodes a table `t` as a JSON array if a value `t[1]` is present or a number `t[0]` is present. If `t[0]` is present, its value is considered as the length of the array. Then the array may contain `nil` and those will be encoded as `null`. Otherwise, this function scans non `nil` values starting from index 1, up to the first `nil` it finds. When the table `t` is not an array, it is an object and all of its keys must be strings. In that case `__pairs` metamethod is respected, even on Lua 5.1.
 
 ### lunajson.newparser(input, saxtbl)
 ### lunajson.newfileparser(filename, saxtbl)

src/lunajson/decoder.lua

@@ -1,5 +1,5 @@
-local setmetatable, tonumber, tostring =
-      setmetatable, tonumber, tostring
+local rawset, setmetatable, tonumber, tostring =
+      rawset, setmetatable, tonumber, tostring
 local floor, inf =
       math.floor, math.huge
 local mininteger, tointeger =
@@ -22,8 +22,46 @@ end
 local _ENV = nil
 
 
+-- Ordered table support
+--   keylist[metafirstkey] = firstkey
+--   keylist[key] = nextkey
+--   keylist[lastkey] = nil
+local metafirstkey = {}
+local function orderedtable(obj)
+	local keylist = {}
+	local lastkey = metafirstkey
+	local function onext(key2val, key)
+		local val
+		repeat
+			key = keylist[key]
+			if key == nil then
+				return
+			end
+			val = key2val[key]
+		until val ~= nil
+		return key, val
+	end
+	local metatable = {
+		__newindex = function(key2val, key, val)
+			rawset(key2val, key, val)
+			-- do the assignment first in case key == lastkey
+			keylist[lastkey] = key
+			if keylist[key] == nil then
+				lastkey = key
+			else
+				keylist[lastkey] = nil
+			end
+		end,
+		__pairs = function(key2val)
+			return onext, key2val, metafirstkey
+		end
+	}
+	return setmetatable(obj, metatable)
+end
+
+
 local function newdecoder()
-	local json, pos, nullv, arraylen, rec_depth
+	local json, pos, nullv, arraylen, preserveorder, rec_depth
 
 	-- `f` is the temporary for dispatcher[c] and
 	-- the dummy for the first return value of `find`
@@ -405,6 +443,9 @@ local function newdecoder()
 			decode_error('too deeply nested json (> 1000)')
 		end
 		local obj = {}
+		if preserveorder then
+			obj = orderedtable(obj)
+		end
 
 		pos = match(json, '^[ \n\r\t]*()', pos)
 		if byte(json, pos) == 0x7D then  -- check closing bracket '}' which means the object empty
@@ -488,8 +529,8 @@ local function newdecoder()
 	--[[
 		run decoder
 	--]]
-	local function decode(json_, pos_, nullv_, arraylen_)
-		json, pos, nullv, arraylen = json_, pos_, nullv_, arraylen_
+	local function decode(json_, pos_, nullv_, arraylen_, preserveorder_)
+		json, pos, nullv, arraylen, preserveorder = json_, pos_, nullv_, arraylen_, preserveorder_
 		rec_depth = 0
 
 		pos = match(json, '^[ \n\r\t]*()', pos)

src/lunajson/encoder.lua

@@ -17,6 +17,33 @@ end
 local _ENV = nil
 
 
+local function patchpairs()
+	local needpatch = true
+	local metatable = {
+		__pairs = function()
+			needpatch = false
+			return function() end, nil, nil
+		end
+	}
+	local tbl = setmetatable({}, metatable)
+	pairs(tbl)
+	if needpatch then
+		local orig_pairs = pairs
+		function pairs(tbl)
+			local metatable = getmetatable(tbl)
+			if metatable ~= nil then
+				local f = metatable.__pairs
+				if f ~= nil then
+					return f(tbl)
+				end
+			end
+			return orig_pairs(tbl)
+		end
+	end
+end
+patchpairs()
+
+
 local function newencoder()
 	local v, nullv
 	local i, builder, visited

test/decoder/decode/lunajson.lua

@@ -1,5 +1,5 @@
 local lj = require 'lunajson'
-return function(json, nv)
-	local v, pos = lj.decode(json, 1, nv)
+return function(json, nv, preserveorder)
+	local v, pos = lj.decode(json, 1, nv, false, preserveorder)
 	return v
 end

test/decoder/decode/lunajson_sax.lua

@@ -1,6 +1,6 @@
 local sax_decoder = require 'sax_decoder'
 
-return function(json, nv)
+return function(json, nv, preserveorder)
 	local i = 1
 	local function gen()
 		local s = string.sub(json, i, i+8191)
@@ -10,5 +10,5 @@ return function(json, nv)
 		end
 		return s
 	end
-	return sax_decoder(gen, nv)
+	return sax_decoder(gen, nv, preserveorder)
 end

test/decoder/decode/lunajson_sax_nobuf.lua

@@ -1,6 +1,6 @@
 local sax_decoder = require 'sax_decoder'
 
-return function(json, nv)
+return function(json, nv, preserveorder)
 	local i = 1
 	local j = 0
 	local function gen()
@@ -15,5 +15,5 @@ return function(json, nv)
 		end
 		return s
 	end
-	return sax_decoder(gen, nv)
+	return sax_decoder(gen, nv, preserveorder)
 end

test/decoder/ordered_data.lua

@@ -0,0 +1,4 @@
+return
+	'{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, ' ..
+	'"j": 10, "i": 9, "h": 8, "g": 7, "f": 6}',
+	{'a', 'b', 'c', 'd', 'e', 'j', 'i', 'h', 'g', 'f'}

test/decoder/test.lua

@@ -85,10 +85,31 @@ local function test_invalid(decode, fn)
 	end
 end
 
+local function test_order(decode, ordered_json, order)
+	local function check()
+		local obj = decode(ordered_json, nullv, true)
+		local i = 0
+		for k, _ in getmetatable(obj).__pairs(obj) do
+			i = i + 1
+			if k ~= order[i] then
+				error('order differs at ' .. tostring(i) .. 'th key')
+			end
+		end
+		if i ~= #order then
+			error('length differs')
+		end
+	end
+	local ok, err = pcall(check)
+	if not ok then
+		return string.format('%q', err)
+	end
+end
+
 
 local decoders = util.load('decoders.lua')
 local valid_data = util.load('valid_data.lua')
 local invalid_data = util.load('invalid_data.lua')
+local ordered_json, order = util.load('ordered_data.lua')
 
 local iserr = false
 io.write('decode:\n')
@@ -109,7 +130,6 @@ for _, decoder in ipairs(decoders) do
 	for _, fn in ipairs(invalid_data) do
 		io.write('    ' .. fn  .. ': ')
 		fn = 'invalid_data/' .. fn
-		test_invalid(decode, fn)
 		local err = test_invalid(decode, fn)
 		if err then
 			iserr = true
@@ -118,6 +138,14 @@ for _, decoder in ipairs(decoders) do
 			io.write('ok\n')
 		end
 	end
+	io.write('    order: ')
+	local err = test_order(decode, ordered_json, order)
+	if err then
+		iserr = true
+		io.write(err .. '\n')
+	else
+		io.write('ok\n')
+	end
 end
 
 return iserr

test/encoder/test.lua

@@ -1,7 +1,7 @@
 local util = require 'util'
 
 
-function test_valid(encode, fn)
+local function test_valid(encode, fn)
 	local data = util.load(fn .. '.lua')
 	local json = encode(data)
 	local ans_fp = util.open(fn ..  '.json')
@@ -15,7 +15,7 @@ function test_valid(encode, fn)
 	end
 end
 
-function test_invalid(encode, fn)
+local function test_invalid(encode, fn)
 	local data = util.load(fn .. '.lua')
 	local ok, err = pcall(encode, data)
 	if ok then

test/encoder/valid_data.lua

@@ -1,3 +1,4 @@
 return {
 	'nonloop',
+	'ordered',
 }

test/encoder/valid_data/ordered.json

@@ -0,0 +1 @@
+{"a":1,"b":2,"c":3,"d":4,"e":5,"j":10,"i":9,"h":8,"g":7,"f":6}

test/encoder/valid_data/ordered.lua

@@ -0,0 +1,17 @@
+return setmetatable({a=1, b=2, c=3, d=4, e=5, j=10, i=9, h=8, g=7, f=6}, {
+	__pairs = function()
+		local function nxt(_, k)
+			if k == nil then return 'a', 1 end
+			if k == 'a' then return 'b', 2 end
+			if k == 'b' then return 'c', 3 end
+			if k == 'c' then return 'd', 4 end
+			if k == 'd' then return 'e', 5 end
+			if k == 'e' then return 'j', 10 end
+			if k == 'j' then return 'i', 9 end
+			if k == 'i' then return 'h', 8 end
+			if k == 'h' then return 'g', 7 end
+			if k == 'g' then return 'f', 6 end
+		end
+		return nxt, nil, nil
+	end
+})

util/sax_decoder.lua

@@ -1,6 +1,45 @@
 local lj = require 'lunajson'
 
-return function(gen, nv)
+
+-- Ordered table support
+--   keylist[metafirstkey] = firstkey
+--   keylist[key] = nextkey
+--   keylist[lastkey] = nil
+local metafirstkey = {}
+local function orderedtable(obj)
+	local keylist = {}
+	local lastkey = metafirstkey
+	local function onext(key2val, key)
+		local val
+		repeat
+			key = keylist[key]
+			if key == nil then
+				return
+			end
+			val = key2val[key]
+		until val ~= nil
+		return key, val
+	end
+	local metatable = {
+		__newindex = function(key2val, key, val)
+			rawset(key2val, key, val)
+			-- do the assignment first in case key == lastkey
+			keylist[lastkey] = key
+			if keylist[key] == nil then
+				lastkey = key
+			else
+				keylist[lastkey] = nil
+			end
+		end,
+		__pairs = function(key2val)
+			return onext, key2val, metafirstkey
+		end
+	}
+	return setmetatable(obj, metatable)
+end
+
+
+return function(gen, nv, preserveorder)
 	local saxtbl = {}
 	local current = {}
 	do
@@ -31,6 +70,9 @@ return function(gen, nv)
 		function saxtbl.startobject()
 			push()
 			current = {}
+			if preserveorder then
+				current = orderedtable(current)
+			end
 			key = nil
 		end
 		function saxtbl.key(s)

util/util.lua

@@ -4,9 +4,11 @@ return {
 		local dir = string.gsub(path, '/[^/]*$', '')
 		local new_path = dir .. '/' .. fn
 		arg[0] = new_path
-		local v = dofile(new_path)
-		arg[0] = path
-		return v
+		local arg_ = arg  -- workaround for Lua 5.1
+		return (function (...)
+			arg_[0] = path
+			return ...
+		end)(dofile(new_path))
 	end,
 	open = function(fn)
 		local path = arg[0]