Skip to content

Commit 1ccec67

Browse files
ludocodeludocode
committed
Code port to GitHub
1 parent 3798627 commit 1ccec67

30 files changed

+1470
-2
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
build/
2+
contrib/
3+
docs/*.1

‎CMakeLists.txt‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
cmake_minimum_required(VERSION 2.6)
2+
3+
project(msgpack2json)
4+
5+
set(CONTRIB_DIR ${CMAKE_BINARY_DIR}/contrib)
6+
file(MAKE_DIRECTORY ${CONTRIB_DIR})
7+
8+
if(NOT CMAKE_BUILD_TYPE)
9+
message(STATUS "No build type selected, default to Release")
10+
set(CMAKE_BUILD_TYPE "Release")
11+
endif()
12+
13+
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
14+
set(CMAKE_VERBOSE_MAKEFILE ON)
15+
endif()
16+
17+
if(CMAKE_BUILD_TOOL MATCHES "(msdev|devenv|nmake)")
18+
# Windows support is not implemented yet
19+
add_definitions(/W2)
20+
else()
21+
set(FLAGS "-std=c99 -Wall -Wextra")
22+
set(CMAKE_C_FLAGS_DEBUG "${FLAGS} -g -O0 -DDEBUG")
23+
set(CMAKE_C_FLAGS_RELEASE "${FLAGS} -O3 -flto -DNDEBUG -fPIC -DPIC")
24+
set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -fwhole-program -s")
25+
endif()
26+
27+
28+
# mpack
29+
30+
set(MPACK_VERSION "0.5.1")
31+
set(MPACK_FILE "mpack-amalgamation-${MPACK_VERSION}.tar.gz")
32+
set(MPACK_DIR "${CONTRIB_DIR}/mpack-amalgamation-${MPACK_VERSION}")
33+
34+
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${CMAKE_SOURCE_DIR}/contrib/${MPACK_FILE} WORKING_DIRECTORY ${CONTRIB_DIR})
35+
36+
file(COPY ${MPACK_DIR}/src/mpack-config.h.sample DESTINATION ${CMAKE_BINARY_DIR})
37+
file(RENAME ${CMAKE_BINARY_DIR}/mpack-config.h.sample ${CMAKE_BINARY_DIR}/mpack-config.h)
38+
39+
file(GLOB_RECURSE MPACK_SRCS ${MPACK_DIR}/src/*.c)
40+
include_directories(SYSTEM ${CMAKE_BINARY_DIR} ${MPACK_DIR}/src)
41+
42+
43+
# yajl
44+
45+
set(YAJL_COMMIT "78764146789a0aa263f2c10316cab1b651166cd7")
46+
set(YAJL_VERSION "2.1.1")
47+
set(YAJL_FILE "yajl-${YAJL_COMMIT}.tar.gz")
48+
set(YAJL_DIR "${CONTRIB_DIR}/yajl-${YAJL_COMMIT}")
49+
50+
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf "${CMAKE_SOURCE_DIR}/contrib/${YAJL_FILE}" WORKING_DIRECTORY "${CONTRIB_DIR}")
51+
52+
execute_process(COMMAND ./configure WORKING_DIRECTORY ${YAJL_DIR})
53+
54+
file(GLOB_RECURSE YAJL_SRCS ${YAJL_DIR}/src/*.c)
55+
include_directories(SYSTEM ${YAJL_DIR}/build/yajl-${YAJL_VERSION}/include)
56+
57+
58+
# libb64
59+
60+
set(LIBB64_VERSION "1.2.1")
61+
set(LIBB64_FILE "libb64-${LIBB64_VERSION}.zip")
62+
set(LIBB64_DIR "${CONTRIB_DIR}/libb64-${LIBB64_VERSION}")
63+
64+
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf "${CMAKE_SOURCE_DIR}/contrib/${LIBB64_FILE}" WORKING_DIRECTORY "${CONTRIB_DIR}")
65+
66+
# Remove libb64's newlines
67+
set(LIBB64_CENCODE_FILE ${LIBB64_DIR}/src/cencode.c)
68+
file(READ ${LIBB64_CENCODE_FILE} LIBB64_CENCODE)
69+
string(REPLACE "*codechar++ = '\\n';" "/* *codechar++ = '\\n'; */" LIBB64_CENCODE "${LIBB64_CENCODE}")
70+
file(WRITE ${LIBB64_CENCODE_FILE} "${LIBB64_CENCODE}")
71+
72+
file(GLOB_RECURSE LIBB64_SRCS ${LIBB64_DIR}/src/*.c)
73+
include_directories(SYSTEM ${LIBB64_DIR}/include)
74+
75+
76+
# targets
77+
78+
add_executable(msgpack2json src/msgpack2json.c ${MPACK_SRCS} ${YAJL_SRCS} ${LIBB64_SRCS})
79+
add_executable(json2msgpack src/json2msgpack.c ${MPACK_SRCS} ${YAJL_SRCS} ${LIBB64_SRCS})
80+
81+
install(TARGETS msgpack2json json2msgpack DESTINATION bin)
82+
83+
install(FILES ${CMAKE_SOURCE_DIR}/docs/msgpack2json.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1)
84+
install(FILES ${CMAKE_SOURCE_DIR}/docs/json2msgpack.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1)
85+
86+
87+
# testing
88+
89+
set(TESTS_DIR "${CMAKE_SOURCE_DIR}/tests")
90+
set(TEST_COMPARE_SCRIPT "${CMAKE_SOURCE_DIR}/tools/test-compare.sh")
91+
set(TEST_FAIL_SCRIPT "${CMAKE_SOURCE_DIR}/tools/test-fail.sh")
92+
93+
enable_testing()
94+
95+
add_test("json2msgpack-basic" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.mp ./json2msgpack -i ${TESTS_DIR}/basic.json)
96+
add_test("json2msgpack-basic-min" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.mp ./json2msgpack -i ${TESTS_DIR}/basic-min.json)
97+
add_test("json2msgpack-basic-lax" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.mp ./json2msgpack -li ${TESTS_DIR}/basic-lax.json)
98+
add_test("json2msgpack-basic-strict-fail" ${TEST_FAIL_SCRIPT} ./json2msgpack -i ${TESTS_DIR}/basic-lax.json)
99+
add_test("json2msgpack-basic-base64" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.mp ./json2msgpack -B 22 -bi ${TESTS_DIR}/basic.json)
100+
101+
add_test("msgpack2json-basic-min" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic-min.json ./msgpack2json -i ${TESTS_DIR}/basic.mp)
102+
add_test("msgpack2json-basic" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.json ./msgpack2json -pi ${TESTS_DIR}/basic.mp)
103+
add_test("msgpack2json-basic-debug" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/basic.json ./msgpack2json -di ${TESTS_DIR}/basic.mp)
104+
105+
add_test("json2msgpack-base64-str-prefix" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-str-prefix.mp ./json2msgpack -i ${TESTS_DIR}/base64-prefix.json)
106+
add_test("json2msgpack-base64-bin" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin-ext.mp ./json2msgpack -bi ${TESTS_DIR}/base64-prefix.json)
107+
add_test("json2msgpack-base64-bin-lax" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin-ext.mp ./json2msgpack -bli ${TESTS_DIR}/base64-prefix-lax.json)
108+
109+
add_test("json2msgpack-base64-detect-str" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-str.mp ./json2msgpack -B 200 -i ${TESTS_DIR}/base64-detect.json)
110+
add_test("json2msgpack-base64-detect-partial" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-partial.mp ./json2msgpack -B 50 -i ${TESTS_DIR}/base64-detect.json)
111+
add_test("json2msgpack-base64-detect-bin" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin.mp ./json2msgpack -B 22 -i ${TESTS_DIR}/base64-detect.json)
112+
add_test("json2msgpack-base64-detect-bin-one" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin.mp ./json2msgpack -B 1 -i ${TESTS_DIR}/base64-detect.json)
113+
114+
add_test("json2msgpack-base64-mixed-partial" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-partial-ext.mp ./json2msgpack -bB 50 -i ${TESTS_DIR}/base64-mixed.json)
115+
add_test("json2msgpack-base64-mixed-bin" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin-ext.mp ./json2msgpack -bB 22 -i ${TESTS_DIR}/base64-mixed.json)
116+
117+
#add_test("json2msgpack-base64-bin" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin.mp ./json2msgpack -bi ${TESTS_DIR}/base64-prefix.json)
118+
#add_test("json2msgpack-base64-bin-lax" ${TEST_COMPARE_SCRIPT} ${TESTS_DIR}/base64-bin.mp ./json2msgpack -lbi ${TESTS_DIR}/base64-prefix-lax.json)

‎LICENSE‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,14 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2020
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2121
SOFTWARE.
2222

23+
------------------------------------------------------------------------
24+
25+
The msgpack2json redistributable package includes packaged code from the
26+
following open source libraries:
27+
28+
MPack - https://github.com/ludocode/mpack
29+
YAJL - http://lloyd.github.io/yajl/
30+
libb64 - http://libb64.sourceforge.net/
31+
32+
These libraries are covered by their respective licenses included in
33+
their packages.

‎README.md‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,57 @@
1-
# msgpack2json
2-
Command-line tools for converting between MessagePack and JSON / msgpack.org[Shell]
1+
2+
## Introduction
3+
4+
`msgpack2json` and `json2msgpack` are simple command-line utilities for converting from [MessagePack](http://msgpack.org/) to [JSON](http://json.org/) and vice-versa. They support options for lax parsing, lossy conversions, pretty-printing, and base64 encoding.
5+
6+
They can be used for dumping MessagePack from a file or web API to a human-readable format, or for converting hand-written or generated JSON to MessagePack.
7+
8+
## Examples
9+
10+
To view a MessagePack file in a human-readable format for debugging purposes:
11+
12+
msgpack2json -di file.mp
13+
14+
To convert a hand-written JSON file to a MessagePack file, ignoring comments and trailing commas, and allowing embedded base64 with a "`base64:`" prefix:
15+
16+
json2msgpack -bli file.json -o file.mp
17+
18+
To fetch MessagePack from a web API and view it in a human-readable format:
19+
20+
curl http://example/url | msgpack2json -d
21+
22+
To view the MessagePack-equivalent encoding of a JSON string:
23+
24+
$ echo '{"compact": true, "schema": 0}' | json2msgpack | hexdump -C
25+
00000000 82 a7 63 6f 6d 70 61 63 74 c3 a6 73 63 68 65 6d |..compact..schem|
26+
00000010 61 00 |a.|
27+
00000012
28+
29+
## Installation
30+
31+
`msgpack2json` currently must be built from source. The latest version of the `msgpack2json` source archive can be downloaded from the releases page. This includes the library dependencies and has pre-generated man pages:
32+
33+
[https://github.com/ludocode/msgpack2json/releases](https://github.com/ludocode/msgpack2json/releases)
34+
35+
`msgpack2json` uses CMake. A `configure` script is provided that calls CMake, so on Linux and Mac OS X, you can simply run the usual:
36+
37+
./configure && make && sudo make install
38+
39+
If you are building from the repository, you will need to fetch the dependencies and generate the man pages. These are done with the scripts under `tools/fetch.sh` and `tools/man.sh`. [md2man](https://github.com/sunaku/md2man) is required to generate the man pages.
40+
41+
## Differences between MessagePack and JSON
42+
43+
MessagePack is intended to be very close to JSON in supported features, so they can usually be transparently converted from one to the other. There are some differences, however, which can complicate conversions.
44+
45+
These are the differences in what objects are representable in each format:
46+
47+
- JSON keys must be strings. MessagePack keys can be any type, including maps and arrays.
48+
49+
- JSON supports "bignums", i.e. integers of any size. MessagePack integers must fit within a 64-bit signed or unsigned integer.
50+
51+
- MessagePack supports binary and extension type objects. JSON does not support binary data. Binary data is often encoded into a base64 string to be embedded into a JSON document.
52+
53+
- The top-level object in a JSON document must be either an array or object (map). The top-level object in MessagePack can be any type.
54+
55+
- A JSON document must be entirely in one encoding. MessagePack does not specify encoding, and different strings in the same document can technically be in different encodings (though this is a really bad idea.)
56+
57+
By default, `msgpack2json` and `json2msgpack` convert in strict mode. If an object in the source format is not representable in the destination format, the converter aborts with an error. A lax mode is available which performs a "lossy" conversion, and base64 conversion modes are available to support binary data in JSON.

‎configure‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
cmake .

‎docs/json2msgpack.md‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
json2msgpack 1
2+
=======================================
3+
4+
NAME
5+
----
6+
7+
json2msgpack - convert JSON to MessagePack
8+
9+
SYNOPSIS
10+
--------
11+
12+
`json2msgpack` [`-lfb`] [`-B` *min-bytes*] [`-i` *in-file*] [`-o` *out-file*]
13+
14+
DESCRIPTION
15+
-----------
16+
17+
`json2msgpack` converts a JSON object to MessagePack. It has options for lax parsing and base64 conversions.
18+
19+
The JSON input is expected to be valid UTF-8. Strings in the resulting MessagePack will be encoded in UTF-8.
20+
21+
OPTIONS
22+
-------
23+
24+
`-i` *in-file*
25+
Read from the given file instead of standard input.
26+
27+
`-o` *out-file*
28+
Write to the given file instead of standard output.
29+
30+
`-l`
31+
Lax parsing mode. The JSON parser will be more forgiving to invalid JSON. This means:
32+
33+
- Comments are allowed and ignored
34+
- Trailing commas in maps and arrays are allowed and ignored
35+
- Single quotes can be used instead of double quotes to delimit strings
36+
37+
`-f`
38+
Convert real numbers to floats instead of doubles.
39+
40+
`-b`
41+
Parse strings with a "`base64:`" prefix as base64 and convert them to bin objects.
42+
43+
Base64 strings with an "`ext:`*#*`:base64:`" prefix will be converted to ext objects.
44+
45+
If base64 parsing fails for a string prefixed with `base64:` or `ext:`, the conversion will abort with error.
46+
47+
`-B` *min-bytes*
48+
Try to parse any non-key string *min-bytes* bytes or longer as base64, and if successful, convert it to a bin object. If base64 parsing fails, the string will be left as-is and written as a UTF-8 string.
49+
50+
Be careful not to set this too low, since any string without spaces (such UNIX paths) might be coincidentally parseable as base64. Carriage returns and line feeds are allowed and ignored, but spaces are not allowed in order to reduce the risk incorrect detection.
51+
52+
This option does not imply `-b`; you may want to specify both.
53+
54+
*min-bytes* must be larger than zero.
55+
56+
`-h`
57+
Print usage.
58+
59+
NOTES
60+
-----
61+
62+
Integers outside of the range \[INT64\_MIN, UINT64\_MAX\] will generate an error, since they cannot be stored in MessagePack.
63+
64+
`json2msgpack` will preserve the ordering of key-value pairs in objects/maps, and does not check that keys are unique.
65+
66+
EXAMPLES
67+
--------
68+
69+
To convert a hand-written JSON file to a MessagePack file, ignoring comments and trailing commas, and allowing embedded base64 with a "`base64:`" prefix:
70+
71+
> `json2msgpack -bli` *file.json* `-o` *file.mp*
72+
73+
AUTHOR
74+
------
75+
76+
Nicholas Fraser <https://github.com/ludocode>
77+
78+
SEE ALSO
79+
--------
80+
81+
msgpack2json(1)
82+
83+
[msgpack2json](https://github.com/ludocode/msgpack2json)
84+
85+
[JSON](http://json.org/)
86+
87+
[MessagePack](http://msgpack.org/)
88+
89+
[MPack](https://github.com/ludocode/mpack)
90+
91+
[YAJL](http://lloyd.github.io/yajl/)
92+
93+
[libb64](http://libb64.sourceforge.net/)

‎docs/msgpack2json.md‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
msgpack2json 1
2+
=======================================
3+
4+
NAME
5+
----
6+
7+
msgpack2json - convert MessagePack to JSON
8+
9+
SYNOPSIS
10+
--------
11+
12+
`msgpack2json` [`-lpbB`] [`-i` *in-file*] [`-o` *out-file*]
13+
14+
DESCRIPTION
15+
-----------
16+
17+
`msgpack2json` converts a MessagePack object to JSON. It has options for lax conversions, pretty-printing, and base64 conversions.
18+
19+
MessagePack strings are expected to be valid UTF-8. The resulting JSON will be output in UTF-8.
20+
21+
OPTIONS
22+
-------
23+
24+
`-i` *in-file*
25+
Read from the given file instead of standard input.
26+
27+
`-o` *out-file*
28+
Write to the given file instead of standard output.
29+
30+
`-d`
31+
Debugging conversion mode for viewing MessagePack in a human-readable format. MessagePack objects that cannot be expressed in JSON will be converted to pseudo-JSON instead of aborting with error. This means:
32+
33+
- Map keys do not have to be strings
34+
- The root node of the input does not have to be a map or array
35+
- Bin objects will be printed as `<bin of size` *###*`>` (unless in a base64 mode)
36+
- Ext objects will be printed as `<ext of type` *###* `size` *###*`>` (unless in a base64 mode)
37+
38+
The resulting output may not be parseable as JSON. This implies `-p`.
39+
40+
`-p`
41+
Pretty-print JSON output. UNIX-style newlines and four space indentation will be used.
42+
43+
`-b`
44+
Convert bin objects to base64 strings with a "`base64:`" prefix.
45+
46+
Ext objects will be converted to base64 strings with an "`ext:`*#*`:base64:`" prefix.
47+
48+
`-B`
49+
Convert bin objects to base64 strings with no prefix.
50+
51+
Ext objects will be converted to base64 strings with an "`ext:`*#*`:base64:`" prefix.
52+
53+
`-h`
54+
Print usage.
55+
56+
NOTES
57+
-----
58+
59+
If both `-b` and `-B` options are given, only the last one specified will take effect.
60+
61+
`msgpack2json` will preserve the ordering of key-value pairs in objects/maps, and does not check that keys are unique.
62+
63+
`msgpack2json` is under construction.
64+
65+
EXAMPLES
66+
--------
67+
68+
To view a MessagePack file in a human-readable format:
69+
70+
> `msgpack2json -di` *file.mp*
71+
72+
To convert a MessagePack file to a JSON file using base64 for embedded binary data:
73+
74+
> `msgpack2json -Bi` *file.mp* `-o` *file.json*
75+
76+
To fetch MessagePack from a web API and view it in a human-readable format:
77+
78+
> `curl` *ht**tp://example/url* `| msgpack2json -d`
79+
80+
AUTHOR
81+
------
82+
83+
Nicholas Fraser <https://github.com/ludocode>
84+
85+
SEE ALSO
86+
--------
87+
88+
json2msgpack(1)
89+
90+
[msgpack2json](https://github.com/ludocode/msgpack2json)
91+
92+
[MessagePack](http://msgpack.org/)
93+
94+
[JSON](http://json.org/)
95+
96+
[MPack](https://github.com/ludocode/mpack)
97+
98+
[YAJL](http://lloyd.github.io/yajl/)
99+
100+
[libb64](http://libb64.sourceforge.net/)

0 commit comments

Comments
 (0)