diff --git a/COPYRIGHT b/COPYRIGHT index 68136a1cf..968566427 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,6 +1,6 @@ Most nmsg code is under the following copyright and license: - Copyright (c) 2008-2014 by Farsight Security, Inc. + Copyright (c) 2008-2015 by Farsight Security, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile.am b/Makefile.am index 926623f8e..814951ed3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,7 +16,8 @@ AM_CFLAGS = \ $(libpcap_CFLAGS) \ $(libprotobuf_c_CFLAGS) \ $(libwdns_CFLAGS) \ - $(libxs_CFLAGS) + $(libxs_CFLAGS) \ + $(yajl_CFLAGS) AM_LDFLAGS = EXTRA_DIST += ChangeLog @@ -74,13 +75,18 @@ nmsg_libnmsg_la_LDFLAGS = \ nmsg_libnmsg_la_LIBADD = \ $(libpcap_LIBS) \ $(libprotobuf_c_LIBS) \ - $(libxs_LIBS) + $(libxs_LIBS) \ + $(yajl_LIBS) nmsg_libnmsg_la_SOURCES = \ libmy/crc32c.c libmy/crc32c.h libmy/crc32c-slicing.c libmy/crc32c-sse42.c \ libmy/list.h \ libmy/my_time.h \ libmy/my_rate.c libmy/my_rate.h \ libmy/tree.h \ + libmy/b64_decode.c \ + libmy/b64_decode.h \ + libmy/b64_encode.c \ + libmy/b64_encode.h \ nmsg/alias.c \ nmsg/asprintf.c \ nmsg/brate.c \ @@ -91,6 +97,7 @@ nmsg_libnmsg_la_SOURCES = \ nmsg/input.c \ nmsg/input_callback.c \ nmsg/input_frag.c \ + nmsg/input_json.c \ nmsg/input_nmsg.c \ nmsg/input_nullnmsg.c \ nmsg/input_pcap.c \ @@ -105,6 +112,7 @@ nmsg_libnmsg_la_SOURCES = \ nmsg/nmsg_port_net.h \ nmsg/output.c \ nmsg/output_frag.c \ + nmsg/output_json.c \ nmsg/output_nmsg.c \ nmsg/output_pres.c \ nmsg/payload.c \ @@ -123,6 +131,7 @@ nmsg_libnmsg_la_SOURCES = \ nmsg/msgmod/msgmod.c \ nmsg/msgmod/transparent.c \ nmsg/msgmod/transparent.h \ + nmsg/msgmod/transparent_json.c \ nmsg/msgmod/transparent_message.c \ nmsg/msgmod/transparent_module.c \ nmsg/msgmod/transparent_payload.c \ diff --git a/README.md b/README.md index 6da865c3d..5a4f9c8cc 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,15 @@ nmsg has the following external dependencies: * [wdns](https://github.com/farsightsec/wdns) +* [yajl](http://lloyd.github.io/yajl/) + * [libxs](http://www.crossroads.io/) * [zlib](http://www.zlib.net/) On Debian systems, the following packages should be installed, if available: - pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libxs-dev zlib1g-dev + pkg-config libpcap0.8-dev libprotobuf-c-dev protobuf-c-compiler libxs-dev libyajl-dev zlib1g-dev Note that on Debian systems, binary packages of nmsg and its dependencies are available from @@ -43,6 +45,7 @@ Debian-based systems. On FreeBSD systems, the following ports should be installed, if available: devel/libxs + devel/yajl devel/pkgconf devel/protobuf devel/protobuf-c @@ -57,6 +60,9 @@ compile and install `libnmsg` and `nmsgtool` to `/usr/local`. If building from a git checkout, run the `./autogen.sh` command first to generate the `configure` script. +Support for `yajl` can be disabled by passing the `--without-yajl` +parameter to the `configure` script. + Support for `libxs` can be disabled by passing the `--without-libxs` parameter to the `configure` script. diff --git a/configure.ac b/configure.ac index 9d13a0216..ff630b501 100644 --- a/configure.ac +++ b/configure.ac @@ -81,7 +81,7 @@ AC_CHECK_MEMBER( ) ### -### External library dependencies: libpcap, libprotobuf-c, libwdns, libxs, libz +### External library dependencies: libpcap, libprotobuf-c, libwdns, libxs, yajl, libz ### MY_CHECK_LIBPCAP @@ -91,7 +91,15 @@ AC_PATH_PROG([PROTOC_C], [protoc-c]) AS_IF([test -z "$PROTOC_C"], [AC_MSG_ERROR([The protoc-c program was not found. Please install the protobuf-c compiler!])]) -PKG_CHECK_MODULES([libwdns], [libwdns]) +PKG_CHECK_MODULES([libwdns], [libwdns >= 0.8.0]) +AC_ARG_WITH([yajl], AS_HELP_STRING([--without-yajl], [Disable yajl support])) +if test "x$with_yajl" != "xno"; then + PKG_CHECK_MODULES([yajl], [yajl >= 2]) + AC_DEFINE([HAVE_YAJL], [1], [Define to 1 if yajl support is enabled.]) + use_yajl="true" +else + use_yajl="false" +fi AC_ARG_WITH([libxs], AS_HELP_STRING([--without-libxs], [Disable libxs support])) if test "x$with_libxs" != "xno"; then @@ -165,6 +173,7 @@ AC_MSG_RESULT([ bigendian: ${ac_cv_c_bigendian} libxs support: ${use_libxs} + yajl support: ${use_yajl} building html docs: ${DOC_HTML_MSG} building manpage docs: ${DOC_MAN_MSG} diff --git a/nmsg/base/dns.c b/nmsg/base/dns.c index 3ce1f939a..f0026db2c 100644 --- a/nmsg/base/dns.c +++ b/nmsg/base/dns.c @@ -1,7 +1,7 @@ /* dns nmsg message module */ /* - * Copyright (c) 2009 by Farsight Security, Inc. + * Copyright (c) 2009, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,23 +29,39 @@ static NMSG_MSGMOD_FIELD_PRINTER(dns_type_print); static NMSG_MSGMOD_FIELD_PRINTER(dns_class_print); static NMSG_MSGMOD_FIELD_PRINTER(dns_rdata_print); +static NMSG_MSGMOD_FIELD_FORMATTER(dns_name_format); +static NMSG_MSGMOD_FIELD_FORMATTER(dns_type_format); +static NMSG_MSGMOD_FIELD_FORMATTER(dns_class_format); +static NMSG_MSGMOD_FIELD_FORMATTER(dns_rdata_format); + +static NMSG_MSGMOD_FIELD_PARSER(dns_name_parse); +static NMSG_MSGMOD_FIELD_PARSER(dns_type_parse); +static NMSG_MSGMOD_FIELD_PARSER(dns_class_parse); +static NMSG_MSGMOD_FIELD_PARSER(dns_rdata_parse); + /* Data. */ struct nmsg_msgmod_field dns_fields[] = { { .type = nmsg_msgmod_ft_bytes, .name = "qname", - .print = dns_name_print + .print = dns_name_print, + .format = dns_name_format, + .parse = dns_name_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "qclass", - .print = dns_class_print + .print = dns_class_print, + .format = dns_class_format, + .parse = dns_class_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "qtype", - .print = dns_type_print + .print = dns_type_print, + .format = dns_type_format, + .parse = dns_type_parse }, { .type = nmsg_msgmod_ft_uint16, @@ -54,17 +70,23 @@ struct nmsg_msgmod_field dns_fields[] = { { .type = nmsg_msgmod_ft_bytes, .name = "rrname", - .print = dns_name_print + .print = dns_name_print, + .format = dns_name_format, + .parse = dns_name_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "rrclass", - .print = dns_class_print + .print = dns_class_print, + .format = dns_class_format, + .parse = dns_class_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "rrtype", - .print = dns_type_print + .print = dns_type_print, + .format = dns_type_format, + .parse = dns_type_parse }, { .type = nmsg_msgmod_ft_uint32, @@ -74,7 +96,9 @@ struct nmsg_msgmod_field dns_fields[] = { .type = nmsg_msgmod_ft_bytes, .name = "rdata", .flags = NMSG_MSGMOD_FIELD_REPEATED, - .print = dns_rdata_print + .print = dns_rdata_print, + .format = dns_rdata_format, + .parse = dns_rdata_parse }, NMSG_MSGMOD_FIELD_END }; @@ -114,6 +138,57 @@ dns_name_print(nmsg_message_t msg, return (res); } +static nmsg_res +dns_name_format(nmsg_message_t m, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) +{ + ProtobufCBinaryData *rrname = ptr; + char name[WDNS_PRESLEN_NAME]; + nmsg_res res = nmsg_res_success; + + if (rrname->data != NULL && + rrname->len > 0 && + rrname->len <= WDNS_MAXLEN_NAME) + { + wdns_domain_to_str(rrname->data, rrname->len, name); + res = nmsg_strbuf_append(sb, "%s", name); + } + return (res); +} + +static nmsg_res +dns_name_parse(nmsg_message_t m, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + + wdns_res res; + wdns_name_t *name; + + name = malloc(sizeof(*name)); + if (name == NULL) { + return (nmsg_res_memfail); + } + + res = wdns_str_to_name_case(value, name); + if (res != wdns_res_success) { + free (name); + return (nmsg_res_parse_error); + } + + *ptr = name->data; + *len = name->len; + + free(name); + + return (nmsg_res_success); +} + static nmsg_res dns_type_print(nmsg_message_t msg, struct nmsg_msgmod_field *field, @@ -121,18 +196,61 @@ dns_type_print(nmsg_message_t msg, struct nmsg_strbuf *sb, const char *endline) { - uint16_t *rrtype = ptr; + uint16_t rrtype; const char *s; nmsg_res res = nmsg_res_success; - s = wdns_rrtype_to_str(*rrtype); - res = nmsg_strbuf_append(sb, "%s: %s (%hu)%s", + memcpy(&rrtype, ptr, sizeof(rrtype)); + s = wdns_rrtype_to_str(rrtype); + res = nmsg_strbuf_append(sb, "%s: %s (%u)%s", field->name, s ? s : "", - *rrtype, endline); + rrtype, endline); return (res); } +static nmsg_res +dns_type_format(nmsg_message_t m, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) { + uint16_t rrtype; + const char *s; + nmsg_res res = nmsg_res_success; + + memcpy(&rrtype, ptr, sizeof(rrtype)); + s = wdns_rrtype_to_str(rrtype); + res = nmsg_strbuf_append(sb, "%s", s ? s : ""); + return (res); +} + +static nmsg_res +dns_type_parse(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + uint16_t *rrtype; + + rrtype = malloc(sizeof(*rrtype)); + if (rrtype == NULL) { + return (nmsg_res_memfail); + } + + *rrtype = wdns_str_to_rrtype(value); + if (*rrtype == 0) { + free(rrtype); + return (nmsg_res_parse_error); + } + + *ptr = rrtype; + *len = sizeof(*rrtype); + + return (nmsg_res_success); +} + static nmsg_res dns_class_print(nmsg_message_t msg, struct nmsg_msgmod_field *field, @@ -140,18 +258,62 @@ dns_class_print(nmsg_message_t msg, struct nmsg_strbuf *sb, const char *endline) { - uint16_t *rrclass = ptr; + uint16_t rrclass; const char *s; nmsg_res res = nmsg_res_success; - s = wdns_rrclass_to_str(*rrclass); - res = nmsg_strbuf_append(sb, "%s: %s (%hu)%s", + memcpy(&rrclass, ptr, sizeof(rrclass)); + s = wdns_rrclass_to_str(rrclass); + res = nmsg_strbuf_append(sb, "%s: %s (%u)%s", field->name, s ? s : "", - *rrclass, endline); + rrclass, endline); + return (res); +} + +static nmsg_res +dns_class_format(nmsg_message_t m, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) { + uint16_t rrclass; + const char *s; + nmsg_res res = nmsg_res_success; + + memcpy(&rrclass, ptr, sizeof(rrclass)); + s = wdns_rrclass_to_str(rrclass); + res = nmsg_strbuf_append(sb, "%s", s ? s : ""); return (res); } +static nmsg_res +dns_class_parse(nmsg_message_t m, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + uint16_t *rrclass; + + rrclass = malloc(sizeof(*rrclass)); + if (rrclass == NULL) { + return (nmsg_res_memfail); + } + + *rrclass = wdns_str_to_rrclass(value); + *rrclass = WDNS_CLASS_IN; + if (*rrclass == 0) { + free(rrclass); + return (nmsg_res_parse_error); + } + + *ptr = rrclass; + *len = sizeof(*rrclass); + + return (nmsg_res_success); +} + static nmsg_res dns_rdata_print(nmsg_message_t msg, struct nmsg_msgmod_field *field __attribute__((unused)), @@ -159,29 +321,110 @@ dns_rdata_print(nmsg_message_t msg, struct nmsg_strbuf *sb, const char *endline) { - Nmsg__Base__Dns *dns = (Nmsg__Base__Dns *) nmsg_message_get_payload(msg); ProtobufCBinaryData *rdata = ptr; nmsg_res res; char *buf; + uint32_t *rrtype, *rrclass; + size_t len; - if (dns == NULL) + res = nmsg_message_get_field(msg, "rrtype", 0, (void**) &rrtype, &len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (len != sizeof(uint32_t)) { return (nmsg_res_failure); + } - if (dns->has_rrtype == false || dns->has_rrclass == false) + res = nmsg_message_get_field(msg, "rrclass", 0, (void**) &rrclass, &len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (len != sizeof(uint32_t)) { return (nmsg_res_failure); + } - buf = wdns_rdata_to_str(rdata->data, rdata->len, dns->rrtype, dns->rrclass); + buf = wdns_rdata_to_str(rdata->data, rdata->len, *rrtype, *rrclass); if (buf == NULL) - goto parse_error; + return (nmsg_res_memfail); res = nmsg_strbuf_append(sb, "rdata: %s%s", buf, endline); free(buf); return (res); +} + +static nmsg_res +dns_rdata_format(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) { + ProtobufCBinaryData *rdata = ptr; + nmsg_res res; + char *buf; + uint32_t *rrtype, *rrclass; + size_t len; + + res = nmsg_message_get_field(msg, "rrtype", 0, (void**) &rrtype, &len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (len != sizeof(uint32_t)) { + return (nmsg_res_failure); + } -parse_error: + res = nmsg_message_get_field(msg, "rrclass", 0, (void**) &rrclass, &len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (len != sizeof(uint32_t)) { + return (nmsg_res_failure); + } + + buf = wdns_rdata_to_str(rdata->data, rdata->len, *rrtype, *rrclass); + if (buf == NULL) + return (nmsg_res_memfail); + + res = nmsg_strbuf_append(sb, "%s", buf); free(buf); - nmsg_strbuf_append(sb, "rdata: ### PARSE ERROR ###\n"); - return (nmsg_res_parse_error); + return (res); +} + +static nmsg_res +dns_rdata_parse(nmsg_message_t m, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + nmsg_res res; + wdns_res w_res; + uint32_t *rrtype, *rrclass; + size_t f_len; + + res = nmsg_message_get_field(m, "rrtype", 0, (void**) &rrtype, &f_len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (f_len != sizeof(uint32_t)) { + return (nmsg_res_failure); + } + + res = nmsg_message_get_field(m, "rrclass", 0, (void**) &rrclass, &f_len); + if (res != nmsg_res_success) { + return (nmsg_res_failure); + } + if (f_len != sizeof(uint32_t)) { + return (nmsg_res_failure); + } + + w_res = wdns_str_to_rdata(value, *rrtype, *rrclass, (uint8_t**)ptr, len); + if (w_res == wdns_res_parse_error) { + return (nmsg_res_parse_error); + } else if (w_res != wdns_res_success) { + return (nmsg_res_failure); + } + + return (nmsg_res_success); } /*! \file nmsg/base/dns.c diff --git a/nmsg/base/dnsqr.c b/nmsg/base/dnsqr.c index 628db5358..28b5022a5 100644 --- a/nmsg/base/dnsqr.c +++ b/nmsg/base/dnsqr.c @@ -1,7 +1,7 @@ /* dnsqr nmsg message module */ /* - * Copyright (c) 2010-2013 by Farsight Security, Inc. + * Copyright (c) 2010-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -154,6 +154,13 @@ static nmsg_res dnsqr_pkt_to_payload(void *clos, nmsg_pcap_t pcap, nmsg_message_ static NMSG_MSGMOD_FIELD_PRINTER(dnsqr_proto_print); static NMSG_MSGMOD_FIELD_PRINTER(dnsqr_message_print); static NMSG_MSGMOD_FIELD_PRINTER(dnsqr_rcode_print); + +static NMSG_MSGMOD_FIELD_FORMATTER(dnsqr_proto_format); +static NMSG_MSGMOD_FIELD_FORMATTER(dnsqr_rcode_format); + +static NMSG_MSGMOD_FIELD_PARSER(dnsqr_proto_parse); +static NMSG_MSGMOD_FIELD_PARSER(dnsqr_rcode_parse); + static NMSG_MSGMOD_FIELD_GETTER(dnsqr_get_delay); static NMSG_MSGMOD_FIELD_GETTER(dnsqr_get_query); static NMSG_MSGMOD_FIELD_GETTER(dnsqr_get_response); @@ -176,7 +183,9 @@ struct nmsg_msgmod_field dnsqr_fields[] = { }, { .type = nmsg_msgmod_ft_uint16, .name = "proto", - .print = dnsqr_proto_print + .print = dnsqr_proto_print, + .format = dnsqr_proto_format, + .parse = dnsqr_proto_parse }, { .type = nmsg_msgmod_ft_uint16, @@ -192,22 +201,30 @@ struct nmsg_msgmod_field dnsqr_fields[] = { { .type = nmsg_msgmod_ft_bytes, .name = "qname", - .print = dns_name_print + .print = dns_name_print, + .format = dns_name_format, + .parse = dns_name_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "qclass", - .print = dns_class_print + .print = dns_class_print, + .format = dns_class_format, + .parse = dns_class_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "qtype", - .print = dns_type_print + .print = dns_type_print, + .format = dns_type_format, + .parse = dns_type_parse }, { .type = nmsg_msgmod_ft_uint16, .name = "rcode", - .print = dnsqr_rcode_print + .print = dnsqr_rcode_print, + .format = dnsqr_rcode_format, + .parse = dnsqr_rcode_parse }, { .type = nmsg_msgmod_ft_bytes, @@ -1006,6 +1023,61 @@ dnsqr_proto_print(nmsg_message_t msg, } } +static nmsg_res +dnsqr_proto_format(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) +{ + uint16_t proto; + + proto = *((uint16_t *) ptr); + + switch (proto) { + case IPPROTO_UDP: + return (nmsg_strbuf_append(sb, "UDP")); + case IPPROTO_TCP: + return (nmsg_strbuf_append(sb, "TCP")); + case IPPROTO_ICMP: + return (nmsg_strbuf_append(sb, "ICMP")); + default: + return (nmsg_strbuf_append(sb, "%hu", proto)); + } +} + +static nmsg_res +dnsqr_proto_parse(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + uint16_t *proto; + + proto = malloc(sizeof(*proto)); + if (proto == NULL) + return (nmsg_res_memfail); + + if (strcasecmp(value, "UDP") == 0) + *proto = IPPROTO_UDP; + else if (strcasecmp(value, "TCP") == 0) + *proto = IPPROTO_TCP; + else if (strcasecmp(value, "ICMP") == 0) + *proto = IPPROTO_ICMP; + else { + if (sscanf(value, "%hu", proto) != 1) { + free(proto); + return (nmsg_res_parse_error); + } + } + + *ptr = proto; + *len = sizeof(*proto); + + return (nmsg_res_success); +} + static nmsg_res dnsqr_message_print(nmsg_message_t msg, struct nmsg_msgmod_field *field, @@ -1057,6 +1129,50 @@ dnsqr_rcode_print(nmsg_message_t msg, *rcode, endline)); } +static nmsg_res +dnsqr_rcode_format(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline) +{ + const char *s; + uint16_t *rcode = ptr; + + s = wdns_rcode_to_str(*rcode); + if (s != NULL) + return (nmsg_strbuf_append(sb, "%s", s)); + else + return (nmsg_strbuf_append(sb, "%hu", *rcode)); +} + +static nmsg_res +dnsqr_rcode_parse(nmsg_message_t msg, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline) { + uint16_t *rcode; + wdns_res res; + + rcode = malloc(sizeof(*rcode)); + if (rcode == NULL) { + return (nmsg_res_memfail); + } + + res = wdns_str_to_rcode(value, rcode); + if (res != wdns_res_success) { + free(rcode); + return (nmsg_res_parse_error); + } + + *ptr = rcode; + *len = sizeof(*rcode); + + return (nmsg_res_success); +} + static nmsg_res dnsqr_get_udp_checksum(nmsg_message_t msg, struct nmsg_msgmod_field *field, diff --git a/nmsg/input.c b/nmsg/input.c index 805dc513b..f7898b2f0 100644 --- a/nmsg/input.c +++ b/nmsg/input.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -127,6 +127,39 @@ nmsg_input_open_pres(int fd, nmsg_msgmod_t msgmod) { return (input); } +#ifdef HAVE_YAJL +nmsg_input_t +nmsg_input_open_json(int fd) { + struct nmsg_input *input; + + input = calloc(1, sizeof(*input)); + if (input == NULL) + return (NULL); + input->type = nmsg_input_type_json; + input->read_fp = _input_json_read; + + input->json = calloc(1, sizeof(*(input->json))); + if (input->json == NULL) { + free(input); + return (NULL); + } + + input->json->fp = fdopen(fd, "r"); + if (input->json->fp == NULL) { + free(input->json); + free(input); + return (NULL); + } + + return (input); +} +#else /* HAVE_YAJL */ +nmsg_input_t +nmsg_input_open_json(int fd, nmsg_msgmod_t msgmod) { + return (NULL); +} +#endif /* HAVE_YAJL */ + nmsg_input_t nmsg_input_open_pcap(nmsg_pcap_t pcap, nmsg_msgmod_t msgmod) { nmsg_res res; @@ -188,6 +221,10 @@ nmsg_input_close(nmsg_input_t *input) { fclose((*input)->pres->fp); free((*input)->pres); break; + case nmsg_input_type_json: + fclose((*input)->json->fp); + free((*input)->json); + break; case nmsg_input_type_callback: free((*input)->callback); break; diff --git a/nmsg/input.h b/nmsg/input.h index 1ea4c7197..40292d8ec 100644 --- a/nmsg/input.h +++ b/nmsg/input.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009 by Farsight Security, Inc. + * Copyright (c) 2008, 2009, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,6 +57,7 @@ typedef enum { nmsg_input_type_stream, /*%< NMSG payloads from file or socket */ nmsg_input_type_pcap, /*%< pcap packets from file or interface */ nmsg_input_type_pres, /*%< presentation form */ + nmsg_input_type_json, /*%< json form */ nmsg_input_type_callback } nmsg_input_type; @@ -163,6 +164,16 @@ nmsg_input_open_null(void); nmsg_input_t nmsg_input_open_pres(int fd, nmsg_msgmod_t msgmod); +/** + * Initialize a new NMSG JSON form input from a file descriptor. + * + * \param[in] fd Readable file descriptor. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_input_t +nmsg_input_open_json(int fd); + /** * Initialize a new NMSG pcap input from a pcap descriptor. * diff --git a/nmsg/input_json.c b/nmsg/input_json.c new file mode 100644 index 000000000..bf8b9cec7 --- /dev/null +++ b/nmsg/input_json.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Import. */ + +#include "private.h" + +/* Internal functions. */ + +#ifdef HAVE_YAJL +nmsg_res +_input_json_read(nmsg_input_t input, nmsg_message_t *msg) { + char line[1024]; + nmsg_res res; + struct nmsg_strbuf *sb; + + sb = nmsg_strbuf_init(); + if (sb == NULL) + return (nmsg_res_memfail); + + while (fgets(line, sizeof(line), input->json->fp) != NULL) { + nmsg_strbuf_append(sb, "%s", line); + + if (sb->pos - sb->data == 0 || sb->pos[-1] != '\n') { + continue; + } + if (sb->pos - sb->data == 1) { + nmsg_strbuf_reset(sb); + continue; + } + + res = nmsg_message_from_json(sb->data, msg); + + /* skip failed messages */ + if (res == nmsg_res_parse_error) { + if (nmsg_get_debug() >= 2) { + sb->pos[-1] = 0; + fprintf(stderr, "JSON parse error: \"%s\"\n", sb->data); + } + nmsg_strbuf_reset(sb); + continue; + } + + nmsg_strbuf_destroy(&sb); + + return (res); + } + + nmsg_strbuf_destroy(&sb); + return (nmsg_res_eof); +} +#else /* HAVE_YAJL */ +nmsg_res +_input_json_read(nmsg_input_t input, nmsg_message_t *msg) { + return (nmsg_res_notimpl); +} +#endif /* HAVE_YAJL */ diff --git a/nmsg/message.h b/nmsg/message.h index 57ca36536..fc6f579bd 100644 --- a/nmsg/message.h +++ b/nmsg/message.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2011 by Farsight Security, Inc. + * Copyright (c) 2009-2011, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -98,6 +98,30 @@ nmsg_message_from_raw_payload(unsigned vid, unsigned msgtype, uint8_t *data, size_t sz, const struct timespec *ts); +/** + * Convert a json format line to an NMSG payload. + * Since the json format stream is line-delimited, not every line + * will necessarily result in a serialized message. + * + * This function will return the serialized payload. The caller is + * responsible for freeing the payload returned. + * + * Msgmods are not required to implement a function to convert json form + * data to payloads, in which case #nmsg_res_notimpl will be returned. + * + * \param[in] json Line of json form input. + * + * \param[out] msg Parsed message. + * + * \return #nmsg_res_success + * \return #nmsg_res_failure + * \return #nmsg_res_memfail + * \return #nmsg_res_notimpl + * \return #nmsg_res_parse_error + */ +nmsg_res +nmsg_message_from_json(const char *json, nmsg_message_t *msg); + /** * Destroy a message object and deallocate any resources associated with it. * @@ -120,6 +144,19 @@ nmsg_message_destroy(nmsg_message_t *msg); nmsg_res nmsg_message_to_pres(nmsg_message_t msg, char **pres, const char *endline); +/** + * Convert a message object to json format. + * + * \param[in] msg Message object. + * \param[out] pres Location to store malloc() allocated json format + * string. + * + * \return #nmsg_res_success if presentation format string was successfully + * rendered, non-success otherwise. + */ +nmsg_res +nmsg_message_to_json(nmsg_message_t msg, char **json); + /** * Return the message module object associated with a message object. */ diff --git a/nmsg/msgmod/message.c b/nmsg/msgmod/message.c index 43dcade69..19463a240 100644 --- a/nmsg/msgmod/message.c +++ b/nmsg/msgmod/message.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2012 by Farsight Security, Inc. + * Copyright (c) 2009-2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -207,6 +207,193 @@ nmsg_message_from_raw_payload(unsigned vid, unsigned msgtype, return (msg); } +#ifdef HAVE_YAJL +nmsg_res +nmsg_message_from_json(const char * json, nmsg_message_t *msg) +{ + nmsg_res res; + yajl_val node; + + yajl_val msgtype_v; + const char *msgtype_path[] = { "msgtype", (const char*) 0 }; + struct nmsg_msgmod *mod; + + yajl_val source_v; + const char *source_path[] = { "source", (const char*) 0 }; + + yajl_val operator_v; + const char *operator_path[] = { "operator", (const char*) 0 }; + + yajl_val group_v; + const char *group_path[] = { "group", (const char*) 0 }; + + yajl_val time_v; + const char *time_path[] = { "time", (const char*) 0 }; + struct timespec ts; + + yajl_val message_v; + const char *message_path[] = { "message", (const char*) 0 }; + + *msg = NULL; + + node = yajl_tree_parse(json, 0, 0); + + if (node == NULL) { + return (nmsg_res_parse_error); + } + + msgtype_v = yajl_tree_get(node, msgtype_path, yajl_t_string); + if (msgtype_v == NULL) { + res = (nmsg_res_parse_error); + goto err; + } else { + char *msgtype_orig, *msgtype_copy = NULL, *vname, *mname; + + msgtype_orig = YAJL_GET_STRING(msgtype_v); + msgtype_copy = strdup(msgtype_orig); + if (msgtype_copy == NULL) { + res = (nmsg_res_memfail); + goto err; + } + + vname = msgtype_copy; + mname = strchr(msgtype_copy, '.'); + if (mname == NULL) { + res = (nmsg_res_parse_error); + free (msgtype_copy); + goto err; + } + *mname = 0; + mname++; + + mod = nmsg_msgmod_lookup_byname(vname, mname); + if (mod == NULL) { + res = (nmsg_res_parse_error); + free (msgtype_copy); + goto err; + } + + free (msgtype_copy); + } + + *msg = nmsg_message_init(mod); + if (*msg == NULL) { + res = (nmsg_res_failure); + goto err; + } + + source_v = yajl_tree_get(node, source_path, yajl_t_any); + if (source_v) { + uint32_t source; + + if (YAJL_IS_STRING(source_v)) { + sscanf(YAJL_GET_STRING(source_v), "%x", &source); + } else if (YAJL_IS_INTEGER(source_v)) { + source = YAJL_GET_INTEGER(source_v); + } else { + res = (nmsg_res_parse_error); + goto err; + } + nmsg_message_set_source(*msg, source); + } + + operator_v = yajl_tree_get(node, operator_path, yajl_t_any); + if (operator_v) { + uint32_t operator; + + if (YAJL_IS_STRING(operator_v)) { + operator = nmsg_alias_by_value(nmsg_alias_operator, YAJL_GET_STRING(operator_v)); + } else if (YAJL_IS_INTEGER(operator_v)) { + operator = YAJL_GET_INTEGER(operator_v); + } else { + res = (nmsg_res_parse_error); + goto err; + } + nmsg_message_set_operator(*msg, operator); + } + + group_v = yajl_tree_get(node, group_path, yajl_t_any); + if (group_v) { + uint32_t group; + + if (YAJL_IS_STRING(group_v)) { + group = nmsg_alias_by_value(nmsg_alias_group, YAJL_GET_STRING(group_v)); + } else if (YAJL_IS_INTEGER(group_v)) { + group = YAJL_GET_INTEGER(group_v); + } else { + res = (nmsg_res_parse_error); + goto err; + } + nmsg_message_set_group(*msg, group); + } + + time_v = yajl_tree_get(node, time_path, yajl_t_any); + if (time_v) { + if (YAJL_IS_STRING(time_v)) { + struct tm tm; + char * remainder; + + remainder = strptime(YAJL_GET_STRING(time_v), "%Y-%m-%d %T", &tm); + if (remainder == NULL) { + res = (nmsg_res_parse_error); + goto err; + } + + ts.tv_sec = timegm(&tm); + + if (sscanf(remainder, ".%ld", &ts.tv_nsec) == 0) { + ts.tv_nsec = 0; + } + } else if (YAJL_IS_INTEGER(time_v)) { + ts.tv_sec = YAJL_GET_INTEGER(time_v); + ts.tv_nsec = 0; + } else if (YAJL_IS_DOUBLE(time_v)) { + nmsg_timespec_from_double(YAJL_GET_DOUBLE(time_v), &ts); + } else { + res = (nmsg_res_parse_error); + goto err; + } + } else { + nmsg_timespec_get(&ts); + } + nmsg_message_set_time(*msg, &ts); + + switch (mod->plugin->type) { + case nmsg_msgmod_type_transparent: + message_v = yajl_tree_get(node, message_path, yajl_t_object); + if (message_v) { + res = (_nmsg_msgmod_json_to_message(message_v, *msg)); + if (res != nmsg_res_success) { + goto err; + } + } else { + res = (nmsg_res_parse_error); + goto err; + } + break; + default: + res = (nmsg_res_notimpl); + goto err; + } + + yajl_tree_free(node); + + return (nmsg_res_success); +err: + if (*msg != NULL) { + nmsg_message_destroy(msg); + } + + yajl_tree_free(node); + return (res); +} +#else /* HAVE_YAJL */ +nmsg_res +nmsg_json_to_payload(const char * json, struct nmsg_msgmod **mod, uint8_t **pbuf, size_t *sz) + return (nmsg_res_notimpl); +} +#endif /* HAVE_YAJL */ + nmsg_res _nmsg_message_init_message(struct nmsg_message *msg) { if (msg->mod->plugin->type == nmsg_msgmod_type_transparent && @@ -328,6 +515,20 @@ nmsg_message_to_pres(struct nmsg_message *msg, char **pres, const char *endline) } } +nmsg_res +nmsg_message_to_json(nmsg_message_t msg, char **json) { + if (msg->mod == NULL) + return (nmsg_res_failure); + switch (msg->mod->plugin->type) { + case nmsg_msgmod_type_transparent: + return (_nmsg_message_payload_to_json(msg, json)); + case nmsg_msgmod_type_opaque: + return (nmsg_res_notimpl); + default: + return (nmsg_res_notimpl); + } +} + nmsg_res nmsg_message_add_allocation(struct nmsg_message *msg, void *ptr) { void *tmp; diff --git a/nmsg/msgmod/msgmod.c b/nmsg/msgmod/msgmod.c index 2d710daa8..f54eac25f 100644 --- a/nmsg/msgmod/msgmod.c +++ b/nmsg/msgmod/msgmod.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2010, 2012 by Farsight Security, Inc. + * Copyright (c) 2008-2010, 2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -152,6 +152,7 @@ _nmsg_msgmod_start(struct nmsg_msgmod_plugin *plugin) { void _nmsg_msgmod_stop(struct nmsg_msgmod **mod) { free((*mod)->fields); + free((*mod)->fields_idx); free(*mod); *mod = NULL; } diff --git a/nmsg/msgmod/transparent.c b/nmsg/msgmod/transparent.c index da43006cc..aa9349b6f 100644 --- a/nmsg/msgmod/transparent.c +++ b/nmsg/msgmod/transparent.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2010, 2012 by Farsight Security, Inc. + * Copyright (c) 2009, 2010, 2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,26 +21,31 @@ static int _nmsg_msgmod_field_cmp(const void *v1, const void *v2) { - const struct nmsg_msgmod_field *f1 = (const struct nmsg_msgmod_field *) v1; - const struct nmsg_msgmod_field *f2 = (const struct nmsg_msgmod_field *) v2; + const struct nmsg_msgmod_field **f1 = (const struct nmsg_msgmod_field **) v1; + const struct nmsg_msgmod_field **f2 = (const struct nmsg_msgmod_field **) v2; - return (strcmp(f1->name, f2->name)); + return (strcmp((*f1)->name, (*f2)->name)); } struct nmsg_msgmod_field * _nmsg_msgmod_lookup_field(struct nmsg_msgmod *mod, const char *name) { - struct nmsg_msgmod_field *res; - struct nmsg_msgmod_field key; + struct nmsg_msgmod_field **res; + struct nmsg_msgmod_field key, *key_ptr; key.name = name; + key_ptr = &key; - res = bsearch(&key, - &mod->fields[0], + res = bsearch(&key_ptr, + &mod->fields_idx[0], mod->n_fields, - sizeof(struct nmsg_msgmod_field), + sizeof(struct nmsg_msgmod_field*), _nmsg_msgmod_field_cmp); - return (res); + if (res) { + return (*res); + } else { + return NULL; + } } nmsg_res @@ -93,9 +98,18 @@ _nmsg_msgmod_load_field_descriptors(struct nmsg_msgmod *mod) { } /* sort field descriptors by name */ - qsort(&mod->fields[0], + mod->fields_idx = calloc(1, sizeof(struct nmsg_msgmod_field*) * (mod->n_fields + 1)); + if (mod->fields_idx == NULL) { + free(mod->fields); + return (nmsg_res_memfail); + } + + for(i = 0; i < mod->n_fields; i++) { + mod->fields_idx[i] = &mod->fields[i]; + } + qsort(&mod->fields_idx[0], mod->n_fields, - sizeof(struct nmsg_msgmod_field), + sizeof(struct nmsg_msgmod_field*), _nmsg_msgmod_field_cmp); return (nmsg_res_success); diff --git a/nmsg/msgmod/transparent.h b/nmsg/msgmod/transparent.h index 113e87729..0fa3e0b67 100644 --- a/nmsg/msgmod/transparent.h +++ b/nmsg/msgmod/transparent.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009 by Farsight Security, Inc. + * Copyright (c) 2009, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,6 +63,24 @@ nmsg_res _nmsg_msgmod_pres_to_payload_finalize(struct nmsg_msgmod *mod, void *cl, uint8_t **pbuf, size_t *sz); +nmsg_res +_nmsg_message_payload_to_json(struct nmsg_message *msg, char **json); + +nmsg_res +_nmsg_message_payload_to_json_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, void *ptr, + void * /* yajl_gen */ gen); + +nmsg_res +_nmsg_msgmod_json_to_message(void * /* yajl_val */ val, + struct nmsg_message *msg); + +nmsg_res +_nmsg_msgmod_json_to_payload_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, + unsigned field_idx, unsigned val_idx, + void * /* yajl_val */ val); + nmsg_res _nmsg_msgmod_load_field_descriptors(struct nmsg_msgmod *mod); diff --git a/nmsg/msgmod/transparent_json.c b/nmsg/msgmod/transparent_json.c new file mode 100644 index 000000000..bf9ac3f16 --- /dev/null +++ b/nmsg/msgmod/transparent_json.c @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2015 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "private.h" + +#include "transparent.h" + +#ifdef HAVE_YAJL +nmsg_res +_nmsg_msgmod_json_to_message(void * val, struct nmsg_message *msg) { + yajl_val message_v = (yajl_val) val; + nmsg_res res; + size_t n; + struct nmsg_msgmod_field *field = NULL; + + for (n = 0; n < msg->mod->n_fields; n++) { + const char* field_path[] = { (const char*) 0, (const char*)0 }; + yajl_val field_v; + + field = &msg->mod->fields[n]; + + /* skip virtual fields */ + if (field->descr == NULL) { + continue; + } + + field_path[0] = field->descr->name; + + if (PBFIELD_REPEATED(field)) { + yajl_val array_v; + size_t v; + + array_v = yajl_tree_get(message_v, field_path, yajl_t_any); + if (array_v) { + if (! YAJL_IS_ARRAY(array_v)) { + return (nmsg_res_parse_error); + } + + for (v = 0; v < YAJL_GET_ARRAY(array_v)->len; v++) { + field_v = YAJL_GET_ARRAY(array_v)->values[v]; + res = _nmsg_msgmod_json_to_payload_load(msg, field, n, v, field_v); + if (res != nmsg_res_success) { + return (res); + } + } + } + } else { + field_v = yajl_tree_get(message_v, field_path, yajl_t_any); + res = _nmsg_msgmod_json_to_payload_load(msg, field, n, 0, field_v); + if (res != nmsg_res_success) { + return (res); + } + } + } + + return (nmsg_res_success); +} + +nmsg_res +_nmsg_msgmod_json_to_payload_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, + unsigned field_idx, unsigned val_idx, + void * val) +{ + yajl_val field_v = (yajl_val) val; + nmsg_res res; + + if (field_v == NULL) { + return (nmsg_res_success); + } + + if (field->parse != NULL) { + if (YAJL_IS_STRING(field_v)) { + char * str = YAJL_GET_STRING(field_v); + uint8_t * ptr = NULL; + size_t len = 0; + + res = field->parse(msg, field, str, (void*)&ptr, &len, ""); + if (res != nmsg_res_success) { + return (res); + } + + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, ptr, len); + free (ptr); + return (res); + } + return (nmsg_res_parse_error); + } + + switch (field->type) { + case nmsg_msgmod_ft_bool: { + protobuf_c_boolean b; + + if (YAJL_IS_TRUE(field_v)) { + b = true; + } else if (YAJL_IS_FALSE(field_v)) { + b = false; + } else if (YAJL_IS_INTEGER(field_v)) { + b = YAJL_GET_INTEGER(field_v) != 0; + } else if (YAJL_IS_STRING(field_v)) { + char * str = YAJL_GET_STRING(field_v); + if (strcasecmp("true", str)) { + b = true; + } else if (strcasecmp("false", str)) { + b = false; + } else { + return (nmsg_res_parse_error); + } + } + + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &b, sizeof(b)); + return (res); + break; + } + case nmsg_msgmod_ft_bytes: { + if (YAJL_IS_STRING(field_v)) { + const char *b64_str = YAJL_GET_STRING(field_v); + char *s; + size_t b64_str_len, len; + base64_decodestate b64; + + base64_init_decodestate(&b64); + b64_str_len = strlen(b64_str); + s = malloc(b64_str_len+1); + if (s == NULL) + return (nmsg_res_memfail); + len = base64_decode_block(b64_str, b64_str_len, s, &b64); + + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*)s, len); + free(s); + return (res); + } else { + return (nmsg_res_parse_error); + } + break; + } + case nmsg_msgmod_ft_enum: { + ProtobufCEnumDescriptor *enum_descr; + unsigned enum_value; + + enum_descr = (ProtobufCEnumDescriptor *) field->descr->descriptor; + + if (YAJL_IS_STRING(field_v)) { + char * str = YAJL_GET_STRING(field_v); + size_t i; + + for (i = 0; i < enum_descr->n_values; i++) { + if (strcasecmp(enum_descr->values[i].name, str) == 0) { + enum_value = enum_descr->values[i].value; + break; + } + } + if (i >= enum_descr->n_values) { + return (nmsg_res_parse_error); + } + } else if (YAJL_IS_INTEGER(field_v)) { + enum_value = YAJL_GET_INTEGER(field_v); + if (enum_value < 0 || enum_value >= enum_descr->n_values) { + return (nmsg_res_parse_error); + } + } else { + return (nmsg_res_parse_error); + } + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &enum_value, sizeof(enum_value)); + return (res); + break; + } + case nmsg_msgmod_ft_string: + case nmsg_msgmod_ft_mlstring: { + if (! YAJL_IS_STRING(field_v)) { + return (nmsg_res_parse_error); + } + char * str = YAJL_GET_STRING(field_v); + size_t len = strlen(str) + 1; /* \0 terminated */ + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) str, len); + return (res); + break; + } + case nmsg_msgmod_ft_ip: { + char addr[16]; + + if (YAJL_IS_STRING(field_v)) { + char * str = YAJL_GET_STRING(field_v); + + if (inet_pton(AF_INET, str, addr) == 1) { + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) addr, 4); + return (res); + } else if (inet_pton(AF_INET6, str, addr) == 1) { + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) addr, 16); + return (res); + } + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_uint16: { + uint32_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + if (YAJL_GET_INTEGER(field_v)> UINT16_MAX) + return (nmsg_res_parse_error); + intval = (uint32_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_uint32: { + uint32_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + if (YAJL_GET_INTEGER(field_v)> UINT32_MAX) + return (nmsg_res_parse_error); + intval = (uint32_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_uint64: { + uint64_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + intval = (uint64_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } else if (YAJL_IS_NUMBER(field_v)) { + char * str = YAJL_GET_NUMBER(field_v); + if (sscanf(str, "%" PRIu64, &intval)) { + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_int16: { + int32_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + if (YAJL_GET_INTEGER(field_v) > INT16_MAX || YAJL_GET_INTEGER(field_v) < INT16_MIN) + return (nmsg_res_parse_error); + intval = (int32_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_int32: { + int32_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + if (YAJL_GET_INTEGER(field_v) > INT32_MAX || YAJL_GET_INTEGER(field_v) < INT32_MIN) + return (nmsg_res_parse_error); + intval = (int32_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_int64: { + int64_t intval; + + if (YAJL_IS_INTEGER(field_v)) { + intval = (int64_t) YAJL_GET_INTEGER(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &intval, sizeof(intval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + case nmsg_msgmod_ft_double: { + double dval; + + if (YAJL_IS_DOUBLE(field_v)) { + dval = YAJL_GET_DOUBLE(field_v); + res = nmsg_message_set_field_by_idx(msg, field_idx, val_idx, (const uint8_t*) &dval, sizeof(dval)); + return (res); + } + return (nmsg_res_parse_error); + break; + } + } /* switch */ + + return (nmsg_res_failure); +} + +#else /* HAVE_YAJL */ +nmsg_res +_nmsg_msgmod_json_to_message(void * val, struct nmsg_message *msg) { + return (nmsg_res_notimpl); +} + +nmsg_res +_nmsg_msgmod_json_to_payload_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, + unsigned field_idx, unsigned val_idx, + void * val) +{ + return (nmsg_res_notimpl); +} +#endif /* HAVE_YAJL */ diff --git a/nmsg/msgmod/transparent_message.c b/nmsg/msgmod/transparent_message.c index ec2a9a9ac..f0df2a97d 100644 --- a/nmsg/msgmod/transparent_message.c +++ b/nmsg/msgmod/transparent_message.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2012 by Farsight Security, Inc. + * Copyright (c) 2009-2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -265,7 +265,7 @@ nmsg_message_set_field_by_idx(struct nmsg_message *msg, unsigned field_idx, CHECK_TRANSPARENT(); GET_FIELD(field_idx); - if (field->get != NULL || field->flags & NMSG_MSGMOD_FIELD_HIDDEN) + if (field->descr == NULL) return (nmsg_res_failure); DESERIALIZE(); diff --git a/nmsg/msgmod/transparent_payload.c b/nmsg/msgmod/transparent_payload.c index 197b6b620..789099d29 100644 --- a/nmsg/msgmod/transparent_payload.c +++ b/nmsg/msgmod/transparent_payload.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2012 by Farsight Security, Inc. + * Copyright (c) 2009-2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -123,6 +123,34 @@ _nmsg_message_payload_to_pres_load(struct nmsg_message *msg, return (field->print(msg, field, ptr, sb, endline)); } + if (field->format != NULL) { + struct nmsg_strbuf *sb_tmp = NULL; + nmsg_res res; + + sb_tmp = nmsg_strbuf_init(); + if (sb_tmp == NULL) + return (nmsg_res_memfail); + + if (field->type == nmsg_msgmod_ft_uint16 || + field->type == nmsg_msgmod_ft_int16) + { + uint16_t val; + uint32_t val32; + memcpy(&val32, ptr, sizeof(uint32_t)); + val = (uint16_t) val32; + res = field->format(msg, field, &val, sb_tmp, endline); + } else { + res = field->format(msg, field, ptr, sb_tmp, endline); + } + + if (res == nmsg_res_success) + nmsg_strbuf_append(sb, "%s: %s%s", field->name, sb_tmp->data, endline); + + nmsg_strbuf_destroy(&sb_tmp); + + return res; + } + switch (field->type) { case nmsg_msgmod_ft_bytes: bdata = (ProtobufCBinaryData *) ptr; @@ -248,3 +276,448 @@ _nmsg_message_payload_to_pres_load(struct nmsg_message *msg, return (nmsg_res_success); } + +#ifdef HAVE_YAJL + +#define add_yajl_string(g, s) do { \ + yajl_gen_status g_status; \ + g_status = yajl_gen_string(g, (const unsigned char *) s, strlen(s)); \ + assert(g_status == yajl_gen_status_ok); \ +} while (0) + +static void +callback_print_yajl_nmsg_strbuf(void *ctx, const char *str, size_t len) +{ + struct nmsg_strbuf *sb = (struct nmsg_strbuf *) ctx; + nmsg_strbuf_append(sb, "%.*s", len, str); +} + +nmsg_res +_nmsg_message_payload_to_json(struct nmsg_message *msg, char **json) { + Nmsg__NmsgPayload *np; + ProtobufCMessage *m; + nmsg_res res; + yajl_gen g = NULL; + yajl_gen_status status; + int yajl_rc; + struct nmsg_strbuf *sb = NULL; + struct nmsg_strbuf *sb_tmp = NULL; + + size_t n; + struct nmsg_msgmod_field *field; + const char *vname, *mname; + + struct tm *tm; + time_t t; + char when[32]; + + /* unpack message */ + res = _nmsg_message_deserialize(msg); + if (res != nmsg_res_success) + return (res); + m = msg->message; + + sb = nmsg_strbuf_init(); + if (sb == NULL) + return (nmsg_res_memfail); + + sb_tmp = nmsg_strbuf_init(); + if (sb_tmp == NULL) { + nmsg_strbuf_destroy(&sb); + res = nmsg_res_memfail; + goto err; + } + + np = msg->np; + + g = yajl_gen_alloc(NULL); + assert (g != NULL); + + yajl_rc = yajl_gen_config(g, yajl_gen_print_callback, callback_print_yajl_nmsg_strbuf, sb); + assert (yajl_rc != 0); + + status = yajl_gen_map_open(g); + assert(status == yajl_gen_status_ok); + + t = np->time_sec; + tm = gmtime(&t); + strftime(when, sizeof(when), "%Y-%m-%d %T", tm); + + nmsg_strbuf_reset(sb_tmp); + nmsg_strbuf_append(sb_tmp, "%s.%09u", when, np->time_nsec); + + add_yajl_string(g, "time"); + status = yajl_gen_string(g, (unsigned char*) sb_tmp->data, strlen(sb_tmp->data)); + assert(status == yajl_gen_status_ok); + + vname = nmsg_msgmod_vid_to_vname(np->vid); + if (vname == NULL) { + vname = "(unknown)"; + } + + mname = nmsg_msgmod_msgtype_to_mname(np->vid, np->msgtype); + if (mname == NULL) { + mname = "(unknown)"; + } + + nmsg_strbuf_reset(sb_tmp); + nmsg_strbuf_append(sb_tmp, "%s.%s", vname, mname); + add_yajl_string(g, "msgtype"); + status = yajl_gen_string(g, (unsigned char*) sb_tmp->data, strlen(sb_tmp->data)); + assert(status == yajl_gen_status_ok); + + if (np->has_source) { + nmsg_strbuf_reset(sb_tmp); + nmsg_strbuf_append(sb_tmp, "%08x", np->source); + add_yajl_string(g, "source"); + status = yajl_gen_string(g, (unsigned char*) sb_tmp->data, strlen(sb_tmp->data)); + assert(status == yajl_gen_status_ok); + } + + if (np->has_operator_) { + const char * operator = nmsg_alias_by_key(nmsg_alias_operator, np->operator_); + add_yajl_string(g, "operator"); + if (operator != NULL) { + status = yajl_gen_string(g, (unsigned char*) operator, strlen(operator)); + } else { + status = yajl_gen_integer(g, np->operator_); + } + assert(status == yajl_gen_status_ok); + } + + if (np->has_group) { + const char * group = nmsg_alias_by_key(nmsg_alias_group, np->group); + add_yajl_string(g, "group"); + if (group != NULL) { + status = yajl_gen_string(g, (unsigned char*) group, strlen(group)); + } else { + status = yajl_gen_integer(g, np->group); + } + assert(status == yajl_gen_status_ok); + } + + add_yajl_string(g, "message"); + + status = yajl_gen_map_open(g); + assert(status == yajl_gen_status_ok); + + for (n = 0; n < msg->mod->n_fields; n++) { + void *ptr; + + field = &msg->mod->plugin->fields[n]; + + /* skip virtual fields unless they have a getter, in which + * case include the text for usability reasons */ + if (field->descr == NULL) { + if (field->get != NULL) { + unsigned val_idx = 0; + + for (;;) { + if (field->type == nmsg_msgmod_ft_ip || + field->type == nmsg_msgmod_ft_bytes) + { + ProtobufCBinaryData bdata; + res = field->get(msg, field, val_idx, (void **) &bdata.data, &bdata.len, msg->msg_clos); + if (res != nmsg_res_success) + break; + ptr = &bdata; + } else { + res = field->get(msg, field, val_idx, &ptr, NULL, msg->msg_clos); + if (res != nmsg_res_success) + break; + } + if (val_idx == 0) { + status = yajl_gen_string(g, (unsigned char *) field->name, strlen(field->name)); + assert(status == yajl_gen_status_ok); + + if (field->flags & (NMSG_MSGMOD_FIELD_REPEATED)) { + status = yajl_gen_array_open(g); + assert(status == yajl_gen_status_ok); + } + } + + + res = _nmsg_message_payload_to_json_load(msg, field, ptr, g); + if (res != nmsg_res_success) + goto err; + val_idx += 1; + + if ((field->flags & (NMSG_MSGMOD_FIELD_REPEATED)) == 0) + break; + } + + if (val_idx > 0 && field->flags & (NMSG_MSGMOD_FIELD_REPEATED)) { + status = yajl_gen_array_close(g); + assert(status == yajl_gen_status_ok); + } + } + continue; + } + + + if (PBFIELD_ONE_PRESENT(m, field)) { + status = yajl_gen_string(g, (unsigned char *) field->name, strlen(field->name)); + assert(status == yajl_gen_status_ok); + + ptr = PBFIELD(m, field, void); + + res = _nmsg_message_payload_to_json_load(msg, field, ptr, g); + if (res != nmsg_res_success) + goto err; + } else if (PBFIELD_REPEATED(field)) { + status = yajl_gen_string(g, (unsigned char *) field->name, strlen(field->name)); + assert(status == yajl_gen_status_ok); + + status = yajl_gen_array_open(g); + assert(status == yajl_gen_status_ok); + + size_t i, n_entries; + + n_entries = *PBFIELD_Q(m, field); + for (i = 0; i < n_entries; i++) { + ptr = PBFIELD(m, field, void); + char *array = *(char **) ptr; + size_t siz = sizeof_elt_in_repeated_array(field->descr->type); + + res = _nmsg_message_payload_to_json_load(msg, field, &array[i * siz], g); + if (res != nmsg_res_success) + goto err; + } + + status = yajl_gen_array_close(g); + assert(status == yajl_gen_status_ok); + } + } + + status = yajl_gen_map_close(g); + assert(status == yajl_gen_status_ok); + + status = yajl_gen_map_close(g); + assert(status == yajl_gen_status_ok); + + yajl_gen_reset(g, ""); + + yajl_gen_free(g); + + *json = sb->data; + free(sb); + + nmsg_strbuf_destroy(&sb_tmp); + + return (nmsg_res_success); + +err: + if (g != NULL) { + yajl_gen_free(g); + } + + nmsg_strbuf_destroy(&sb); + nmsg_strbuf_destroy(&sb_tmp); + + return (res); +} + +nmsg_res +_nmsg_message_payload_to_json_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, void *ptr, + void * gen) { + ProtobufCBinaryData *bdata; + unsigned i; + + yajl_gen g; + yajl_gen_status status; + + g = (yajl_gen)gen; + + if (field->format != NULL) { + struct nmsg_strbuf *sb = NULL; + nmsg_res res; + char * endline = ""; + + sb = nmsg_strbuf_init(); + if (sb == NULL) + return (nmsg_res_memfail); + + if (field->type == nmsg_msgmod_ft_uint16 || + field->type == nmsg_msgmod_ft_int16) + { + uint16_t val; + uint32_t val32; + memcpy(&val32, ptr, sizeof(uint32_t)); + val = (uint16_t) val32; + res = field->format(msg, field, &val, sb, endline); + } else { + res = field->format(msg, field, ptr, sb, endline); + } + + if (res == nmsg_res_success) { + status = yajl_gen_string(g, (const unsigned char*) sb->data, strlen(sb->data)); + assert(status == yajl_gen_status_ok); + } else { + status = yajl_gen_null(g); + assert(status == yajl_gen_status_ok); + } + + nmsg_strbuf_destroy(&sb); + + return (nmsg_res_success); + } + + switch(field->type) { + case nmsg_msgmod_ft_bytes: { + base64_encodestate b64; + char *b64_str; + size_t b64_str_len; + + base64_init_encodestate(&b64); + bdata = (ProtobufCBinaryData *) ptr; + b64_str = malloc(2 * bdata->len + 1); + if (b64_str == NULL) + return (nmsg_res_memfail); + + b64_str_len = base64_encode_block((const char*)bdata->data, bdata->len, b64_str, &b64); + b64_str_len += base64_encode_blockend(b64_str + b64_str_len, &b64); + status = yajl_gen_string(g, (const unsigned char*)b64_str, b64_str_len); + free(b64_str); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_string: + case nmsg_msgmod_ft_mlstring: { + bdata = (ProtobufCBinaryData *) ptr; + status = yajl_gen_string(g, (const unsigned char*) bdata->data, bdata->len); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_bool: { + protobuf_c_boolean *b = (protobuf_c_boolean *) ptr; + status = yajl_gen_bool(g, *b); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_enum: { + ProtobufCEnumDescriptor *enum_descr; + bool enum_found; + unsigned enum_value; + + enum_found = false; + enum_descr = (ProtobufCEnumDescriptor *) field->descr->descriptor; + + enum_value = *((unsigned *) ptr); + for (i = 0; i < enum_descr->n_values; i++) { + if ((unsigned) enum_descr->values[i].value == enum_value) { + status = yajl_gen_string(g, (unsigned char*) enum_descr->values[i].name, strlen(enum_descr->values[i].name)); + assert(status == yajl_gen_status_ok); + enum_found = true; + break; + } + } + if (enum_found == false) { + status = yajl_gen_integer(g, enum_value); + assert(status == yajl_gen_status_ok); + } + break; + } + case nmsg_msgmod_ft_ip: { + char sip[INET6_ADDRSTRLEN]; + int family = 0; + + bdata = (ProtobufCBinaryData *) ptr; + + if (bdata->len == 4) { + family = AF_INET; + } else if (bdata->len == 16) { + family = AF_INET6; + } + + if (family && inet_ntop(family, bdata->data, sip, sizeof(sip))) { + status = yajl_gen_string(g, (const unsigned char*)sip, strlen(sip)); + assert(status == yajl_gen_status_ok); + } else { + status = yajl_gen_null(g); + assert(status == yajl_gen_status_ok); + } + break; + } + case nmsg_msgmod_ft_uint16: { + uint32_t val; + memcpy(&val, ptr, sizeof(uint32_t)); + status = yajl_gen_integer(g, (uint16_t) val); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_uint32: { + uint32_t val; + memcpy(&val, ptr, sizeof(uint32_t)); + status = yajl_gen_integer(g, val); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_uint64: { + uint64_t val; + struct nmsg_strbuf *sb = NULL; + + sb = nmsg_strbuf_init(); + if (sb == NULL) + return (nmsg_res_memfail); + + memcpy(&val, ptr, sizeof(uint64_t)); + nmsg_strbuf_append(sb, "%" PRIu64, val); + + status = yajl_gen_number(g, (const char*) sb->data, strlen(sb->data)); + assert(status == yajl_gen_status_ok); + + nmsg_strbuf_destroy(&sb); + + break; + } + case nmsg_msgmod_ft_int16: { + int32_t val; + memcpy(&val, ptr, sizeof(int32_t)); + status = yajl_gen_integer(g, (int16_t) val); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_int32: { + int32_t val; + memcpy(&val, ptr, sizeof(int32_t)); + status = yajl_gen_integer(g, val); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_int64: { + int64_t val; + memcpy(&val, ptr, sizeof(int64_t)); + status = yajl_gen_integer(g, val); + assert(status == yajl_gen_status_ok); + break; + } + case nmsg_msgmod_ft_double: { + double val; + memcpy(&val, ptr, sizeof(double)); + status = yajl_gen_double(g, val); + assert(status == yajl_gen_status_ok); + break; + } + default: { + status = yajl_gen_null(g); + assert(status == yajl_gen_status_ok); + break; + } + } /* end switch */ + + return (nmsg_res_success); +} +#else /* HAVE_YAJL */ +nmsg_res +_nmsg_message_payload_to_json(struct nmsg_message *msg, char **pres) { + return (nmsg_res_notimpl); +} + +nmsg_res +_nmsg_message_payload_to_json_load(struct nmsg_message *msg, + struct nmsg_msgmod_field *field, void *ptr, + void * gen) { + return (nmsg_res_notimpl); +} +#endif /* HAVE_YAJL */ diff --git a/nmsg/msgmod_plugin.h b/nmsg/msgmod_plugin.h index 36ffdfb18..3e00948a6 100644 --- a/nmsg/msgmod_plugin.h +++ b/nmsg/msgmod_plugin.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008, 2009, 2013 by Farsight Security, Inc. + * Copyright (c) 2008, 2009, 2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,6 +70,21 @@ typedef nmsg_res (*nmsg_msgmod_field_get_fp)(nmsg_message_t m, size_t *len, void *msg_clos); +/** Custom field formatter function. */ +typedef nmsg_res (*nmsg_msgmod_field_format_fp)(nmsg_message_t m, + struct nmsg_msgmod_field *field, + void *ptr, + struct nmsg_strbuf *sb, + const char *endline); + +/** Custom field parser function. */ +typedef nmsg_res (*nmsg_msgmod_field_parse_fp)(nmsg_message_t m, + struct nmsg_msgmod_field *field, + const char *value, + void **ptr, + size_t *len, + const char *endline); + /** Convenience macro. */ #define NMSG_MSGMOD_FIELD_PRINTER(funcname) \ nmsg_res funcname(nmsg_message_t m, \ @@ -87,6 +102,23 @@ typedef nmsg_res (*nmsg_msgmod_field_get_fp)(nmsg_message_t m, size_t *len, \ void *msg_clos) +/** Convenience macro. */ +#define NMSG_MSGMOD_FIELD_FORMATTER(funcname) \ + nmsg_res funcname(nmsg_message_t m, \ + struct nmsg_msgmod_field *field, \ + void *ptr, \ + struct nmsg_strbuf *sb, \ + const char *endline) + +/** Convenience macro. */ +#define NMSG_MSGMOD_FIELD_PARSER(funcname) \ + nmsg_res funcname(nmsg_message_t m, \ + struct nmsg_msgmod_field *field, \ + const char *value, \ + void **ptr, \ + size_t *len, \ + const char *endline) + /** Convenience macro. */ #define NMSG_MSGMOD_REQUIRED_INIT \ .msgver = NMSG_MSGMOD_VERSION, \ @@ -119,9 +151,13 @@ struct nmsg_msgmod_field { /** \private, must be initialized to NULL. */ const ProtobufCFieldDescriptor *descr; + /* Optional custom field formatter function. */ + nmsg_msgmod_field_format_fp format; + + /* Optional custom field parser function. */ + nmsg_msgmod_field_parse_fp parse; + /** \private Reserved fields. */ - void *_reserved3; - void *_reserved2; void *_reserved1; void *_reserved0; }; diff --git a/nmsg/output.c b/nmsg/output.c index e7fa0eea8..874192c3d 100644 --- a/nmsg/output.c +++ b/nmsg/output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -85,6 +85,39 @@ nmsg_output_open_pres(int fd) { return (output); } +#ifdef HAVE_YAJL +nmsg_output_t +nmsg_output_open_json(int fd) { + struct nmsg_output *output; + + output = calloc(1, sizeof(*output)); + if (output == NULL) + return (NULL); + output->type = nmsg_output_type_json; + output->write_fp = _output_json_write; + + output->json = calloc(1, sizeof(*(output->json))); + if (output->json == NULL) { + free(output); + return (NULL); + } + output->json->fp = fdopen(fd, "w"); + if (output->json->fp == NULL) { + free(output->json); + free(output); + return (NULL); + } + pthread_mutex_init(&output->json->lock, NULL); + + return (output); +} +#else /* HAVE_YAJL */ +nmsg_output_t +nmsg_output_open_json(int fd) { + return (NULL); +} +#endif /* HAVE_YAJL */ + nmsg_output_t nmsg_output_open_callback(nmsg_cb_message cb, void *user) { struct nmsg_output *output; @@ -160,6 +193,10 @@ nmsg_output_close(nmsg_output_t *output) { free((*output)->pres->endline); free((*output)->pres); break; + case nmsg_output_type_json: + fclose((*output)->json->fp); + free((*output)->json); + break; case nmsg_output_type_callback: free((*output)->callback); break; diff --git a/nmsg/output.h b/nmsg/output.h index eced5405e..3371c4146 100644 --- a/nmsg/output.h +++ b/nmsg/output.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2012 by Farsight Security, Inc. + * Copyright (c) 2008-2012, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ typedef enum { nmsg_output_type_stream, nmsg_output_type_pres, + nmsg_output_type_json, nmsg_output_type_callback } nmsg_output_type; @@ -128,6 +129,16 @@ nmsg_output_open_xs_endpoint(void *xs_ctx, const char *ep, size_t bufsz); nmsg_output_t nmsg_output_open_pres(int fd); +/** + * Initialize a new JSON format nmsg output. + * + * \param[in] fd Writable file descriptor. + * + * \return Opaque pointer that is NULL on failure or non-NULL on success. + */ +nmsg_output_t +nmsg_output_open_json(int fd); + /** * Initialize a new nmsg output closure. This allows a user-provided callback to * function as an nmsg output, for instance to participate in an nmsg_io loop. diff --git a/nmsg/output_json.c b/nmsg/output_json.c new file mode 100644 index 000000000..2a08b5c01 --- /dev/null +++ b/nmsg/output_json.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Import. */ + +#include "private.h" + +/* Internal functions. */ + +nmsg_res +_output_json_write(nmsg_output_t output, nmsg_message_t msg) { + nmsg_res res; + char *json_data; + + /* lock output */ + pthread_mutex_lock(&output->json->lock); + + res = nmsg_message_to_json(msg, &json_data); + if (res != nmsg_res_success) { + goto out; + } + + fprintf(output->pres->fp, "%s\n", json_data); + if (output->pres->flush) + fflush(output->pres->fp); + free(json_data); + +out: + /* unlock output */ + pthread_mutex_unlock(&output->json->lock); + + return (nmsg_res_success); +} diff --git a/nmsg/private.h b/nmsg/private.h index 51470a03c..4b1bd3c63 100644 --- a/nmsg/private.h +++ b/nmsg/private.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2014 by Farsight Security, Inc. + * Copyright (c) 2008-2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,6 +55,11 @@ #include +#ifdef HAVE_YAJL +#include +#include +#endif /* HAVE_YAJL */ + #ifdef HAVE_LIBXS # include #endif /* HAVE_LIBXS */ @@ -69,6 +74,8 @@ #include "libmy/list.h" #include "libmy/tree.h" #include "libmy/ubuf.h" +#include "libmy/b64_decode.h" +#include "libmy/b64_encode.h" /* Macros. */ @@ -115,6 +122,7 @@ struct nmsg_frag; struct nmsg_frag_key; struct nmsg_frag_tree; struct nmsg_input; +struct nmsg_json; struct nmsg_output; struct nmsg_msgmod; struct nmsg_msgmod_field; @@ -219,6 +227,15 @@ struct nmsg_pres { char *endline; }; +/* nmsg_json: used by nmsg_input and nmsg_output */ +struct nmsg_json { +#ifdef HAVE_YAJL +#endif /* HAVE_YAJL */ + pthread_mutex_t lock; + FILE *fp; + bool flush; +}; + /* nmsg_stream_input: used by nmsg_input */ struct nmsg_stream_input { nmsg_stream_type type; @@ -294,6 +311,7 @@ struct nmsg_input { struct nmsg_stream_input *stream; struct nmsg_pcap *pcap; struct nmsg_pres *pres; + struct nmsg_json *json; struct nmsg_callback_input *callback; }; nmsg_input_read_fp read_fp; @@ -311,6 +329,7 @@ struct nmsg_output { union { struct nmsg_stream_output *stream; struct nmsg_pres *pres; + struct nmsg_json *json; struct nmsg_callback_output *callback; }; nmsg_output_write_fp write_fp; @@ -392,6 +411,7 @@ struct nmsg_msgvendor { struct nmsg_msgmod { struct nmsg_msgmod_plugin *plugin; struct nmsg_msgmod_field *fields; + struct nmsg_msgmod_field **fields_idx; size_t n_fields; }; @@ -479,6 +499,9 @@ nmsg_res _input_pcap_read_raw(nmsg_input_t, nmsg_message_t *); /* from input_pres.c */ nmsg_res _input_pres_read(nmsg_input_t, nmsg_message_t *); +/* from input_json.c */ +nmsg_res _input_json_read(nmsg_input_t, nmsg_message_t *); + /* from input_seqsrc.c */ struct nmsg_seqsrc * _input_seqsrc_get(nmsg_input_t, Nmsg__Nmsg *); void _input_seqsrc_destroy(nmsg_input_t); @@ -503,6 +526,9 @@ nmsg_res _output_nmsg_write_xs(nmsg_output_t, uint8_t *buf, size_t len); /* from output_pres.c */ nmsg_res _output_pres_write(nmsg_output_t, nmsg_message_t); +/* from output_json.c */ +nmsg_res _output_json_write(nmsg_output_t, nmsg_message_t); + /* from brate.c */ struct nmsg_brate * _nmsg_brate_init(size_t target_byte_rate); void _nmsg_brate_destroy(struct nmsg_brate **); diff --git a/src/io.c b/src/io.c index f3769cf11..7311147cd 100644 --- a/src/io.c +++ b/src/io.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -547,3 +547,72 @@ add_pres_output(nmsgtool_ctx *c, const char *fname) { fname); c->n_outputs += 1; } + +#ifdef HAVE_YAJL +void +add_json_input(nmsgtool_ctx *c, const char *fname) { + nmsg_input_t input; + nmsg_res res; + + input = nmsg_input_open_json(open_rfile(fname)); + res = nmsg_io_add_input(c->io, input, NULL); + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_input() failed\n", + argv_program); + exit(1); + } + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg json input: %s\n", argv_program, + fname); + c->n_inputs += 1; +} +#else /* HAVE_YAJL */ +void +add_json_input(nmsgtool_ctx *c, const char *fname) { + fprintf(stderr, "%s: Error: compiled without yajl support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_YAJL */ + +#ifdef HAVE_YAJL +void +add_json_output(nmsgtool_ctx *c, const char *fname) { + nmsg_output_t output; + nmsg_res res; + + if (c->kicker != NULL) { + struct kickfile *kf; + kf = calloc(1, sizeof(*kf)); + assert(kf != NULL); + + kf->cmd = c->kicker; + kf->basename = strdup(fname); + kickfile_rotate(kf); + + output = nmsg_output_open_json(open_wfile(kf->tmpname)); + setup_nmsg_output(c, output); + res = nmsg_io_add_output(c->io, output, (void *) kf); + } else { + output = nmsg_output_open_json(open_wfile(fname)); + setup_nmsg_output(c, output); + res = nmsg_io_add_output(c->io, output, NULL); + } + if (res != nmsg_res_success) { + fprintf(stderr, "%s: nmsg_io_add_output() failed\n", + argv_program); + exit(1); + } + if (c->debug >= 2) + fprintf(stderr, "%s: nmsg json output: %s\n", argv_program, + fname); + c->n_outputs += 1; +} +#else /* HAVE_YAJL */ +void +add_json_output(nmsgtool_ctx *c, const char *fname) { + fprintf(stderr, "%s: Error: compiled without yajl support\n", + argv_program); + exit(EXIT_FAILURE); +} +#endif /* HAVE_YAJL */ diff --git a/src/nmsgtool.c b/src/nmsgtool.c index f955d761a..2069728f0 100644 --- a/src/nmsgtool.c +++ b/src/nmsgtool.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -112,6 +112,16 @@ static argv_t args[] = { "file", "read pres format data from file" }, + { 'j', "readjson", + ARGV_CHAR_P | ARGV_FLAG_ARRAY, + &ctx.r_json, + "file", +#ifdef HAVE_YAJL + "read json format data from file" }, +#else /* HAVE_YAJL */ + "read json format data from file (no support)" }, +#endif /* HAVE_YAJL */ + { 'l', "readsock", ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.r_sock, @@ -164,6 +174,16 @@ static argv_t args[] = { "file", "write pres format data to file" }, + { 'J', "writejson", + ARGV_CHAR_P | ARGV_FLAG_ARRAY, + &ctx.w_json, + "file", +#ifdef HAVE_YAJL + "write json format data to file" }, +#else /* HAVE_YAJL */ + "write json format data to file (no support)" }, +#endif /* HAVE_YAJL */ + { 's', "writesock", ARGV_CHAR_P | ARGV_FLAG_ARRAY, &ctx.w_sock, diff --git a/src/nmsgtool.h b/src/nmsgtool.h index ff347dec5..b820d852f 100644 --- a/src/nmsgtool.h +++ b/src/nmsgtool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,9 +44,9 @@ typedef union nmsgtool_sockaddr nmsgtool_sockaddr; typedef struct { /* parameters */ - argv_array_t r_nmsg, r_pres, r_sock, r_xsock, r_channel, r_xchannel; + argv_array_t r_nmsg, r_pres, r_sock, r_xsock, r_channel, r_xchannel, r_json; argv_array_t r_pcapfile, r_pcapif; - argv_array_t w_nmsg, w_pres, w_sock, w_xsock; + argv_array_t w_nmsg, w_pres, w_sock, w_xsock, w_json; bool help, mirror, unbuffered, zlibout, daemon, version; char *endline, *kicker, *mname, *vname, *bpfstr; int debug; @@ -102,6 +102,8 @@ void add_pcapfile_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pcapif_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pres_input(nmsgtool_ctx *, nmsg_msgmod_t, const char *); void add_pres_output(nmsgtool_ctx *, const char *); +void add_json_input(nmsgtool_ctx *, const char *); +void add_json_output(nmsgtool_ctx *, const char *); void add_sock_input(nmsgtool_ctx *, const char *); void add_sock_output(nmsgtool_ctx *, const char *); void add_xsock_input(nmsgtool_ctx *, const char *); diff --git a/src/process_args.c b/src/process_args.c index 17905929c..e046d1d0c 100644 --- a/src/process_args.c +++ b/src/process_args.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2013 by Farsight Security, Inc. + * Copyright (c) 2008-2013, 2015 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -309,6 +309,10 @@ process_args(nmsgtool_ctx *c) { process_args_loop_mod(c->r_pres, add_pres_input, mod); process_args_loop(c->w_pres, add_pres_output); + /* json inputs and outputs */ + process_args_loop(c->r_json, add_json_input); + process_args_loop(c->w_json, add_json_output); + #undef process_args_loop #undef process_args_loop_mod