diff --git a/README.md b/README.md index 378ba414..dd6da682 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ OpenLI -- open source ETSI-compliant Lawful Intercept software -Version: 1.1.5 +Version: 1.1.6 --------------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 8dfc6da8..fc0d5047 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ # Super primitive configure script -AC_INIT([openli],[1.1.5],[shane@alcock.co.nz]) +AC_INIT([openli],[1.1.6],[shane@alcock.co.nz]) AM_INIT_AUTOMAKE([subdir-objects]) AC_CONFIG_SRCDIR(src/collector/collector.c) diff --git a/debian/changelog b/debian/changelog index 5cca3992..b630e5fa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,25 @@ +openli (1.1.6-1) unstable; urgency=medium + + * Fix incorrect encoding of userLocationInformation field. + * Add support for IMSI and IMEI as target identifiers for mobile + data (IP) intercepts. + * Added new parameter for IP intercepts: mobileident -- this is + used to indicate whether the user identifier for a mobile data + intercept is an MSISDN, IMEI or IMSI. + * Add support for including SIP packets in pcapdisk output for + VoIP intercepts. + * Fix bug where mediators receiving message from a collector via + RabbitMQ would be disconnected due to regular consumer timeouts. + This in turn should resolve issues where old IRIs or CCs would be + periodically retransmitted by a mediator to the LEA. + * Fix memory errors when reassembling TCP segments in the collector + libtrace threads. + * Generate error log messages when a component (either mediator or + collector) cannot publish to RabbitMQ due to the connection being + blocked. + + -- Shane Alcock Mon, 1 Jul 2024 09:57:07 +1200 + openli (1.1.5-1) unstable; urgency=medium * Fix bug where pcap disk output was not producing pcap files for IP diff --git a/doc/ProvisionerDoc.md b/doc/ProvisionerDoc.md index 3f3ba2d9..a740d22a 100644 --- a/doc/ProvisionerDoc.md +++ b/doc/ProvisionerDoc.md @@ -165,7 +165,11 @@ intercept must be configured with the following parameters: * Access type -- the technology used to provide the target with Internet access (e.g. DSL, Fiber, Wireless, etc). * User -- the username assigned to that user within your AAA system. This is - required, even if the target is only using static IP addresses. + required, even if the target is only using static IP addresses. For mobile + intercepts, this should be either the MSISDN, IMSI, or IMEI of the target + device. +* Mobile Identifier -- (for mobile intercepts only) indicates whether the + target is to be identified based on their MSISDN, IMSI, or IMEI. An IP intercept may also include ONE of the following parameters, which is used to identify the intercept target. @@ -177,6 +181,10 @@ used to identify the intercept target. traffic into the OpenLI collector(s), any mirrored traffic with an intercept ID that matches this value will be treated as belonging to this OpenLI IP intercept. +* Cisco Mirror ID -- if you are using Cisco packet mirroring to feed + intercepted traffic into an OpenLI collector, any mirrored traffic with + an intercept ID that matches this value will be assumed to belong to this + OpenLI IP intercept. * Static IPs -- if the target has a static IP (range), you can use this parameter to tell OpenLI which IPs belong to the target. @@ -194,8 +202,11 @@ as the one that is receiving the mirrored packets. For mobile IP intercepts, there are some slight differences. The Access type must be set to "mobile" to tell OpenLI to detect IP sessions using mobile session management protocols (such as GTP), instead of RADIUS. The User must -also be set to the target's phone number (MSISDN). The ALU Shim and JMirror -methods do not apply to mobile IP intercepts. +also be set to either the MSISDN, IMSI, or IMEI of the device that is to be +intercepted. You must use the "Mobile Identifier" parameter to tell OpenLI +which type of identifier is described by the User field. + +The vendor mirroring interception methods do not apply to mobile IP intercepts. #### Using the RADIUS Calling Station ID to Identify IP Intercept Targets In a conventional RADIUS deployment, the identity of the subscriber can be @@ -234,17 +245,25 @@ to or from your SIP and RADIUS servers. SIP servers are defined using the sipservers option. Each SIP server that you have in your network should be included as a list item within the 'sipservers' option. Failure to configure SIP servers will prevent OpenLI from -performing any VOIP intercepts. A SIP server is configured using two parameters: +performing any VOIP intercepts. A SIP server is configured using the +following parameters: * ip -- the IP address of the SIP server -* port -- the port that the SIP server is listening on. +* port_lower -- the lowest port number that the SIP server is listening on. +* port_upper -- the highest port number that the SIP server is listening on. RADIUS servers are defined using the 'radiusservers' option. The configuration works much the same as for SIP, except that most RADIUS deployments will need -TWO server entries: one for the auth service and one for the accounting service, -as these are usually listening on different ports. A RADIUS server entry is -configured using two parameters: +to ensure that their port range covers both the auth service and the accounting +service, as these are usually listening on different ports. A RADIUS server +entry is configured using the same parameters as a SIP server, i.e.: * ip -- the IP address of the RADIUS server -* port -- the port that the RADIUS server is communicating on. +* port_lower -- the lowest port number that the RADIUS server is listening on. +* port_upper -- the highest port number that the RADIUS server is listening on. + +For SIP and RADIUS servers that are only listening on a single port, you may +choose to omit `port_lower` and `port_upper` and instead provide the following +parameter: +* port -- the single port that the server is listening on. ### Email Servers @@ -442,21 +461,28 @@ An IP intercept must contain the following key-value elements: * `liid` -- the LIID * `authcountrycode` -- the authorisation country code * `deliverycountrycode` -- the delivery country code -* `user` -- the AAA username for the target +* `user` -- the AAA username for the target, or the target + identifier for mobile intercepts * `mediator` -- the ID of the mediator which will forward the intercept * `agencyid` -- the internal identifier of the agency that requested the intercept -* `accesstype` -- the access type providied to the user, will - default to 'undefined' if not set. +* `accesstype` -- the access type provided to the user, will + default to 'undefined' if not set +* `mobileident` -- (required for mobile intercepts only) the type + of identifier specified in the `user` element Valid access types are: 'dialup', 'adsl', 'vdsl', 'fiber', 'wireless', 'lan', 'satellite', 'wimax', 'cable', 'mobile' and 'wireless-other'. +Valid mobileident values are: + 'imsi', 'msisdn', and 'imei'. If not specified, the default is `msisdn`. + Note that setting the access type to 'mobile' will cause OpenLI to use GTPv2 traffic to identify the target's IP sessions, and the resulting ETSI records -will conform to the UMTS format (as opposed to the standard IP format). +will conform to the UMTS format (as opposed to the standard IP format +defined in ETSI TS 102 232-3). Optional key-value elements for an IP intercept are: diff --git a/doc/exampleconfigs/running-intercept-example.yaml b/doc/exampleconfigs/running-intercept-example.yaml index 00a3d9bf..83f77be8 100644 --- a/doc/exampleconfigs/running-intercept-example.yaml +++ b/doc/exampleconfigs/running-intercept-example.yaml @@ -13,19 +13,23 @@ # List of SIP servers on our network (for managing VOIP intercepts) sipservers: + + # This server is listening on a single port -- we could have used the + # 'port' parameter instead, but I want to demonstrate how port_lower and + # port_upper would look for a single port configuration. - ip: 192.168.110.100 - port: 5060 + port_lower: 5060 + port_upper: 5060 # List of RADIUS servers on our network (for managing IP intercepts). -# NOTE: we have two entries with the same IP -- one for the Auth service -# on port 1812 and one for the Accounting service on port 1813. +# NOTE: the port range below must cover the listening ports for both the +# Authentication and Accounting services. # NOTE: make sure you are mirroring ALL RADIUS traffic into your collector(s), # even the Accounting-Response records as these are necessary for ETSI LI. radiusservers: - ip: 10.199.0.253 - port: 1812 - - ip: 10.199.0.253 - port: 1813 + port_lower: 1812 + port_upper: 1813 # List of GTP servers on our network (for managing mobile IP intercepts) gtpservers: @@ -143,6 +147,8 @@ ipintercepts: mediator: 6001 # ID of the mediator to send intercept via agencyid: "Police" # ID of agency to send intercept to accesstype: "mobile" # Must be "mobile" for UMTS intercepts + mobileident: "msisdn" # Intercept sessions where the MSISDN matches + # the "user" value payloadencryption: "aes-192-cbc" # Encrypt IP content using AES-192-CBC encryptionkey: "alongencryptionkeyisgood" # Key to use for encryption, # should be provided by the agency diff --git a/rpm/openli.spec b/rpm/openli.spec index 86573f78..eae409ea 100644 --- a/rpm/openli.spec +++ b/rpm/openli.spec @@ -1,5 +1,5 @@ Name: openli -Version: 1.1.5 +Version: 1.1.6 Release: 1%{?dist} Summary: Software for performing ETSI-compliant lawful intercept @@ -283,6 +283,9 @@ fi %changelog +* Mon Jul 1 2024 Shane Alcock - 1.1.6-1 +- Updated for 1.1.6 release + * Wed May 8 2024 Shane Alcock - 1.1.5-1 - Updated for 1.1.5 release diff --git a/src/collector/accessplugins/gtp.c b/src/collector/accessplugins/gtp.c index 7db7a87a..32994a86 100644 --- a/src/collector/accessplugins/gtp.c +++ b/src/collector/accessplugins/gtp.c @@ -114,12 +114,9 @@ struct gtp_infelem { gtp_infoelem_t *next; }; -typedef struct gtp_userid { - char *imsi; - char *msisdn; - -} PACKED gtp_user_identity_t; - +/* Stored copies of the IEs that we need to include in IRI messages, in their + * original binary format (for easier encoding). + */ typedef struct gtp_sess_saved { uint8_t *imsi; uint16_t imsi_len; @@ -134,14 +131,19 @@ typedef struct gtp_sess_saved { uint8_t loc_version; } PACKED gtp_sess_saved_t; +typedef struct gtp_saved_packet gtp_saved_pkt_t; + typedef struct gtp_session { char *sessid; - gtp_user_identity_t userid; gtp_sess_saved_t saved; - char idstr[64]; - int idstr_len; + char idstr_msisdn[64]; + int idstr_msisdn_len; + char idstr_imsi[64]; + int idstr_imsi_len; + char idstr_imei[64]; + int idstr_imei_len; uint32_t teid; internetaccess_ip_t *pdpaddrs; @@ -153,9 +155,15 @@ typedef struct gtp_session { uint8_t serveripfamily; session_state_t current; -} gtp_session_t; -typedef struct gtp_saved_packet gtp_saved_pkt_t; + + uint64_t last_reqid; + uint8_t last_reqtype; + session_state_t savedoldstate; + session_state_t savednewstate; + gtp_saved_pkt_t *lastsavedpkt; + +} gtp_session_t; struct gtp_saved_packet { uint64_t reqid; @@ -189,6 +197,7 @@ typedef struct gtp_parsed { char imsi[16]; char msisdn[16]; + char imei[16]; gtp_saved_pkt_t *request; gtp_saved_pkt_t *response; @@ -225,6 +234,7 @@ static void reset_parsed_pkt(gtp_parsed_t *parsed) { parsed->serveripfamily = 0; memset(parsed->serverid, 0, 16); memset(parsed->imsi, 0, 16); + memset(parsed->imei, 0, 16); memset(parsed->msisdn, 0, 16); parsed->ies = NULL; @@ -254,14 +264,6 @@ static inline void destroy_gtp_session(gtp_session_t *sess) { free(sess->sessid); } - if (sess->userid.imsi) { - free(sess->userid.imsi); - } - - if (sess->userid.msisdn) { - free(sess->userid.msisdn); - } - if (sess->saved.imei) { free(sess->saved.imei); } @@ -579,6 +581,9 @@ static int walk_gtpv1_ies(gtp_parsed_t *parsedpkt, uint8_t *ptr, uint32_t rem, if (ietype == GTPV1_IE_IMSI) { get_gtpnum_from_ie(gtpel, parsedpkt->imsi, 0); } + if (ietype == GTPV1_IE_MEI) { + get_gtpnum_from_ie(gtpel, parsedpkt->imei, 0); + } if (ietype == GTPV1_IE_MSISDN) { get_gtpnum_from_ie(gtpel, parsedpkt->msisdn, 1); } @@ -631,6 +636,9 @@ static void walk_gtpv2_ies(gtp_parsed_t *parsedpkt, uint8_t *ptr, uint32_t rem, if (ietype == GTPV2_IE_IMSI) { get_gtpnum_from_ie(gtpel, parsedpkt->imsi, 0); } + if (ietype == GTPV2_IE_MEI) { + get_gtpnum_from_ie(gtpel, parsedpkt->imei, 0); + } if (ietype == GTPV2_IE_MSISDN) { get_gtpnum_from_ie(gtpel, parsedpkt->msisdn, 0); } @@ -681,7 +689,7 @@ static int gtp_parse_v2_teid(gtp_global_t *glob, libtrace_packet_t *pkt, uint8_t *gtpstart, uint32_t rem) { uint8_t *ptr; - uint16_t len; + uint32_t len; gtpv2_header_teid_t *header = (gtpv2_header_teid_t *)gtpstart; @@ -720,7 +728,7 @@ static int gtp_parse_v1_teid(gtp_global_t *glob, libtrace_packet_t *pkt, uint8_t *gtpstart, uint32_t rem) { uint8_t *ptr; - uint16_t len; + uint32_t len; gtpv1_header_t *header = (gtpv1_header_t *)gtpstart; @@ -871,6 +879,60 @@ static void *gtp_parse_packet(access_plugin_t *p, libtrace_packet_t *pkt) { return glob->parsedpkt; } +static inline user_identity_t *copy_identifiers(gtp_parsed_t *gparsed, + int *numberids) { + + int x = 0; + user_identity_t *uids; + + uids = calloc(3, sizeof(user_identity_t)); + + if (gparsed->matched_session->idstr_msisdn[0] != '\0') { + uids[x].method = USER_IDENT_GTP_MSISDN; + uids[x].idstr = strdup(gparsed->matched_session->idstr_msisdn); + uids[x].idlength = gparsed->matched_session->idstr_msisdn_len; + *numberids += 1; + x ++; + } + if (gparsed->matched_session->idstr_imsi[0] != '\0') { + uids[x].method = USER_IDENT_GTP_IMSI; + uids[x].idstr = strdup(gparsed->matched_session->idstr_imsi); + uids[x].idlength = gparsed->matched_session->idstr_imsi_len; + *numberids += 1; + x ++; + } + if (gparsed->matched_session->idstr_imei[0] != '\0') { + uids[x].method = USER_IDENT_GTP_IMEI; + uids[x].idstr = strdup(gparsed->matched_session->idstr_imei); + uids[x].idlength = gparsed->matched_session->idstr_imei_len; + *numberids += 1; + x ++; + } + return uids; +} + +static void save_identifier_strings(gtp_parsed_t *gparsed, gtp_session_t *sess) +{ + if (gparsed->msisdn[0] != '\0') { + snprintf(sess->idstr_msisdn, 64, "%s", gparsed->msisdn); + sess->idstr_msisdn_len = strlen(sess->idstr_msisdn); + } else { + sess->idstr_msisdn_len = 0; + } + if (gparsed->imsi[0] != '\0') { + snprintf(sess->idstr_imsi, 64, "%s", gparsed->imsi); + sess->idstr_imsi_len = strlen(sess->idstr_imsi); + } else { + sess->idstr_imsi_len = 0; + } + if (gparsed->imei[0] != '\0') { + snprintf(sess->idstr_imei, 64, "%s", gparsed->imei); + sess->idstr_imei_len = strlen(sess->idstr_imei); + } else { + sess->idstr_imei_len = 0; + } +} + #define GEN_SESSID(sessid, gparsed, teid) \ if (gparsed->serveripfamily == 4) { \ snprintf(sessid, 64, "%u-%u", *(uint32_t *)gparsed->serverid, teid); \ @@ -900,18 +962,15 @@ static user_identity_t *gtp_get_userid(access_plugin_t *p, void *parsed, } if (gparsed->matched_session) { - uids = calloc(1, sizeof(user_identity_t)); - uids[0].method = USER_IDENT_GTP_MSISDN; - uids[0].idstr = strdup(gparsed->matched_session->idstr); - uids[0].idlength = gparsed->matched_session->idstr_len; - *numberids = 1; + uids = copy_identifiers(gparsed, numberids); return uids; } /* Need to look up the session */ GEN_SESSID((char *)sessid, gparsed, gparsed->teid); - if (gparsed->msgtype == GTPV1_DELETE_PDP_CONTEXT_REQUEST) { + if (gparsed->msgtype == GTPV1_DELETE_PDP_CONTEXT_REQUEST || + gparsed->msgtype == GTPV1_UPDATE_PDP_CONTEXT_REQUEST) { search = glob->alt_session_map; } else { search = glob->session_map; @@ -922,13 +981,9 @@ static user_identity_t *gtp_get_userid(access_plugin_t *p, void *parsed, if (pval) { gparsed->matched_session = (gtp_session_t *)(*pval); - if (gparsed->matched_session->idstr_len == 0 && + if (gparsed->matched_session->idstr_msisdn_len == 0 && gparsed->msisdn[0] != '\0') { - gparsed->matched_session->userid.msisdn = strdup(gparsed->msisdn); - snprintf(gparsed->matched_session->idstr, 64, "%s", - gparsed->msisdn); - gparsed->matched_session->idstr_len = - strlen(gparsed->matched_session->idstr); + save_identifier_strings(gparsed, gparsed->matched_session); } if (search == glob->alt_session_map) { @@ -950,16 +1005,7 @@ static user_identity_t *gtp_get_userid(access_plugin_t *p, void *parsed, } - if (gparsed->matched_session->idstr_len == 0) { - *numberids = 0; - return NULL; - } - - uids = calloc(1, sizeof(user_identity_t)); - uids[0].method = USER_IDENT_GTP_MSISDN; - uids[0].idstr = strdup(gparsed->matched_session->idstr); - uids[0].idlength = gparsed->matched_session->idstr_len; - *numberids = 1; + uids = copy_identifiers(gparsed, numberids); return uids; } @@ -1009,38 +1055,13 @@ static user_identity_t *gtp_get_userid(access_plugin_t *p, void *parsed, memcpy(sess->serverid, gparsed->serverid, 16); sess->serveripfamily = gparsed->serveripfamily; - /* For now, I'm going to just use the MSISDN as the user identity - * until I'm told otherwise. - */ - if (gparsed->msisdn[0] != '\0') { - sess->userid.msisdn = strdup(gparsed->msisdn); - snprintf(sess->idstr, 64, "%s", sess->userid.msisdn); - sess->idstr_len = strlen(sess->idstr); - } else { - sess->userid.msisdn = NULL; - sess->idstr_len = 0; - } - - if (gparsed->imsi[0] != '\0') { - sess->userid.imsi = strdup(gparsed->imsi); - } - JSLI(pval, glob->session_map, (unsigned char *)sess->sessid); *pval = (Word_t)sess; gparsed->matched_session = sess; + save_identifier_strings(gparsed, sess); - if (sess->idstr_len == 0) { - *numberids = 0; - return NULL; - } - - uids = calloc(1, sizeof(user_identity_t)); - uids[0].method = USER_IDENT_GTP_MSISDN; - uids[0].idstr = strdup(gparsed->matched_session->idstr); - uids[0].idlength = gparsed->matched_session->idstr_len; - uids[0].plugindata = NULL; - *numberids = 1; + uids = copy_identifiers(gparsed, numberids); return uids; } @@ -1224,9 +1245,8 @@ static void copy_session_params_v2(gtp_parsed_t *gparsed, } static void apply_gtp_fsm_logic(gtp_parsed_t *gparsed, access_action_t *action, - access_session_t *sess, gtp_saved_pkt_t *gpkt) { - - session_state_t current = gparsed->matched_session->current; + access_session_t *sess, gtp_saved_pkt_t *gpkt, + session_state_t current) { if (gpkt->version == 1) { copy_session_params_v1(gparsed, gpkt); @@ -1264,7 +1284,7 @@ static void apply_gtp_fsm_logic(gtp_parsed_t *gparsed, access_action_t *action, extract_gtp_assigned_ip_address(gpkt, sess, gparsed->matched_session); - } else if (gpkt->response_cause >= 192 && gpkt->response_cause <= 255) { + } else if (gpkt->response_cause >= 192) { current = SESSION_STATE_OVER; *action = ACCESS_ACTION_REJECT; } @@ -1322,12 +1342,33 @@ static access_session_t *gtp_update_session_state(access_plugin_t *p, gtp_parsed_t *gparsed = (gtp_parsed_t *)parsed; PWord_t pval; Word_t rcint; + uint64_t reqid = (((uint64_t)gparsed->teid) << 32) | + ((uint64_t)gparsed->seqno); + + if (gparsed->matched_session == NULL) { + *action = ACCESS_ACTION_NONE; + return NULL; + } + + if (reqid == gparsed->matched_session->last_reqid && + gparsed->msgtype == gparsed->matched_session->last_reqtype) { + + /* Do NOT save the packet, because we've already saved it when + * this method was called on a previous identity found in + * the packet. + */ + thissess = find_matched_session(p, sesslist, gparsed->matched_session, + gparsed->teid); + *oldstate = gparsed->matched_session->savedoldstate; + *action = gparsed->action; + *newstate = gparsed->matched_session->savednewstate; + return thissess; + } saved = calloc(1, sizeof(gtp_saved_pkt_t)); saved->type = gparsed->msgtype; - saved->reqid = (((uint64_t)gparsed->teid) << 32) | - ((uint64_t)gparsed->seqno); + saved->reqid = reqid; saved->ies = gparsed->ies; saved->version = gparsed->version; saved->matched_session = gparsed->matched_session; @@ -1336,7 +1377,11 @@ static access_session_t *gtp_update_session_state(access_plugin_t *p, saved->ipcontent = NULL; saved->iplen = 0; saved->response_cause = gparsed->response_cause; + gparsed->ies = NULL; + gparsed->matched_session->last_reqid = reqid; + gparsed->matched_session->last_reqtype = gparsed->msgtype; + gparsed->matched_session->lastsavedpkt = saved; openli_copy_ipcontent(gparsed->origpkt, &(saved->ipcontent), &(saved->iplen)); @@ -1349,17 +1394,25 @@ static access_session_t *gtp_update_session_state(access_plugin_t *p, if (gparsed->msgtype == GTPV2_CREATE_SESSION_REQUEST || gparsed->msgtype == GTPV2_DELETE_SESSION_REQUEST || gparsed->msgtype == GTPV1_CREATE_PDP_CONTEXT_REQUEST || - gparsed->msgtype == GTPV1_DELETE_PDP_CONTEXT_REQUEST) { + gparsed->msgtype == GTPV1_DELETE_PDP_CONTEXT_REQUEST || + gparsed->msgtype == GTPV1_UPDATE_PDP_CONTEXT_REQUEST) { thissess = find_matched_session(p, sesslist, gparsed->matched_session, gparsed->teid); if (thissess) { *oldstate = gparsed->matched_session->current; + gparsed->matched_session->savedoldstate = *oldstate; apply_gtp_fsm_logic(gparsed, &(gparsed->action), thissess, - saved); + saved, *oldstate); *newstate = gparsed->matched_session->current; + gparsed->matched_session->savednewstate = *newstate; saved->applied = 1; } + } else { + /* response but we've never seen the request? */ + gparsed->matched_session = NULL; + *action = ACCESS_ACTION_NONE; + return NULL; } } else { @@ -1433,24 +1486,29 @@ static access_session_t *gtp_update_session_state(access_plugin_t *p, gparsed->request->matched_session, gparsed->teid); *oldstate = gparsed->request->matched_session->current; gparsed->matched_session = gparsed->request->matched_session; + gparsed->matched_session->savedoldstate = *oldstate; } else if (gparsed->response->matched_session) { thissess = find_matched_session(p, sesslist, gparsed->response->matched_session, gparsed->teid); *oldstate = gparsed->response->matched_session->current; gparsed->matched_session = gparsed->response->matched_session; + gparsed->matched_session->savedoldstate = *oldstate; } if (thissess) { if (gparsed->request->applied == 0) { apply_gtp_fsm_logic(gparsed, &(gparsed->action), thissess, - gparsed->request); + gparsed->request, gparsed->matched_session->current); + gparsed->request->applied = 1; } if (gparsed->response->applied == 0) { apply_gtp_fsm_logic(gparsed, &(gparsed->action), thissess, - gparsed->response); + gparsed->response, gparsed->matched_session->current); + gparsed->response->applied = 1; } } *newstate = gparsed->matched_session->current; + gparsed->matched_session->savednewstate = *newstate; } *action = gparsed->action; @@ -1776,7 +1834,7 @@ static int gtp_generate_iri_data(access_plugin_t *p, void *parseddata, return 0; } else if (gparsed->action == ACCESS_ACTION_REJECT) { - printf("need to generate a PDP context activation failed IRI\n"); + /* TODO need to generate a PDP context activation failed IRI */ } else if (gparsed->action == ACCESS_ACTION_INTERIM_UPDATE) { *iritype = ETSILI_IRI_CONTINUE; diff --git a/src/collector/accessplugins/radius.c b/src/collector/accessplugins/radius.c index 0c2b3bc6..f39a92a5 100644 --- a/src/collector/accessplugins/radius.c +++ b/src/collector/accessplugins/radius.c @@ -1041,19 +1041,9 @@ static inline void nasid_to_string(radius_attribute_t *nasattr, char *strspace, assert (spacelen >= 256); - if (nasattr->att_len < 256) { - memcpy(strspace, nasattr->att_val, nasattr->att_len); - strspace[nasattr->att_len] = '\0'; - *keylen = nasattr->att_len; - } else { - memcpy(strspace, nasattr->att_val, 255); - strspace[255] = '\0'; - *keylen = 255; - logger(LOG_INFO, - "OpenLI RADIUS: NAS-Identifier is too long, truncated to %s", - strspace); - } - + memcpy(strspace, nasattr->att_val, nasattr->att_len); + strspace[nasattr->att_len] = '\0'; + *keylen = nasattr->att_len; } static inline void process_nasid_attribute(radius_parsed_t *raddata) { diff --git a/src/collector/alushim_parser.c b/src/collector/alushim_parser.c index eac6787c..f56780b9 100644 --- a/src/collector/alushim_parser.c +++ b/src/collector/alushim_parser.c @@ -95,7 +95,7 @@ int check_alu_intercept(collector_identity_t *info, colthread_local_t *loc, uint32_t rem = 0, shimintid, cin; void *l3, *l2; - if ((cs = match_packet_to_coreserver(alusources, pinfo)) == NULL) { + if ((cs = match_packet_to_coreserver(alusources, pinfo, 1)) == NULL) { return 0; } @@ -151,8 +151,8 @@ int check_alu_intercept(collector_identity_t *info, colthread_local_t *loc, if (!l3 || rem == 0) { logger(LOG_INFO, - "Warning: unable to find IP header of ALU-intercepted packet from mirror %s:%s (ID: %u)", - cs->ipstr, cs->portstr, shimintid); + "Warning: unable to find IP header of ALU-intercepted packet from mirror (ID: %u)", + cs->serverkey, shimintid); return -1; } diff --git a/src/collector/collector.c b/src/collector/collector.c index c01417f2..6c32f29c 100644 --- a/src/collector/collector.c +++ b/src/collector/collector.c @@ -504,42 +504,15 @@ static inline void send_packet_to_sync(libtrace_packet_t *pkt, void *q, uint8_t updatetype) { openli_state_update_t syncup; libtrace_packet_t *copy; - int caplen = trace_get_capture_length(pkt); - int framelen = trace_get_framing_length(pkt); - - if (caplen == -1 || framelen == -1) { - logger(LOG_INFO, "OpenLI: unable to copy packet for sync thread (caplen=%d, framelen=%d)", caplen, framelen); - exit(1); - } /* We do this ourselves instead of calling trace_copy_packet() because * we don't want to be allocating 64K per copied packet -- we could be * doing this a lot and don't want to be wasteful */ - copy = (libtrace_packet_t *)calloc((size_t)1, sizeof(libtrace_packet_t)); - if (!copy) { - logger(LOG_INFO, "OpenLI: out of memory while copying packet for sync thread"); + copy = openli_copy_packet(pkt); + if (copy == NULL) { exit(1); } - copy->trace = pkt->trace; - copy->buf_control = TRACE_CTRL_PACKET; - copy->buffer = malloc(framelen + caplen); - copy->type = pkt->type; - copy->header = copy->buffer; - copy->payload = ((char *)copy->buffer) + framelen; - copy->order = pkt->order; - copy->hash = pkt->hash; - copy->error = pkt->error; - copy->which_trace_start = pkt->which_trace_start; - copy->cached.capture_length = caplen; - copy->cached.framing_length = framelen; - copy->cached.wire_length = -1; - copy->cached.payload_length = -1; - /* everything else in cache should be 0 or NULL due to our earlier - * calloc() */ - memcpy(copy->header, pkt->header, framelen); - memcpy(copy->payload, pkt->payload, caplen); - syncup.type = updatetype; syncup.data.pkt = copy; @@ -624,35 +597,17 @@ static void add_payload_info_from_packet(libtrace_packet_t *pkt, } -static uint8_t is_sms_over_sip(packet_info_t *pinfo, libtrace_packet_t *pkt, - colthread_local_t *loc, collector_global_t *glob) { - uint8_t x = 0, doonce; +static void do_sms_check(colthread_local_t *loc, + libtrace_packet_t *pkt, collector_global_t *glob) { + + uint8_t ipsrc[16], ipdest[16]; int ipfamily; int is_sip = 0; - libtrace_packet_t *ref; uint32_t hashval = 0; char *callid, *cseq, *sipcontents; uint16_t siplen; uint32_t queueid; - uint8_t ipsrc[16], ipdest[16]; - - x = add_sip_packet_to_parser(&(loc->sipparser), pkt, 0); - if (x == SIP_ACTION_USE_PACKET) { - /* No fragments, no TCP reassembly required */ - doonce = 1; - ref = pkt; - } else if (x == SIP_ACTION_REASSEMBLE_TCP) { - /* Reassembled TCP, could contain multiple messages */ - ref = NULL; - doonce = 0; - } else if (x == SIP_ACTION_REASSEMBLE_IPFRAG) { - /* Reassembled IP/UDP fragment */ - doonce = 1; - ref = NULL; - } else { - return 0; - } memset(ipsrc, 0, 16); memset(ipdest, 0, 16); @@ -660,57 +615,118 @@ static uint8_t is_sms_over_sip(packet_info_t *pinfo, libtrace_packet_t *pkt, /* This error will get caught and logged by the VoIP sync thread * so we don't need to log it ourselves. */ + return; + } + + sipcontents = get_sip_contents(loc->sipparser, &siplen); + /* payload begins with "MESSAGE" == SMS */ + if (siplen > 8 && memcmp("MESSAGE ", sipcontents, 8) == 0) { + is_sip = 1; + } else { + /* CSEQ ends with " MESSAGE" == server response to SMS */ + cseq = get_sip_cseq(loc->sipparser); + if (cseq != NULL) { + int slen = strlen(cseq); + if (slen > 8 && memcmp(cseq + (slen - 8), " MESSAGE", 8) == 0) { + is_sip = 1; + } + } + free(cseq); + } + + if (is_sip) { + callid = get_sip_callid(loc->sipparser); + if (callid == NULL) { + logger(LOG_INFO, + "OpenLI: warning -- SIP SMS MESSAGE has no Call-Id"); + return; + } + hashval = hashlittle(callid, strlen(callid), 0xfffffffb); + queueid = hashval % glob->sms_threads; + send_packet_to_smsworker(sipcontents, siplen, ipsrc, ipdest, + ipfamily, loc->sms_worker_queues[queueid], + trace_get_timeval(pkt)); + + /* update global stats */ + pthread_mutex_lock(&(glob->stats_mutex)); + glob->stats.packets_sms ++; + pthread_mutex_unlock(&(glob->stats_mutex)); + } + +} + +static uint8_t sms_check_fast_path(colthread_local_t *loc, + libtrace_packet_t *pkt, collector_global_t *glob) { + + int x; + + x = parse_next_sip_message(loc->sipparser, NULL, NULL); + if (x <= 0) { return 0; } + do_sms_check(loc, pkt, glob); + return 0; +} + +static uint8_t sms_check_slow_path(colthread_local_t *loc, + libtrace_packet_t *pkt, collector_global_t *glob, + uint8_t doonce) { + + int x, i; + libtrace_packet_t **pkts = NULL; + int pkt_cnt = 0; do { - is_sip = 0; - x = parse_next_sip_message(loc->sipparser, ref); - if (x == 0) { - /* no more SIP content available */ - break; + if (pkts != NULL) { + for (i = 0; i < pkt_cnt; i++) { + if (pkts[i]) { + trace_destroy_packet(pkts[i]); + } + } + free(pkts); + pkt_cnt = 0; + pkts = NULL; } - if (x < 0) { + + x = parse_next_sip_message(loc->sipparser, &pkts, &pkt_cnt); + if (x == 0) { return 0; } + if (x < 0 || pkt_cnt == 0) { + continue; + } + do_sms_check(loc, pkts[0], glob); + } while (!doonce); - sipcontents = get_sip_contents(loc->sipparser, &siplen); - /* payload begins with "MESSAGE" == SMS */ - if (siplen > 8 && memcmp("MESSAGE ", sipcontents, 8) == 0) { - is_sip = 1; - } else { - /* CSEQ ends with " MESSAGE" == server response to SMS */ - cseq = get_sip_cseq(loc->sipparser); - if (cseq != NULL) { - int slen = strlen(cseq); - if (slen > 8 && memcmp(cseq + (slen - 8), " MESSAGE", 8) == 0) { - is_sip = 1; - } + if (pkts) { + for (i = 0; i < pkt_cnt; i++) { + if (pkts[i]) { + trace_destroy_packet(pkts[i]); } - free(cseq); } + free(pkts); + } - if (is_sip) { - callid = get_sip_callid(loc->sipparser); - if (callid == NULL) { - logger(LOG_INFO, - "OpenLI: warning -- SIP SMS MESSAGE has no Call-Id"); - continue; - } - hashval = hashlittle(callid, strlen(callid), 0xfffffffb); - queueid = hashval % glob->sms_threads; - send_packet_to_smsworker(sipcontents, siplen, ipsrc, ipdest, - ipfamily, loc->sms_worker_queues[queueid], - trace_get_timeval(pkt)); + return 0; +} - /* update global stats */ - pthread_mutex_lock(&(glob->stats_mutex)); - glob->stats.packets_sms ++; - pthread_mutex_unlock(&(glob->stats_mutex)); - } +static uint8_t is_sms_over_sip(packet_info_t *pinfo, libtrace_packet_t *pkt, + colthread_local_t *loc, collector_global_t *glob) { - } while (!doonce); + uint8_t x = 0, doonce; + libtrace_packet_t *ref; + x = add_sip_packet_to_parser(&(loc->sipparser), pkt, 0); + if (x == SIP_ACTION_USE_PACKET) { + /* No fragments, no TCP reassembly required */ + return sms_check_fast_path(loc, pkt, glob); + } else if (x == SIP_ACTION_REASSEMBLE_TCP) { + /* Reassembled TCP, could contain multiple messages */ + return sms_check_slow_path(loc, pkt, glob, 0); + } else if (x == SIP_ACTION_REASSEMBLE_IPFRAG) { + /* Reassembled IP/UDP fragment */ + return sms_check_slow_path(loc, pkt, glob, 1); + } return 0; } @@ -771,61 +787,10 @@ static inline uint8_t check_for_invalid_sip(packet_info_t *pinfo, static inline uint32_t is_core_server_packet(libtrace_packet_t *pkt, packet_info_t *pinfo, coreserver_t *servers) { - coreserver_t *rad, *tmp; coreserver_t *found = NULL; uint32_t hashval = 0; - if (pinfo->srcport == 0 || pinfo->destport == 0) { - return 0; - } - - HASH_ITER(hh, servers, rad, tmp) { - if (rad->info == NULL) { - rad->info = populate_addrinfo(rad->ipstr, rad->portstr, - SOCK_DGRAM); - if (!rad->info) { - logger(LOG_INFO, - "Removing %s:%s from %s server list due to getaddrinfo error", - rad->ipstr, rad->portstr, - coreserver_type_to_string(rad->servertype)); - - HASH_DELETE(hh, servers, rad); - continue; - } - if (rad->info->ai_family == AF_INET) { - rad->portswapped = ntohs(CS_TO_V4(rad)->sin_port); - } else if (rad->info->ai_family == AF_INET6) { - rad->portswapped = ntohs(CS_TO_V6(rad)->sin6_port); - } - } - - if (pinfo->family == AF_INET) { - struct sockaddr_in *sa; - sa = (struct sockaddr_in *)(&(pinfo->srcip)); - - if (CORESERVER_MATCH_V4(rad, sa, pinfo->srcport)) { - found = rad; - break; - } - sa = (struct sockaddr_in *)(&(pinfo->destip)); - if (CORESERVER_MATCH_V4(rad, sa, pinfo->destport)) { - found = rad; - break; - } - } else if (pinfo->family == AF_INET6) { - struct sockaddr_in6 *sa6; - sa6 = (struct sockaddr_in6 *)(&(pinfo->srcip)); - if (CORESERVER_MATCH_V6(rad, sa6, pinfo->srcport)) { - found = rad; - break; - } - sa6 = (struct sockaddr_in6 *)(&(pinfo->destip)); - if (CORESERVER_MATCH_V6(rad, sa6, pinfo->destport)) { - found = rad; - break; - } - } - } + found = match_packet_to_coreserver(servers, pinfo, 0); /* Doesn't match any of our known core servers */ if (found == NULL) { @@ -1005,7 +970,7 @@ static libtrace_packet_t *process_packet(libtrace_t *trace, if (glob->ciscomirrors) { coreserver_t *cs; if ((cs = match_packet_to_coreserver(glob->ciscomirrors, - &pinfo)) != NULL) { + &pinfo, 1)) != NULL) { if (glob->sharedinfo.cisco_noradius) { pthread_rwlock_rdlock(&(glob->config_mutex)); ret = generate_cc_from_cisco( @@ -2248,6 +2213,7 @@ int main(int argc, char *argv[]) { (glob->sslconf.ctx && glob->etsitls) ? glob->sslconf.ctx : NULL; //forwarder only needs CTX if ctx exists and is enabled glob->forwarders[i].RMQ_conf = glob->RMQ_conf; + glob->forwarders[i].ampq_blocked = 0; pthread_create(&(glob->forwarders[i].threadid), NULL, start_forwarding_thread, (void *)&(glob->forwarders[i])); diff --git a/src/collector/collector_base.h b/src/collector/collector_base.h index 8daa3d16..ba41b20f 100644 --- a/src/collector/collector_base.h +++ b/src/collector/collector_base.h @@ -72,6 +72,7 @@ typedef struct export_dest { int ssllasterror; amqp_bytes_t rmq_queueid; + uint8_t rmq_declared; UT_hash_handle hh_fd; UT_hash_handle hh_medid; @@ -150,7 +151,7 @@ struct upcoming_intercept_event { }; typedef struct upcoming_intercept_time { - uint64_t timestamp; + time_t timestamp; struct upcoming_intercept_event *events; } upcoming_intercept_time_t; @@ -241,6 +242,7 @@ typedef struct forwarding_thread_data { amqp_connection_state_t ampq_conn; amqp_socket_t *ampq_sock; + uint8_t ampq_blocked; openli_RMQ_config_t RMQ_conf; } forwarding_thread_data_t; diff --git a/src/collector/collector_forwarder.c b/src/collector/collector_forwarder.c index f07098fd..5bd4c541 100644 --- a/src/collector/collector_forwarder.c +++ b/src/collector/collector_forwarder.c @@ -94,6 +94,7 @@ static int add_new_destination(forwarding_thread_data_t *fwd, newdest->ssl = NULL; newdest->ssllasterror = 0; newdest->waitingforhandshake = 0; + newdest->rmq_declared = 0; if (fwd->ampq_conn) { snprintf(stringspace, 32, "ID%d", newdest->mediatorid); @@ -605,21 +606,6 @@ static void connect_export_targets(forwarding_thread_data_t *fwd) { continue; } - if (fwd->ampq_conn) { - amqp_queue_declare( - fwd->ampq_conn, - 1, - dest->rmq_queueid, - 0, - 1, - 0, - 0, - amqp_empty_table); - - if (amqp_get_rpc_reply(fwd->ampq_conn).reply_type != AMQP_RESPONSE_NORMAL ) { - logger(LOG_INFO, "OpenLI: Failed to declare queue"); - } - } JLI(jval2, fwd->destinations_by_fd, dest->fd); if (jval2 == NULL) { @@ -759,7 +745,7 @@ static int process_control_message(forwarding_thread_data_t *fwd) { return 1; } -static void rmq_write_buffered(forwarding_thread_data_t *fwd) { +static int rmq_write_buffered(forwarding_thread_data_t *fwd) { export_dest_t *dest; PWord_t jval; @@ -772,14 +758,9 @@ static void rmq_write_buffered(forwarding_thread_data_t *fwd) { JLN(jval, fwd->destinations_by_id, index); if (dest->fd != -1 && fwd->forcesend_rmq && - !dest->waitingforhandshake) { - /* XXX Warning: will block */ - if (transmit_heartbeat(dest->fd, dest->ssl) < 0) { - logger(LOG_INFO, - "OpenLI: failed to send heartbeat to mediator %s:%s", - dest->ipstr, dest->portstr); - disconnect_mediator(fwd, dest); - } + !dest->waitingforhandshake && !fwd->ampq_blocked) { + + append_heartbeat_to_buffer(&(dest->buffer)); } availsend = get_buffered_amount(&(dest->buffer)); @@ -787,21 +768,35 @@ static void rmq_write_buffered(forwarding_thread_data_t *fwd) { continue; } - /* - if (availsend < MIN_SEND_AMOUNT && fwd->forcesend_rmq == 0) { - continue; + if (fwd->ampq_conn && !fwd->ampq_blocked && dest->rmq_declared == 0) { + amqp_queue_declare( + fwd->ampq_conn, + 1, + dest->rmq_queueid, + 0, + 1, + 0, + 0, + amqp_empty_table); + + if (amqp_get_rpc_reply(fwd->ampq_conn).reply_type != AMQP_RESPONSE_NORMAL ) { + logger(LOG_INFO, "OpenLI: Failed to declare queue"); + } + dest->rmq_declared = 1; } - */ if (transmit_buffered_records_RMQ(&(dest->buffer), fwd->ampq_conn, 1, amqp_cstring_bytes(""), dest->rmq_queueid, - BUF_BATCH_SIZE) < 0 ) { + BUF_BATCH_SIZE, + &(fwd->ampq_blocked)) < 0 ) { logger(LOG_INFO, "OpenLI: Error Publishing to RMQ"); + return -1; } } + return 0; } static void complete_ssl_handshake(forwarding_thread_data_t *fwd, @@ -838,6 +833,7 @@ static inline int forwarder_main_loop(forwarding_thread_data_t *fwd) { int topollc, x, i; int towait = 10000; + /* Add the mediator confirmation timer to our poll item list, if * required. */ @@ -851,6 +847,69 @@ static inline int forwarder_main_loop(forwarding_thread_data_t *fwd) { topollc = fwd->nextpoll; } + if (fwd->RMQ_conf.enabled && fwd->ampq_conn == NULL) { + amqp_table_entry_t login_properties[1]; + amqp_table_t login_properties_table; + + amqp_table_entry_t client_capabilities[1]; + amqp_table_t client_capabilities_table; + if ( fwd->RMQ_conf.name && fwd->RMQ_conf.pass ) { + fwd->ampq_conn = amqp_new_connection(); + fwd->ampq_sock = amqp_tcp_socket_new(fwd->ampq_conn); + + //TODO RMQ instance will always be on localhost? (for collector) + if (amqp_socket_open(fwd->ampq_sock, "localhost", 5672 )){ + logger(LOG_INFO, + "OpenLI: RMQ forwarding thread %d failed to open amqp socket", + fwd->forwardid); + return 0; + } + + client_capabilities[0].key = amqp_cstring_bytes("connection.blocked"); + client_capabilities[0].value.kind = AMQP_FIELD_KIND_BOOLEAN; + client_capabilities[0].value.value.boolean = 1; + + client_capabilities_table.entries = client_capabilities; + client_capabilities_table.num_entries = 1; + + login_properties[0].key = amqp_cstring_bytes("capabilities"); + login_properties[0].value.kind = AMQP_FIELD_KIND_TABLE; + login_properties[0].value.value.table = client_capabilities_table; + + login_properties_table.entries = login_properties; + login_properties_table.num_entries = 1; + + /* login using PLAIN, must specify username and password */ + if ( (amqp_login_with_properties(fwd->ampq_conn, "OpenLI", 0, + AMQP_FRAME_MAX,0, &login_properties_table, + AMQP_SASL_METHOD_PLAIN, fwd->RMQ_conf.name, + fwd->RMQ_conf.pass) + ).reply_type != AMQP_RESPONSE_NORMAL ) { + logger(LOG_ERR, "OpenLI: RMQ Failed to login to broker using PLAIN auth"); + return 0; + } + + amqp_channel_open(fwd->ampq_conn, 1); + + if ( (amqp_get_rpc_reply(fwd->ampq_conn).reply_type) != AMQP_RESPONSE_NORMAL ) { + logger(LOG_ERR, "OpenLI: Failed to open channel"); + return 0; + } + logger(LOG_INFO, "OpenLI: Connected to RMQ instance"); + + if (check_rmq_connection_block_status(fwd->ampq_conn, + &(fwd->ampq_blocked)) < 0) { + logger(LOG_ERR, + "OpenLI: Error while checking status of new RMQ instance"); + return 0; + } + + } else { + logger(LOG_INFO, "OpenLI: Incomplete RMQ login information supplied"); + return 0; + } + } + while (1) { if ((x = zmq_poll(fwd->topoll, topollc, 10)) < 0) { if (errno == EINTR) { @@ -889,7 +948,7 @@ static inline int forwarder_main_loop(forwarding_thread_data_t *fwd) { fwd->forcesend_rmq = 1; its.it_interval.tv_sec = 0; its.it_interval.tv_nsec = 0; - its.it_value.tv_sec = 1; + its.it_value.tv_sec = 5; its.it_value.tv_nsec = 0; timerfd_settime(fwd->conntimerfd, 0, &its, NULL); @@ -915,11 +974,17 @@ static inline int forwarder_main_loop(forwarding_thread_data_t *fwd) { } } + if (fwd->ampq_conn) { /* Loop over all destinations and see if they have anything to * write to their queue. */ - rmq_write_buffered(fwd); + if (rmq_write_buffered(fwd) < 0) { + amqp_connection_close(fwd->ampq_conn, AMQP_REPLY_SUCCESS); + amqp_destroy_connection(fwd->ampq_conn); + fwd->ampq_conn = NULL; + fwd->ampq_sock = NULL; + } fwd->forcesend_rmq = 0; } @@ -955,6 +1020,7 @@ static inline int forwarder_main_loop(forwarding_thread_data_t *fwd) { } if (fwd->ampq_conn) { + fwd->topoll[i].events = 0; continue; } @@ -1098,40 +1164,6 @@ void *start_forwarding_thread(void *data) { goto haltforwarder; } - if (fwd->RMQ_conf.enabled) { - if ( fwd->RMQ_conf.name && fwd->RMQ_conf.pass ) { - fwd->ampq_conn = amqp_new_connection(); - fwd->ampq_sock = amqp_tcp_socket_new(fwd->ampq_conn); - - //TODO RMQ instance will always be on localhost? (for collector) - if (amqp_socket_open(fwd->ampq_sock, "localhost", 5672 )){ - logger(LOG_INFO, - "OpenLI: RMQ forwarding thread %d failed to open amqp socket", - fwd->forwardid); - goto haltforwarder; - } - - /* login using PLAIN, must specify username and password */ - if ( (amqp_login(fwd->ampq_conn, "OpenLI", 0, AMQP_FRAME_MAX,0, - AMQP_SASL_METHOD_PLAIN, fwd->RMQ_conf.name, - fwd->RMQ_conf.pass) - ).reply_type != AMQP_RESPONSE_NORMAL ) { - logger(LOG_ERR, "OpenLI: RMQ Failed to login to broker using PLAIN auth"); - goto haltforwarder; - } - - amqp_channel_open(fwd->ampq_conn, 1); - - if ( (amqp_get_rpc_reply(fwd->ampq_conn).reply_type) != AMQP_RESPONSE_NORMAL ) { - logger(LOG_ERR, "OpenLI: Failed to open channel"); - goto haltforwarder; - } - logger(LOG_INFO, "OpenLI: Connected to RMQ instance"); - } else { - logger(LOG_INFO, "OpenLI: Incomplete RMQ login information supplied"); - goto haltforwarder; - } - } forwarder_main(fwd); diff --git a/src/collector/collector_publish.c b/src/collector/collector_publish.c index 87ebe13f..71231811 100644 --- a/src/collector/collector_publish.c +++ b/src/collector/collector_publish.c @@ -126,7 +126,8 @@ void free_published_message(openli_export_recv_t *msg) { free(msg->data.mobiri.liid); } } else if (msg->type == OPENLI_EXPORT_RAW_SYNC || - msg->type == OPENLI_EXPORT_RAW_CC) { + msg->type == OPENLI_EXPORT_RAW_CC || + msg->type == OPENLI_EXPORT_RAW_IRI) { if (msg->data.rawip.liid) { free(msg->data.rawip.liid); } @@ -138,8 +139,9 @@ void free_published_message(openli_export_recv_t *msg) { free(msg); } -openli_export_recv_t *create_rawip_cc_job_from_ip(char *liid, - uint32_t destid, void *l3, uint32_t l3_len, struct timeval tv) { +static openli_export_recv_t *create_rawip_job_from_ip(char *liid, + uint32_t destid, void *l3, uint32_t l3_len, struct timeval tv, + uint8_t msgtype) { openli_export_recv_t *msg = NULL; openli_pcap_header_t *pcap; @@ -149,7 +151,7 @@ openli_export_recv_t *create_rawip_cc_job_from_ip(char *liid, return msg; } - msg->type = OPENLI_EXPORT_RAW_CC; + msg->type = msgtype; msg->destid = destid; msg->ts = tv; @@ -187,7 +189,29 @@ openli_export_recv_t *create_rawip_cc_job(char *liid, uint32_t destid, } tv = trace_get_timeval(pkt); - return create_rawip_cc_job_from_ip(liid, destid, l3, rem, tv); + return create_rawip_job_from_ip(liid, destid, l3, rem, tv, + OPENLI_EXPORT_RAW_CC); + +} + +openli_export_recv_t *create_rawip_iri_job(char *liid, uint32_t destid, + libtrace_packet_t *pkt) { + + void *l3; + uint32_t rem; + uint16_t ethertype; + struct timeval tv; + + l3 = trace_get_layer3(pkt, ðertype, &rem); + + if (l3 == NULL || rem == 0 || (ethertype != TRACE_ETHERTYPE_IP && + ethertype != TRACE_ETHERTYPE_IPV6)) { + return NULL; + } + + tv = trace_get_timeval(pkt); + return create_rawip_job_from_ip(liid, destid, l3, rem, tv, + OPENLI_EXPORT_RAW_IRI); } @@ -199,8 +223,8 @@ int push_vendor_mirrored_ipcc_job(void *pubqueue, if (common->targetagency == NULL || strcmp(common->targetagency, "pcapdisk") == 0) { - msg = create_rawip_cc_job_from_ip(common->liid, - common->destid, l3, rem, tv); + msg = create_rawip_job_from_ip(common->liid, + common->destid, l3, rem, tv, OPENLI_EXPORT_RAW_CC); } else { msg = calloc(1, sizeof(openli_export_recv_t)); diff --git a/src/collector/collector_publish.h b/src/collector/collector_publish.h index c8d9b7dd..9e700f88 100644 --- a/src/collector/collector_publish.h +++ b/src/collector/collector_publish.h @@ -60,6 +60,7 @@ enum { OPENLI_EXPORT_EMAILCC = 21, OPENLI_EXPORT_EMAILIRI = 22, OPENLI_EXPORT_RAW_CC = 23, + OPENLI_EXPORT_RAW_IRI = 24, }; /* This structure is also used for IPMMCCs since they require the same @@ -200,11 +201,35 @@ openli_export_recv_t *create_ipcc_job( uint32_t cin, char *liid, uint32_t destid, libtrace_packet_t *pkt, uint8_t dir); +/** Creates a raw IP packet encoding job using the OPENLI_EXPORT_RAW_CC type. + * + * Used to export IP packets that are being intercepted by pcapdisk + * IP data intercepts. + * + * @param liid The LIID that this packet has been intercepted for + * @param destid The mediator that should receive the raw IP packet + * @param pkt The packet that was intercepted + * + * @return an encoding job that is ready to be published using + * publish_openli_msg() + */ openli_export_recv_t *create_rawip_cc_job(char *liid, uint32_t destid, libtrace_packet_t *pkt); -openli_export_recv_t *create_rawip_cc_job_from_ip(char *liid, - uint32_t destid, void *l3, uint32_t l3_len, struct timeval tv); +/** Creates a raw IP packet encoding job using the OPENLI_EXPORT_RAW_IRI type. + * + * Used to export SIP packets that are being intercepted by pcapdisk + * VOIP intercepts. + * + * @param liid The LIID that this packet has been intercepted for + * @param destid The mediator that should receive the raw IP packet + * @param pkt The packet that was intercepted + * + * @return an encoding job that is ready to be published using + * publish_openli_msg() + */ +openli_export_recv_t *create_rawip_iri_job(char *liid, uint32_t destid, + libtrace_packet_t *pkt); int push_vendor_mirrored_ipcc_job(void *pubqueue, intercept_common_t *common, struct timeval tv, diff --git a/src/collector/collector_seqtracker.c b/src/collector/collector_seqtracker.c index 7c15bb52..ce0077e9 100644 --- a/src/collector/collector_seqtracker.c +++ b/src/collector/collector_seqtracker.c @@ -76,6 +76,7 @@ static inline char *extract_liid_from_job(openli_export_recv_t *recvd) { return recvd->data.mobiri.liid; case OPENLI_EXPORT_RAW_SYNC: case OPENLI_EXPORT_RAW_CC: + case OPENLI_EXPORT_RAW_IRI: return recvd->data.rawip.liid; case OPENLI_EXPORT_EMAILIRI: return recvd->data.emailiri.liid; @@ -99,6 +100,7 @@ static inline uint32_t extract_cin_from_job(openli_export_recv_t *recvd) { case OPENLI_EXPORT_UMTSIRI: return recvd->data.mobiri.cin; case OPENLI_EXPORT_RAW_SYNC: + case OPENLI_EXPORT_RAW_IRI: case OPENLI_EXPORT_RAW_CC: return recvd->data.rawip.cin; case OPENLI_EXPORT_EMAILIRI: @@ -475,14 +477,11 @@ static void seqtracker_main(seqtracker_thread_data_t *seqdata) { case OPENLI_EXPORT_EMAILCC: case OPENLI_EXPORT_RAW_SYNC: case OPENLI_EXPORT_RAW_CC: - run_encoding_job(seqdata, job); - sincepurge ++; - break; + case OPENLI_EXPORT_RAW_IRI: case OPENLI_EXPORT_IPCC: run_encoding_job(seqdata, job); sincepurge ++; - break; - + break; } } diff --git a/src/collector/collector_sync.c b/src/collector/collector_sync.c index 536460c4..fefd2e98 100644 --- a/src/collector/collector_sync.c +++ b/src/collector/collector_sync.c @@ -284,9 +284,15 @@ static int forward_provmsg_to_voipsync(collector_sync_t *sync, openli_intersync_msg_t topush; topush.msgtype = msgtype; - topush.msgbody = (uint8_t *)malloc(msglen); - memcpy(topush.msgbody, provmsg, msglen); - topush.msglen = msglen; + + if (provmsg && msglen > 0) { + topush.msgbody = (uint8_t *)malloc(msglen); + memcpy(topush.msgbody, provmsg, msglen); + topush.msglen = msglen; + } else { + topush.msgbody = NULL; + topush.msglen = 0; + } libtrace_message_queue_put(sync->intersyncq, &topush); return 1; @@ -440,8 +446,7 @@ static void generate_startend_ipiris(collector_sync_t *sync, create_ipiri_job_from_iprange(sync, ipr, ipint, irirequired); } - HASH_FIND(hh, sync->allusers, ipint->username, ipint->username_len, - user); + user = lookup_user_by_intercept(sync->allusers, ipint); if (user == NULL) { return; @@ -583,7 +588,7 @@ static void push_all_coreservers(coreserver_t *servers, static int send_to_provisioner(collector_sync_t *sync) { int ret; - openli_proto_msgtype_t err; + openli_proto_msgtype_t err = OPENLI_PROTO_NO_MESSAGE; ret = transmit_net_buffer(sync->outgoing, &err); if (ret == -1) { @@ -841,8 +846,7 @@ static inline void push_ipintercept_halt_to_threads(collector_sync_t *sync, remove_staticiprange(sync, ipr); } - HASH_FIND(hh, sync->allusers, ipint->username, ipint->username_len, - user); + user = lookup_user_by_intercept(sync->allusers, ipint); if (user == NULL) { return; @@ -916,8 +920,7 @@ static void push_ipintercept_update_to_threads(collector_sync_t *sync, } } - HASH_FIND(hh, sync->allusers, ipint->username, ipint->username_len, - user); + user = lookup_user_by_intercept(sync->allusers, ipint); if (user == NULL) { return; @@ -1089,7 +1092,7 @@ static void push_existing_user_sessions(collector_sync_t *sync, sync_sendq_t *tmp, *sendq; internet_user_t *user; - HASH_FIND(hh, sync->allusers, cept->username, cept->username_len, user); + user = lookup_user_by_intercept(sync->allusers, cept); if (user) { access_session_t *sess, *tmp2; @@ -1233,12 +1236,15 @@ static int update_modified_intercept(collector_sync_t *sync, int changed = 0; int encodingchanged = 0; - if (strcmp(ipint->username, modified->username) != 0) { + if (strcmp(ipint->username, modified->username) != 0 + || ipint->mobileident != modified->mobileident) { push_ipintercept_halt_to_threads(sync, ipint); remove_intercept_from_user_intercept_list(&sync->userintercepts, ipint); free(ipint->username); ipint->username = modified->username; + ipint->mobileident = modified->mobileident; + modified->username = NULL; add_intercept_to_user_intercept_list(&sync->userintercepts, ipint); push_existing_user_sessions(sync, ipint); @@ -1894,7 +1900,7 @@ static void push_all_active_intercepts(collector_sync_t *sync, HASH_ITER(hh_liid, intlist, orig, tmp) { /* Do we have a valid user that matches the target username? */ if (orig->username != NULL) { - HASH_FIND(hh, allusers, orig->username, orig->username_len, user); + user = lookup_user_by_intercept(allusers, orig); if (user) { HASH_ITER(hh, user->sessions, sess, tmp2) { push_single_ipintercept(sync, q, orig, sess); @@ -1932,12 +1938,6 @@ static int remove_ip_to_session_mapping(collector_sync_t *sync, sizeof(internetaccess_ip_t), mapping); if (!mapping) { - logger(LOG_INFO, - "OpenLI: attempt to remove session mapping for IP %s, but the mapping doesn't exist?", - sockaddr_to_string( - (struct sockaddr *)&(sess->sessionips[i].assignedip), - ipstr, 128)); - errs ++; continue; } @@ -2003,7 +2003,8 @@ static inline int report_silent_logoffs(collector_sync_t *sync, } if (remove_session_ip(prev->session[i], &(prev->ip)) == 1) { - free_single_session(prev->owner[i], prev->session[i]); + HASH_DELETE(hh, prev->owner[i]->sessions, prev->session[i]); + free_single_session(prev->session[i]); } } HASH_DELETE(hh, sync->activeips, prev); @@ -2055,6 +2056,7 @@ static int add_ip_to_session_mapping(collector_sync_t *sync, newmap->session[0] = sess; newmap->owner[0] = iuser; + HASH_ADD_KEYPTR(hh, sync->activeips, &newmap->ip, sizeof(internetaccess_ip_t), newmap); @@ -2067,7 +2069,7 @@ static inline internet_user_t *lookup_userid(collector_sync_t *sync, internet_user_t *iuser; - HASH_FIND(hh, sync->allusers, userid->idstr, userid->idlength, iuser); + iuser = lookup_user_by_identity(sync->allusers, userid); if (iuser == NULL) { iuser = (internet_user_t *)malloc(sizeof(internet_user_t)); @@ -2075,12 +2077,10 @@ static inline internet_user_t *lookup_userid(collector_sync_t *sync, logger(LOG_INFO, "OpenLI: unable to allocate memory for new Internet user"); return NULL; } - iuser->userid = userid->idstr; - userid->idstr = NULL; + iuser->userid = NULL; iuser->sessions = NULL; - HASH_ADD_KEYPTR(hh, sync->allusers, iuser->userid, - userid->idlength, iuser); + add_userid_to_allusers_map(&(sync->allusers), iuser, userid); } return iuser; } @@ -2174,7 +2174,7 @@ static int update_user_sessions(collector_sync_t *sync, libtrace_packet_t *pkt, access_plugin_t *p = NULL; user_identity_t *identities = NULL; internet_user_t *iuser; - access_session_t *sess; + access_session_t *sess = NULL; access_action_t accessaction; session_state_t oldstate, newstate; user_intercept_list_t *userint; @@ -2183,6 +2183,9 @@ static int update_user_sessions(collector_sync_t *sync, libtrace_packet_t *pkt, void *parseddata = NULL; int i, ret, useridcnt = 0; + oldstate = SESSION_STATE_NEW; + newstate = SESSION_STATE_NEW; + if (accesstype == ACCESS_RADIUS) { p = sync->radiusplugin; } else if (accesstype == ACCESS_GTP) { @@ -2224,16 +2227,16 @@ static int update_user_sessions(collector_sync_t *sync, libtrace_packet_t *pkt, ret = -1; break; } - - sess = p->update_session_state(p, parseddata, identities[i].plugindata, - &(iuser->sessions), &oldstate, &newstate, &accessaction); + sess = p->update_session_state(p, parseddata, + identities[i].plugindata, &(iuser->sessions), &oldstate, + &newstate, &accessaction); if (!sess) { /* Unable to assign packet to a session, just quietly ignore it */ continue; } HASH_FIND(hh, sync->userintercepts, iuser->userid, - identities[i].idlength, userint); + strlen(iuser->userid), userint); if (oldstate != newstate) { if (newstate == SESSION_STATE_ACTIVE) { @@ -2289,12 +2292,13 @@ static int update_user_sessions(collector_sync_t *sync, libtrace_packet_t *pkt, } } } - - if (oldstate != newstate && newstate == SESSION_STATE_OVER) { - free_single_session(iuser, sess); + if (sess && oldstate != newstate && newstate == SESSION_STATE_OVER) { + HASH_DELETE(hh, iuser->sessions, sess); + free_single_session(sess); } } + endupdate: if (parseddata) { p->destroy_parsed_data(p, parseddata); diff --git a/src/collector/collector_sync_voip.c b/src/collector/collector_sync_voip.c index 1a9f3c3d..26e6b48a 100644 --- a/src/collector/collector_sync_voip.c +++ b/src/collector/collector_sync_voip.c @@ -46,7 +46,6 @@ #include "util.h" #include "ipmmiri.h" - collector_sync_voip_t *init_voip_sync_data(collector_global_t *glob) { int i; @@ -869,7 +868,7 @@ static int process_sip_200ok(collector_sync_voip_t *sync, static inline void create_sip_ipiri(collector_sync_voip_t *sync, voipintercept_t *vint, openli_export_recv_t *irimsg, etsili_iri_type_t iritype, int64_t cin, openli_location_t *loc, - int loc_count) { + int loc_count, libtrace_packet_t **pkts, int pkt_cnt) { openli_export_recv_t *copy; @@ -886,12 +885,23 @@ static inline void create_sip_ipiri(collector_sync_voip_t *sync, return; } - /* TODO add ability to include SIP packets in pcapdisk output */ - if (vint->common.targetagency == NULL || strcmp(vint->common.targetagency, - "pcapdisk") == 0) { + if (vint->common.targetagency == NULL || + strcmp(vint->common.targetagency, "pcapdisk") == 0) { + int i; + if (pkts == NULL) { + return; + } + for (i = 0; i < pkt_cnt; i++) { + if (pkts[i] == NULL) { + continue; + } + copy = create_rawip_iri_job(vint->common.liid, vint->common.destid, + pkts[i]); + publish_openli_msg(sync->zmq_pubsocks[vint->common.seqtrackerid], + copy); + } return; } - /* TODO consider recycling IRI messages like we do with IPCCs */ /* Wrap this packet up in an IRI and forward it on to the exporter. @@ -908,7 +918,6 @@ static inline void create_sip_ipiri(collector_sync_voip_t *sync, copy->data.ipmmiri.content = malloc(copy->data.ipmmiri.contentlen); memcpy(copy->data.ipmmiri.content, irimsg->data.ipmmiri.content, irimsg->data.ipmmiri.contentlen); - copy_location_into_ipmmiri_job(copy, loc, loc_count); pthread_mutex_lock(sync->glob->stats_mutex); @@ -919,16 +928,18 @@ static inline void create_sip_ipiri(collector_sync_voip_t *sync, static int process_sip_register_followup(collector_sync_voip_t *sync, voipintercept_t *vint, sipregister_t *sipreg, - openli_export_recv_t *irimsg) { + openli_export_recv_t *irimsg, libtrace_packet_t **pkts, + int pkt_cnt) { create_sip_ipiri(sync, vint, irimsg, ETSILI_IRI_REPORT, sipreg->cin, - NULL, 0); + NULL, 0, pkts, pkt_cnt); return 1; } static int process_sip_other(collector_sync_voip_t *sync, char *callid, - sip_sdp_identifier_t *sdpo, openli_export_recv_t *irimsg) { + sip_sdp_identifier_t *sdpo, openli_export_recv_t *irimsg, + libtrace_packet_t **pkts, int pkt_cnt) { voipintercept_t *vint, *tmp; voipcinmap_t *findcin; @@ -964,7 +975,7 @@ static int process_sip_other(collector_sync_voip_t *sync, char *callid, strlen(callid), findreg); if (findreg) { exportcount += process_sip_register_followup(sync, vint, - findreg, irimsg); + findreg, irimsg, pkts, pkt_cnt); } continue; } @@ -1056,7 +1067,7 @@ static int process_sip_other(collector_sync_voip_t *sync, char *callid, /* Wrap this packet up in an IRI and forward it on to the exporter */ create_sip_ipiri(sync, vint, irimsg, iritype, vshared->cin, locptr, - loc_cnt); + loc_cnt, pkts, pkt_cnt); exportcount += 1; } if (locptr) { @@ -1070,7 +1081,7 @@ static int process_sip_other(collector_sync_voip_t *sync, char *callid, } static int process_sip_register(collector_sync_voip_t *sync, char *callid, - openli_export_recv_t *irimsg) { + openli_export_recv_t *irimsg, libtrace_packet_t **pkts, int pkt_cnt) { openli_sip_identity_t *matched = NULL; voipintercept_t *vint, *tmp; @@ -1117,7 +1128,7 @@ static int process_sip_register(collector_sync_voip_t *sync, char *callid, continue; } create_sip_ipiri(sync, vint, irimsg, ETSILI_IRI_REPORT, sipreg->cin, - locptr, loc_cnt); + locptr, loc_cnt, pkts, pkt_cnt); exportcount += 1; } @@ -1130,7 +1141,8 @@ static int process_sip_register(collector_sync_voip_t *sync, char *callid, } static int process_sip_invite(collector_sync_voip_t *sync, char *callid, - sip_sdp_identifier_t *sdpo, openli_export_recv_t *irimsg) { + sip_sdp_identifier_t *sdpo, openli_export_recv_t *irimsg, + libtrace_packet_t **pkts, int pkt_cnt) { voipintercept_t *vint, *tmp; voipcinmap_t *findcin; @@ -1295,7 +1307,7 @@ static int process_sip_invite(collector_sync_voip_t *sync, char *callid, thisrtp->invitecseq = get_sip_cseq(sync->sipparser); create_sip_ipiri(sync, vint, irimsg, iritype, vshared->cin, locptr, - loc_cnt); + loc_cnt, pkts, pkt_cnt); exportcount += 1; } @@ -1312,7 +1324,7 @@ static int process_sip_invite(collector_sync_voip_t *sync, char *callid, } static int update_sip_state(collector_sync_voip_t *sync, - libtrace_packet_t *pkt, openli_export_recv_t *irimsg) { + libtrace_packet_t **pkts, int pkt_cnt, openli_export_recv_t *irimsg) { char *callid, *sessid, *sessversion, *sessaddr, *sessuser; sip_sdp_identifier_t sdpo; @@ -1380,7 +1392,8 @@ static int update_sip_state(collector_sync_voip_t *sync, ret = 0; if (sip_is_invite(sync->sipparser)) { - if ((ret = process_sip_invite(sync, callid, &sdpo, irimsg)) < 0) { + if ((ret = process_sip_invite(sync, callid, &sdpo, irimsg, pkts, + pkt_cnt)) < 0) { iserr = 1; if (sync->log_bad_sip) { logger(LOG_INFO, "OpenLI: error while processing SIP invite"); @@ -1388,7 +1401,8 @@ static int update_sip_state(collector_sync_voip_t *sync, goto sipgiveup; } } else if (sip_is_register(sync->sipparser)) { - if ((ret = process_sip_register(sync, callid, irimsg)) < 0) { + if ((ret = process_sip_register(sync, callid, irimsg, pkts, + pkt_cnt)) < 0) { iserr = 1; if (sync->log_bad_sip) { logger(LOG_INFO, "OpenLI: error while processing SIP register"); @@ -1397,7 +1411,8 @@ static int update_sip_state(collector_sync_voip_t *sync, } } else if (lookup_sip_callid(sync, callid) != 0) { /* SIP packet matches a "known" call of interest */ - if ((ret = process_sip_other(sync, callid, &sdpo, irimsg)) < 0) { + if ((ret = process_sip_other(sync, callid, &sdpo, irimsg, pkts, + pkt_cnt)) < 0) { iserr = 1; if (sync->log_bad_sip) { logger(LOG_INFO, "OpenLI: error while processing non-invite SIP"); @@ -1859,50 +1874,110 @@ static inline void get_ip_addresses(libtrace_packet_t *pkt, } +static inline void handle_bad_sip_update(collector_sync_voip_t *sync, + libtrace_packet_t **packets, int pkt_cnt, uint8_t during) { -static void examine_sip_update(collector_sync_voip_t *sync, - libtrace_packet_t *recvdpkt) { - - int ret, doonce; - libtrace_packet_t *pktref; - openli_export_recv_t baseirimsg; - - memset(&baseirimsg, 0, sizeof(openli_export_recv_t)); - - ret = add_sip_packet_to_parser(&(sync->sipparser), recvdpkt, - sync->log_bad_sip); + int i; - if (ret == SIP_ACTION_ERROR) { - if (sync->log_bad_sip) { + if (sync->log_bad_sip) { + if (during == SIP_PROCESSING_PARSING) { + logger(LOG_INFO, + "OpenLI: sync thread parsed an invalid SIP packet?"); + } else if (during == SIP_PROCESSING_UPDATING_STATE) { + logger(LOG_INFO, + "OpenLI: error while updating SIP state in collector."); + } else if (during == SIP_PROCESSING_EXTRACTING_IPS) { + logger(LOG_INFO, + "OpenLI: error while extracting IP addresses from SIP packet"); + } else if (during == SIP_PROCESSING_ADD_PARSER) { logger(LOG_INFO, "OpenLI: sync thread received an invalid SIP packet?"); + } else { logger(LOG_INFO, - "OpenLI: will not log any further invalid SIP instances."); - sync->log_bad_sip = 0; + "OpenLI: unexpected error when processing SIP packet"); + } - if (sync->sipdebugfile) { - if (!sync->sipdebugout) { - sync->sipdebugout = open_debug_output(sync->sipdebugfile, - "invalid"); - } - if (sync->sipdebugout) { - trace_write_packet(sync->sipdebugout, recvdpkt); + logger(LOG_INFO, + "OpenLI: will not log any further invalid SIP instances."); + sync->log_bad_sip = 0; + } + pthread_mutex_lock(sync->glob->stats_mutex); + sync->glob->stats->bad_sip_packets ++; + pthread_mutex_unlock(sync->glob->stats_mutex); + + if (sync->sipdebugfile && packets) { + if (!sync->sipdebugout) { + sync->sipdebugout = open_debug_output( + sync->sipdebugfile, "invalid"); + } + if (sync->sipdebugout) { + for (i = 0; i < pkt_cnt; i++) { + if (packets[i] == NULL) { + continue; + } + trace_write_packet(sync->sipdebugout, packets[i]); } } + } +} + +static void sip_update_fast_path(collector_sync_voip_t *sync, + libtrace_packet_t *recvdpkt) { + + int ret; + openli_export_recv_t baseirimsg; + + /* The provided packet contains an entire SIP message, so we + * don't need to worry about segmentation or multiple + * messages within the same packet. + */ + + memset(&baseirimsg, 0, sizeof(openli_export_recv_t)); + + baseirimsg.type = OPENLI_EXPORT_IPMMIRI; + baseirimsg.data.ipmmiri.ipmmiri_style = OPENLI_IPMMIRI_SIP; + baseirimsg.ts = trace_get_timeval(recvdpkt); + + if (extract_ip_addresses(recvdpkt, baseirimsg.data.ipmmiri.ipsrc, + baseirimsg.data.ipmmiri.ipdest, + &(baseirimsg.data.ipmmiri.ipfamily)) != 0) { + handle_bad_sip_update(sync, &recvdpkt, 1, + SIP_PROCESSING_EXTRACTING_IPS); + trace_destroy_packet(recvdpkt); return; - } else if (ret == SIP_ACTION_USE_PACKET) { - pktref = recvdpkt; - doonce = 1; - } else if (ret == SIP_ACTION_REASSEMBLE_TCP) { - pktref = NULL; - doonce = 0; - } else if (ret == SIP_ACTION_REASSEMBLE_IPFRAG) { - doonce = 1; - pktref = NULL; - } else { + } + + ret = parse_next_sip_message(sync->sipparser, NULL, NULL); + if (ret == 0) { + trace_destroy_packet(recvdpkt); + return; + } + if (ret < 0) { + handle_bad_sip_update(sync, &recvdpkt, 1, SIP_PROCESSING_PARSING); + trace_destroy_packet(recvdpkt); return; + + } + baseirimsg.data.ipmmiri.content = get_sip_contents(sync->sipparser, + &(baseirimsg.data.ipmmiri.contentlen)); + + if (update_sip_state(sync, &recvdpkt, 1, &baseirimsg) < 0) { + handle_bad_sip_update(sync, &recvdpkt, 1, + SIP_PROCESSING_UPDATING_STATE); } +} + +static void sip_update_slow_path(collector_sync_voip_t *sync, + libtrace_packet_t *recvdpkt, uint8_t doonce) { + + int ret, i; + openli_export_recv_t baseirimsg; + libtrace_packet_t **packets = NULL; + int pkt_cnt = 0; + + memset(&baseirimsg, 0, sizeof(openli_export_recv_t)); + baseirimsg.type = OPENLI_EXPORT_IPMMIRI; baseirimsg.data.ipmmiri.ipmmiri_style = OPENLI_IPMMIRI_SIP; baseirimsg.ts = trace_get_timeval(recvdpkt); @@ -1910,71 +1985,76 @@ static void examine_sip_update(collector_sync_voip_t *sync, if (extract_ip_addresses(recvdpkt, baseirimsg.data.ipmmiri.ipsrc, baseirimsg.data.ipmmiri.ipdest, &(baseirimsg.data.ipmmiri.ipfamily)) != 0) { - if (sync->log_bad_sip) { - logger(LOG_INFO, - "OpenLI: error while extracting IP addresses from SIP packet"); - logger(LOG_INFO, - "OpenLI: will not log any further invalid SIP instances."); - sync->log_bad_sip = 0; - } - ret = SIP_ACTION_IGNORE; + handle_bad_sip_update(sync, &recvdpkt, 1, + SIP_PROCESSING_EXTRACTING_IPS); + trace_destroy_packet(recvdpkt); + return; } - /* reassembled TCP streams can contain multiple messages, so - * we need to keep trying until we have no new usable messages. */ do { - ret = parse_next_sip_message(sync->sipparser, pktref); - if (ret == 0) { - break; - } - - if (ret < 0) { - if (sync->log_bad_sip) { - logger(LOG_INFO, - "OpenLI: sync thread parsed an invalid SIP packet?"); - logger(LOG_INFO, - "OpenLI: will not log any further invalid SIP instances."); - sync->log_bad_sip = 0; - } - pthread_mutex_lock(sync->glob->stats_mutex); - sync->glob->stats->bad_sip_packets ++; - pthread_mutex_unlock(sync->glob->stats_mutex); - - if (sync->sipdebugfile && pktref) { - if (!sync->sipdebugout) { - sync->sipdebugout = open_debug_output( - sync->sipdebugfile, "invalid"); - } - if (sync->sipdebugout) { - trace_write_packet(sync->sipdebugout, pktref); + if (packets != NULL) { + for (i = 0; i < pkt_cnt; i++) { + if (packets[i]) { + trace_destroy_packet(packets[i]); } } + free(packets); + pkt_cnt = 0; + packets = NULL; } + ret = parse_next_sip_message(sync->sipparser, &packets, &pkt_cnt); + if (ret == 0) { + return; + } + if (ret < 0) { + handle_bad_sip_update(sync, packets, pkt_cnt, + SIP_PROCESSING_PARSING); + continue; + } baseirimsg.data.ipmmiri.content = get_sip_contents(sync->sipparser, &(baseirimsg.data.ipmmiri.contentlen)); + if (update_sip_state(sync, packets, pkt_cnt, &baseirimsg) < 0) { + handle_bad_sip_update(sync, packets, pkt_cnt, + SIP_PROCESSING_UPDATING_STATE); + continue; + } - if (ret > 0 && update_sip_state(sync, pktref, &baseirimsg) < 0) { - if (sync->log_bad_sip) { - logger(LOG_INFO, - "OpenLI: error while updating SIP state in collector."); - logger(LOG_INFO, - "OpenLI: will not log any further invalid SIP instances."); - sync->log_bad_sip = 0; - } - if (sync->sipdebugfile && pktref) { - if (!sync->sipdebugupdate) { - sync->sipdebugupdate = open_debug_output( - sync->sipdebugfile, - "update"); - } - if (sync->sipdebugupdate) { - trace_write_packet(sync->sipdebugupdate, pktref); - } + } while (!doonce); + + if (packets) { + for (i = 0; i < pkt_cnt; i++) { + if (packets[i]) { + trace_destroy_packet(packets[i]); } } - } while (!doonce); + free(packets); + } + +} +static void examine_sip_update(collector_sync_voip_t *sync, + libtrace_packet_t *recvdpkt) { + + int ret; + + ret = add_sip_packet_to_parser(&(sync->sipparser), recvdpkt, + sync->log_bad_sip); + + if (ret == SIP_ACTION_ERROR) { + handle_bad_sip_update(sync, &recvdpkt, 1, + SIP_PROCESSING_ADD_PARSER); + } else if (ret == SIP_ACTION_USE_PACKET) { + sip_update_fast_path(sync, recvdpkt); + } else if (ret == SIP_ACTION_REASSEMBLE_TCP) { + sip_update_slow_path(sync, recvdpkt, 0); + } else if (ret == SIP_ACTION_REASSEMBLE_IPFRAG) { + sip_update_slow_path(sync, recvdpkt, 1); + } + + if (recvdpkt) { + trace_destroy_packet(recvdpkt); + } } static inline int process_colthread_message(collector_sync_voip_t *sync) { @@ -2010,7 +2090,6 @@ static inline int process_colthread_message(collector_sync_voip_t *sync) { if (recvd.type == OPENLI_UPDATE_SIP) { examine_sip_update(sync, recvd.data.pkt); - trace_destroy_packet(recvd.data.pkt); } } while (rc > 0); diff --git a/src/collector/email_ingest_service.c b/src/collector/email_ingest_service.c index 47d36602..74cc138b 100644 --- a/src/collector/email_ingest_service.c +++ b/src/collector/email_ingest_service.c @@ -168,7 +168,7 @@ static MHD_RESULT iterate_post (void *coninfo_cls, enum MHD_ValueKind kind, ptr ++; } - if (*ptr == '\0' || ptr - data >= size) { + if (*ptr == '\0' || (size_t)(ptr - data) >= size) { free(con_info->thismsg->content); con_info->thismsg->content = NULL; } diff --git a/src/collector/email_worker.c b/src/collector/email_worker.c index c8753666..b0f3dec6 100644 --- a/src/collector/email_worker.c +++ b/src/collector/email_worker.c @@ -526,7 +526,7 @@ void clear_email_sender(emailsession_t *sess) { static void free_email_session(openli_email_worker_t *state, emailsession_t *sess) { - int i; + uint32_t i; Word_t rc; if (!sess) { @@ -827,7 +827,7 @@ static void remove_email_intercept(openli_email_worker_t *state, static int update_default_email_compression(openli_email_worker_t *state, provisioner_msg_t *provmsg) { - uint8_t newval; + uint8_t newval = OPENLI_EMAILINT_DELIVER_COMPRESSED_NOT_SET; if (decode_default_email_compression_announcement(provmsg->msgbody, provmsg->msglen, &newval) < 0) { @@ -1000,8 +1000,7 @@ static int add_email_target(openli_email_worker_t *state, EVP_MD_CTX *ctx; const EVP_MD *md; unsigned char shaspace[EVP_MAX_MD_SIZE]; - unsigned int sha_len; - int i; + unsigned int sha_len, i; tgt = calloc(1, sizeof(email_target_t)); if (decode_email_target_announcement(provmsg->msgbody, provmsg->msglen, @@ -1180,7 +1179,8 @@ static int find_and_update_active_session(openli_email_worker_t *state, char sesskey[256]; emailsession_t *sess; - int r = 0, i; + int r = 0; + size_t i; if (cap->session_id == NULL) { logger(LOG_INFO, diff --git a/src/collector/email_worker.h b/src/collector/email_worker.h index 6c428791..6b6c82d3 100644 --- a/src/collector/email_worker.h +++ b/src/collector/email_worker.h @@ -122,7 +122,7 @@ typedef struct openli_email_captured { char *datasource; uint8_t direction; - uint64_t timestamp; + time_t timestamp; uint32_t mail_id; uint32_t part_id; uint32_t msg_length; @@ -237,13 +237,13 @@ int generate_email_logoff_iri_for_user(openli_email_worker_t *state, /* Defined in emailcc.c */ int generate_email_cc_from_smtp_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, const char *participant, uint8_t dir, + time_t timestamp, const char *participant, uint8_t dir, int command_index); int generate_email_cc_from_imap_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, uint8_t dir, uint8_t deflated); + time_t timestamp, uint8_t dir, uint8_t deflated); int generate_email_cc_from_pop3_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, uint8_t dir); + time_t timestamp, uint8_t dir); #endif // vim: set sw=4 tabstop=4 softtabstop=4 expandtab : diff --git a/src/collector/emailcc.c b/src/collector/emailcc.c index 7ed1d6bc..77a851ab 100644 --- a/src/collector/emailcc.c +++ b/src/collector/emailcc.c @@ -33,7 +33,7 @@ #include "etsili_core.h" static openli_export_recv_t *create_emailcc_job(char *liid, - emailsession_t *sess, uint32_t destid, uint64_t timestamp, + emailsession_t *sess, uint32_t destid, time_t timestamp, uint8_t *content, int content_len, uint8_t format, uint8_t dir) { openli_export_recv_t *msg = NULL; @@ -62,7 +62,7 @@ static openli_export_recv_t *create_emailcc_job(char *liid, static void create_emailccs_for_intercept_list(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, uint8_t format, email_intercept_ref_t *intlist, - uint64_t timestamp, uint8_t dir, const char *key, uint8_t deflated) { + time_t timestamp, uint8_t dir, const char *key, uint8_t deflated) { openli_export_recv_t *ccjob = NULL; email_intercept_ref_t *ref, *tmp; @@ -150,7 +150,7 @@ static void create_emailccs_for_intercept_list(openli_email_worker_t *state, int generate_email_cc_from_smtp_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, const char *address, uint8_t dir, + time_t timestamp, const char *address, uint8_t dir, int command_index) { email_address_set_t *active_addr = NULL; @@ -198,7 +198,7 @@ int generate_email_cc_from_smtp_payload(openli_email_worker_t *state, int generate_email_cc_from_pop3_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, uint8_t etsidir) { + time_t timestamp, uint8_t etsidir) { email_address_set_t *active = NULL; email_participant_t *recip, *tmp; @@ -224,7 +224,7 @@ int generate_email_cc_from_pop3_payload(openli_email_worker_t *state, int generate_email_cc_from_imap_payload(openli_email_worker_t *state, emailsession_t *sess, uint8_t *content, int content_len, - uint64_t timestamp, uint8_t etsidir, uint8_t deflated) { + time_t timestamp, uint8_t etsidir, uint8_t deflated) { email_address_set_t *active = NULL; email_participant_t *recip, *tmp; diff --git a/src/collector/emailiri.c b/src/collector/emailiri.c index 3af003f0..e50cd42b 100644 --- a/src/collector/emailiri.c +++ b/src/collector/emailiri.c @@ -38,7 +38,7 @@ void free_email_iri_content(etsili_email_iri_content_t *content) { - int i; + size_t i; if (content->recipients) { for (i = 0; i < content->recipient_count; i++) { @@ -83,7 +83,7 @@ static void add_recipients(emailsession_t *sess, static openli_export_recv_t *create_emailiri_job(char *liid, emailsession_t *sess, uint8_t iritype, uint8_t emailev, - uint8_t status, uint32_t destid, uint64_t timestamp, + uint8_t status, uint32_t destid, time_t timestamp, const char **tgtaddrs, int tgtaddr_count) { openli_export_recv_t *msg = NULL; @@ -162,7 +162,7 @@ static openli_export_recv_t *create_emailiri_job(char *liid, static void create_emailiris_for_intercept_list(openli_email_worker_t *state, emailsession_t *sess, uint8_t iri_type, uint8_t email_ev, - uint8_t status, email_intercept_ref_t *intlist, uint64_t ts, + uint8_t status, email_intercept_ref_t *intlist, time_t ts, const char *key, const char *tgtaddr, uint8_t full_recip_list) { openli_export_recv_t *irijob = NULL; @@ -281,7 +281,7 @@ static void create_emailiris_for_intercept_list(openli_email_worker_t *state, static inline int generate_iris_for_participants(openli_email_worker_t *state, emailsession_t *sess, uint8_t email_ev, uint8_t iri_type, - uint8_t status, uint64_t timestamp) { + uint8_t status, time_t timestamp) { email_address_set_t *active_addr = NULL; email_target_set_t *active_tgt = NULL; @@ -378,7 +378,7 @@ static inline int generate_iris_for_participants(openli_email_worker_t *state, static int generate_iris_for_mailbox(openli_email_worker_t *state, emailsession_t *sess, uint8_t email_ev, uint8_t iri_type, - uint8_t status, uint64_t timestamp, const char *mailbox) { + uint8_t status, time_t timestamp, const char *mailbox) { email_address_set_t *active_addr = NULL; email_target_set_t *active_tgt = NULL; @@ -583,7 +583,7 @@ int generate_email_login_failure_iri(openli_email_worker_t *state, static inline void emailiri_free_recipients( etsili_email_recipients_t *recipients) { - int i; + size_t i; for (i = 0; i < recipients->count; i++) { free(recipients->addresses[i]); } @@ -594,7 +594,7 @@ static inline void emailiri_populate_recipients( etsili_email_recipients_t *recipients, uint32_t count, char **reciplist) { - int i; + size_t i; recipients->count = count; recipients->addresses = calloc(count, sizeof(char *)); diff --git a/src/collector/emailprotocols/imap.c b/src/collector/emailprotocols/imap.c index 3cd5f412..e6dfc77e 100644 --- a/src/collector/emailprotocols/imap.c +++ b/src/collector/emailprotocols/imap.c @@ -97,14 +97,14 @@ struct compress_state { typedef struct imapsession { uint8_t *contbuffer; - int contbufsize; - int contbufused; - int contbufread; + size_t contbufsize; + size_t contbufused; + size_t contbufread; uint8_t *deflatebuffer; - int deflatebufsize; - int deflatebufused; - int deflatebufread; + size_t deflatebufsize; + size_t deflatebufused; + size_t deflatebufread; imap_cc_index_t *deflate_ccs; int deflate_ccs_size; @@ -117,8 +117,8 @@ typedef struct imapsession { char *mailbox; char *mail_sender; - int reply_start; - int next_comm_start; + size_t reply_start; + size_t next_comm_start; uint8_t next_command_type; char *next_comm_tag; char *next_command_name; @@ -128,7 +128,7 @@ typedef struct imapsession { int idle_command_index; int auth_command_index; int auth_token_index; - int auth_read_from; + size_t auth_read_from; openli_email_auth_type_t auth_type; struct compress_state decompress_server; @@ -327,7 +327,7 @@ static int complete_imap_authentication(openli_email_worker_t *state, static int generate_ccs_from_imap_command(openli_email_worker_t *state, emailsession_t *sess, imap_session_t *imapsess, - imap_command_t *comm, uint64_t timestamp) { + imap_command_t *comm, time_t timestamp) { int i, len; uint8_t dir; @@ -1212,8 +1212,8 @@ static int parse_id_command(emailsession_t *sess, imap_session_t *imapsess) { return ret; } -static int find_next_crlf(imap_session_t *sess, int start_index) { - int rem, regres; +static int find_next_crlf(imap_session_t *sess, size_t start_index) { + int regres; uint8_t *found = NULL; uint8_t *openparent = NULL; uint8_t *closeparent = NULL; @@ -1221,13 +1221,18 @@ static int find_next_crlf(imap_session_t *sess, int start_index) { regmatch_t matches[1]; regex_t regex; int nests = 0; + size_t rem; if (regcomp(®ex, "\\{[0-9]+\\}", REG_EXTENDED) != 0) { logger(LOG_INFO, "OpenLI: failed to compile regex pattern for matching curly braces in IMAP content?"); return -1; } - rem = sess->contbufused - start_index; + if (sess->contbufused > start_index) { + rem = sess->contbufused - start_index; + } else { + return 0; + } sess->contbuffer[sess->contbufused] = '\0'; while (1) { @@ -1394,7 +1399,7 @@ static int find_command_end(emailsession_t *sess, imap_session_t *imapsess) { } static int find_reply_end(openli_email_worker_t *state, - emailsession_t *sess, imap_session_t *imapsess, uint64_t timestamp) { + emailsession_t *sess, imap_session_t *imapsess, time_t timestamp) { int r; imap_command_t *comm; @@ -1955,7 +1960,7 @@ static int find_next_imap_message(openli_email_worker_t *state, } static int process_next_imap_state(openli_email_worker_t *state, - emailsession_t *sess, imap_session_t *imapsess, uint64_t timestamp) { + emailsession_t *sess, imap_session_t *imapsess, time_t timestamp) { int r; diff --git a/src/collector/emailprotocols/pop3.c b/src/collector/emailprotocols/pop3.c index 04d93614..b36a0184 100644 --- a/src/collector/emailprotocols/pop3.c +++ b/src/collector/emailprotocols/pop3.c @@ -82,19 +82,19 @@ enum { typedef struct pop3session { uint8_t *contbuffer; - int contbufsize; - int contbufused; - int contbufread; + size_t contbufsize; + size_t contbufused; + size_t contbufread; int auth_state; int last_command_type; int server_indicator; - int command_start; - int command_end; - int reply_start; + size_t command_start; + size_t command_end; + size_t reply_start; - int auth_read_from; + size_t auth_read_from; openli_email_auth_type_t auth_type; char *mailbox; @@ -654,7 +654,7 @@ static int extract_pop3_email_sender(openli_email_worker_t *state, } static int handle_multi_reply_state(openli_email_worker_t *state, - emailsession_t *sess, pop3_session_t *pop3sess, uint64_t timestamp) { + emailsession_t *sess, pop3_session_t *pop3sess, time_t timestamp) { int r; @@ -763,7 +763,7 @@ static int handle_client_command(emailsession_t *sess, static int handle_server_reply_state(openli_email_worker_t *state, - emailsession_t *sess, pop3_session_t *pop3sess, uint64_t timestamp) { + emailsession_t *sess, pop3_session_t *pop3sess, time_t timestamp) { int r = 1; @@ -858,7 +858,7 @@ static int handle_server_reply_state(openli_email_worker_t *state, } static int process_next_pop3_line(openli_email_worker_t *state, - emailsession_t *sess, pop3_session_t *pop3sess, uint64_t timestamp) { + emailsession_t *sess, pop3_session_t *pop3sess, time_t timestamp) { int r; diff --git a/src/collector/emailprotocols/smtp.c b/src/collector/emailprotocols/smtp.c index cf54b7a3..b0c5bdaa 100644 --- a/src/collector/emailprotocols/smtp.c +++ b/src/collector/emailprotocols/smtp.c @@ -59,7 +59,7 @@ enum { typedef struct smtp_comm { uint8_t command_type; - uint64_t timestamp; + time_t timestamp; uint16_t reply_code; int command_index; @@ -79,18 +79,18 @@ typedef struct smtp_cc_list { typedef struct smtp_participant { smtp_cc_list_t ccs; uint8_t active; - uint64_t last_mail_from; + time_t last_mail_from; } smtp_participant_t; typedef struct smtpsession { char *messageid; uint8_t *contbuffer; - int contbufsize; - int contbufused; - int contbufread; - int command_start; - int reply_start; + size_t contbufsize; + size_t contbufused; + size_t contbufread; + size_t command_start; + size_t reply_start; uint16_t reply_code; int next_command_index; @@ -327,7 +327,7 @@ static int add_new_smtp_command(smtp_cc_list_t *ccs, static int add_new_smtp_reply(smtp_cc_list_t *ccs, int reply_start, int reply_end, uint16_t reply_code, - uint64_t timestamp) { + time_t timestamp) { int ind = ccs->curr_command; smtp_command_t *cmd = &(ccs->commands[ind]); @@ -730,7 +730,7 @@ static int find_ehlo_start(emailsession_t *mailsess, smtp_session_t *sess) { } static int save_latest_command(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp, + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp, uint8_t command_type, uint8_t publish_now, uint8_t sender_only) { PWord_t pval; @@ -833,7 +833,7 @@ static int process_auth_message(smtp_session_t *smtpsess) { } static int other_command_reply(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { /* Emit CCs for the command, but only if the sender is the intercept * target. We probably don't care about weird SMTP behaviour if the @@ -848,7 +848,7 @@ static int other_command_reply(openli_email_worker_t *state, } static int rcpt_to_reply(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { int r, i; PWord_t pval; @@ -905,7 +905,7 @@ static int rcpt_to_reply(openli_email_worker_t *state, } static void activate_latest_sender(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp, + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp, smtp_participant_t **sender) { PWord_t pval; @@ -1051,7 +1051,7 @@ static int parse_mail_content(openli_email_worker_t *state, } static void data_content_over(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { PWord_t pval; char index[1024]; @@ -1125,7 +1125,7 @@ static void data_content_over(openli_email_worker_t *state, } static int set_sender_using_mail_from(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp, + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp, smtp_participant_t **sender) { if (extract_smtp_participant(sess, smtpsess, @@ -1139,7 +1139,7 @@ static int set_sender_using_mail_from(openli_email_worker_t *state, } static int mail_from_reply(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { int i; smtp_participant_t *sender = NULL; @@ -1326,7 +1326,7 @@ static int extract_sender_from_auth_creds(emailsession_t *sess, } static int authenticate_success(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { int r, i; smtp_participant_t *sender = NULL; @@ -1375,7 +1375,7 @@ static int authenticate_success(openli_email_worker_t *state, } static int authenticate_failure(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { char *sendername = NULL; int r, i; @@ -1398,7 +1398,7 @@ static int authenticate_failure(openli_email_worker_t *state, static int process_next_smtp_state(openli_email_worker_t *state, - emailsession_t *sess, smtp_session_t *smtpsess, uint64_t timestamp) { + emailsession_t *sess, smtp_session_t *smtpsess, time_t timestamp) { int r; /* TODO consider adding state parsing for AUTH, STARTTLS, VRFY, EXPN diff --git a/src/collector/encoder_worker.c b/src/collector/encoder_worker.c index 0498cbd3..85151bbc 100644 --- a/src/collector/encoder_worker.c +++ b/src/collector/encoder_worker.c @@ -860,6 +860,8 @@ static int process_job(openli_encoder_t *enc, void *socket) { encode_rawip(enc, &job, &(result[batch]), OPENLI_PROTO_RAWIP_SYNC); } else if (job.origreq->type == OPENLI_EXPORT_RAW_CC) { encode_rawip(enc, &job, &(result[batch]), OPENLI_PROTO_RAWIP_CC); + } else if (job.origreq->type == OPENLI_EXPORT_RAW_IRI) { + encode_rawip(enc, &job, &(result[batch]), OPENLI_PROTO_RAWIP_IRI); } else { if ((x = encode_etsi(enc, &job, &(result[batch]))) <= 0) { diff --git a/src/collector/internetaccess.c b/src/collector/internetaccess.c index d2dda63e..a2643361 100644 --- a/src/collector/internetaccess.c +++ b/src/collector/internetaccess.c @@ -103,6 +103,73 @@ static inline char *fast_strdup(char *orig, int origlen) { return dup; } +static int generate_tagged_userid(user_identity_t *userid, char *taggedid, + int space) { + + char *ptr = taggedid; + memset(taggedid, 0, space); + + if (userid->method == USER_IDENT_GTP_MSISDN) { + memcpy(ptr, "msisdn:", strlen("msisdn:")); + ptr += strlen("msisdn:"); + } else if (userid->method == USER_IDENT_GTP_IMSI) { + memcpy(ptr, "imsi:", strlen("imsi:")); + ptr += strlen("imsi:"); + } else if (userid->method == USER_IDENT_GTP_IMEI) { + memcpy(ptr, "imei:", strlen("imei:")); + ptr += strlen("imei:"); + } + + if ((ptr - taggedid) + userid->idlength + 1 > space) { + logger(LOG_INFO, + "OpenLI: user identity string is too long!"); + return -1; + } + + memcpy(ptr, userid->idstr, userid->idlength); + return userid->idlength + (ptr - taggedid); +} + +internet_user_t *lookup_user_by_identity(internet_user_t *allusers, + user_identity_t *userid) { + + char taggedid[2048]; + internet_user_t *found = NULL; + + if (generate_tagged_userid(userid, taggedid, 2048) < 0) { + return NULL; + } + HASH_FIND(hh, allusers, taggedid, strlen(taggedid), found); + return found; +} + +internet_user_t *lookup_user_by_intercept(internet_user_t *allusers, + ipintercept_t *ipint) { + + char taggedid[2048]; + internet_user_t *found = NULL; + + if (generate_ipint_userkey(ipint, taggedid, 2048) < 0) { + return NULL; + } + HASH_FIND(hh, allusers, taggedid, strlen(taggedid), found); + return found; +} + +int add_userid_to_allusers_map(internet_user_t **allusers, + internet_user_t *newuser, user_identity_t *userid) { + + char taggedid[2048]; + + if (generate_tagged_userid(userid, taggedid, 2048) < 0) { + return -1; + } + newuser->userid = strdup(taggedid); + HASH_ADD_KEYPTR(hh, *allusers, newuser->userid, strlen(newuser->userid), + newuser); + return 0; +} + access_session_t *create_access_session(access_plugin_t *p, char *sessid, int sessid_len) { access_session_t *newsess; @@ -191,15 +258,7 @@ void add_new_session_ip(access_session_t *sess, void *att_val, sess->sessipcount ++; } -int free_single_session(internet_user_t *user, access_session_t *sess) { - - if (user == NULL) { - logger(LOG_INFO, - "OpenLI: called free_single_session() for a NULL user!"); - return -1; - } - - HASH_DELETE(hh, user->sessions, sess); +int free_single_session(access_session_t *sess) { free_session(sess); return 0; } diff --git a/src/collector/internetaccess.h b/src/collector/internetaccess.h index 088c079e..d19da36a 100644 --- a/src/collector/internetaccess.h +++ b/src/collector/internetaccess.h @@ -72,6 +72,7 @@ typedef enum { USER_IDENT_RADIUS_CSID, USER_IDENT_GTP_MSISDN, USER_IDENT_GTP_IMSI, + USER_IDENT_GTP_IMEI, USER_IDENT_MAX } user_identity_method_t; @@ -187,7 +188,7 @@ access_plugin_t *init_access_plugin(uint8_t accessmethod); void destroy_access_plugin(access_plugin_t *p); void free_all_users(internet_user_t *users); -int free_single_session(internet_user_t *user, access_session_t *sess); +int free_single_session(access_session_t *sess); access_plugin_t *get_radius_access_plugin(void); access_plugin_t *get_gtp_access_plugin(void); @@ -200,5 +201,12 @@ int remove_session_ip(access_session_t *sess, internetaccess_ip_t *sessip); const char *accesstype_to_string(internet_access_method_t am); +internet_user_t *lookup_user_by_identity(internet_user_t *allusers, + user_identity_t *userid); +int add_userid_to_allusers_map(internet_user_t **allusers, + internet_user_t *newuser, user_identity_t *userid); +internet_user_t *lookup_user_by_intercept(internet_user_t *allusers, + ipintercept_t *ipint); + #endif // vim: set sw=4 tabstop=4 softtabstop=4 expandtab : diff --git a/src/collector/ipiri.c b/src/collector/ipiri.c index d3b416b4..de096bca 100644 --- a/src/collector/ipiri.c +++ b/src/collector/ipiri.c @@ -361,6 +361,11 @@ int create_ipiri_job_from_iprange(collector_sync_t *sync, openli_export_recv_t *irimsg; char *uname = NULL; + if (ipint->common.targetagency == NULL || + strcmp(ipint->common.targetagency, "pcapdisk") == 0) { + return 0; + } + prefix = ascii2prefix(0, staticsess->rangestr); if (prefix == NULL) { logger(LOG_INFO, @@ -431,6 +436,11 @@ int create_ipiri_job_from_packet(collector_sync_t *sync, return -1; } + if (ipint->common.targetagency == NULL || + strcmp(ipint->common.targetagency, "pcapdisk") == 0) { + return 0; + } + do { irimsg = _create_ipiri_job_basic(sync, ipint, ipint->username, sess->cin); @@ -469,6 +479,11 @@ int create_ipiri_job_from_session(collector_sync_t *sync, openli_export_recv_t *irimsg; int ret = 0; + if (ipint->common.targetagency == NULL || + strcmp(ipint->common.targetagency, "pcapdisk") == 0) { + return 0; + } + irimsg = _create_ipiri_job_basic(sync, ipint, ipint->username, sess->cin); ret = sess->plugin->generate_iri_from_session(sess->plugin, sess, diff --git a/src/collector/jmirror_parser.c b/src/collector/jmirror_parser.c index 0ba15162..58ecead9 100644 --- a/src/collector/jmirror_parser.c +++ b/src/collector/jmirror_parser.c @@ -56,7 +56,7 @@ int check_jmirror_intercept(collector_identity_t *info, colthread_local_t *loc, vendmirror_intercept_list_t *vmilist; char *l3; - if ((cs = match_packet_to_coreserver(jmirror_sources, pinfo)) == NULL) { + if ((cs = match_packet_to_coreserver(jmirror_sources, pinfo, 1)) == NULL) { return 0; } diff --git a/src/collector/location.c b/src/collector/location.c index 28e53af1..693ebd51 100644 --- a/src/collector/location.c +++ b/src/collector/location.c @@ -89,7 +89,7 @@ static inline uint8_t mnc_three_digits(const char *mcc, const char *mnc) { int parse_e_utran_fdd_field(const char *field, openli_location_t **loc, int *loc_cnt) { - int step = 0; + size_t step = 0; const char *ptr = NULL; char tacbuf[5]; char ecibuf[8]; @@ -147,14 +147,13 @@ int encode_user_location_information(char *uli, int space, int *uli_len, openli_location_t *locations, uint8_t location_cnt, uint32_t location_types) { - uint16_t *lenfield = (uint16_t *)uli; - uint16_t used = 3; - uint8_t *ptr; - int i, n = 1; + uint16_t used = 0; + uint8_t *ptr = (uint8_t *)uli; + uint8_t i; + uint32_t n = 1; memset(uli, 0, space); - ptr = ((uint8_t *)uli) + used; if (location_types > 255) { logger(LOG_INFO, "OpenLI: invalid location type flags: %u\n", location_types); @@ -184,9 +183,7 @@ int encode_user_location_information(char *uli, int space, int *uli_len, n *= 2; } - *lenfield = htons(used - 3); *uli_len = used; - return 1; } diff --git a/src/collector/reassembler.c b/src/collector/reassembler.c index cf1a7fd3..7e2e4ea9 100644 --- a/src/collector/reassembler.c +++ b/src/collector/reassembler.c @@ -335,12 +335,16 @@ tcp_reassemble_stream_t *create_new_tcp_reassemble_stream( memcpy(stream->streamid, streamid, sizeof(tcp_streamid_t)); stream->lastts = 0; stream->established = TCP_STATE_OPENING; + stream->packets = calloc(4, sizeof(libtrace_packet_t *)); + stream->pkt_alloc = 4; + stream->pkt_cnt = 0; return stream; } void destroy_tcp_reassemble_stream(tcp_reassemble_stream_t *stream) { tcp_reass_segment_t *iter, *tmp; + int i; HASH_ITER(hh, stream->segments, iter, tmp) { HASH_DELETE(hh, stream->segments, iter); @@ -348,6 +352,14 @@ void destroy_tcp_reassemble_stream(tcp_reassemble_stream_t *stream) { free(iter); } + if (stream->packets) { + for (i = 0; i < stream->pkt_cnt; i++) { + if (stream->packets[i]) { + trace_destroy_packet(stream->packets[i]); + } + } + free(stream->packets); + } free(stream->streamid); free(stream); } @@ -489,7 +501,8 @@ int update_ipfrag_reassemble_stream(ip_reassemble_stream_t *stream, int update_tcp_reassemble_stream(tcp_reassemble_stream_t *stream, - uint8_t *content, uint16_t plen, uint32_t seqno) { + uint8_t *content, uint16_t plen, uint32_t seqno, + libtrace_packet_t *pkt) { tcp_reass_segment_t *seg, *existing; @@ -507,7 +520,15 @@ int update_tcp_reassemble_stream(tcp_reassemble_stream_t *stream, plen -= existing->length; seqno += existing->length; content = content + existing->length; - return update_tcp_reassemble_stream(stream, content, plen, seqno); + assert(stream->pkt_cnt > 0); + if (pkt) { + if (stream->packets[stream->pkt_cnt - 1] != NULL) { + trace_destroy_packet(stream->packets[stream->pkt_cnt - 1]); + } + stream->packets[stream->pkt_cnt - 1] = pkt; + } + return update_tcp_reassemble_stream(stream, content, plen, seqno, + NULL); } /* segment is shorter? probably don't care... */ @@ -538,6 +559,16 @@ int update_tcp_reassemble_stream(tcp_reassemble_stream_t *stream, seg->content = (uint8_t *)malloc(plen); memcpy(seg->content, content, plen); + if (pkt) { + if (stream->pkt_cnt == stream->pkt_alloc) { + stream->packets = realloc(stream->packets, + (stream->pkt_alloc + 4) * sizeof(libtrace_packet_t *)); + stream->pkt_alloc += 4; + } + stream->packets[stream->pkt_cnt] = openli_copy_packet(pkt); + stream->pkt_cnt ++; + } + HASH_ADD_KEYPTR(hh, stream->segments, &(seg->seqno), sizeof(seg->seqno), seg); stream->sorted = 0; @@ -658,7 +689,7 @@ int get_next_ip_reassembled(ip_reassemble_stream_t *stream, char **content, } int get_next_tcp_reassembled(tcp_reassemble_stream_t *stream, char **content, - uint16_t *len) { + uint16_t *len, libtrace_packet_t ***packets, int *pkt_cnt) { tcp_reass_segment_t *iter, *tmp; uint16_t contused = 0; @@ -715,6 +746,27 @@ int get_next_tcp_reassembled(tcp_reassemble_stream_t *stream, char **content, used = endfound - (uint8_t *)(*content); stream->expectedseqno += used; + + /* give all of the packets thus far back to the caller, so + * they can decide if they need to "intercept" them -- the + * raw packets are only required for pcapdisk intercepts, but + * we may not be able to know if the packets are part of a + * pcapdisk intercept until after we have reassembled the + * initial INVITE. + */ + if (packets) { + *packets = NULL; + *pkt_cnt = 0; + } + + if (packets && stream->pkt_cnt > 0) { + *packets = stream->packets; + *pkt_cnt = stream->pkt_cnt; + stream->packets = calloc(4, sizeof(libtrace_packet_t *)); + stream->pkt_cnt = 0; + stream->pkt_alloc = 4; + } + if (contstart + iter->length == endfound) { /* We've used the entire segment */ *len = contused + iter->length; @@ -753,7 +805,7 @@ int get_next_tcp_reassembled(tcp_reassemble_stream_t *stream, char **content, */ if (contused > 0 || expseqno > stream->expectedseqno) { update_tcp_reassemble_stream(stream, (uint8_t *)(*content), - contused, stream->expectedseqno); + contused, stream->expectedseqno, NULL); } *len = 0; return 0; diff --git a/src/collector/reassembler.h b/src/collector/reassembler.h index c4fc854b..7c8bbc32 100644 --- a/src/collector/reassembler.h +++ b/src/collector/reassembler.h @@ -61,6 +61,9 @@ typedef struct reass_stream { uint32_t lastts; uint32_t expectedseqno; tcp_reass_segment_t *segments; + libtrace_packet_t **packets; + int pkt_cnt; + int pkt_alloc; uint8_t sorted; uint8_t established; UT_hash_handle hh; @@ -117,9 +120,10 @@ tcp_reassemble_stream_t *create_new_tcp_reassemble_stream( reassembly_method_t method, tcp_streamid_t *streamid, uint32_t synseq); void destroy_tcp_reassemble_stream(tcp_reassemble_stream_t *reass); int update_tcp_reassemble_stream(tcp_reassemble_stream_t *reass, - uint8_t *content, uint16_t plen, uint32_t seqno); + uint8_t *content, uint16_t plen, uint32_t seqno, + libtrace_packet_t *pkt); int get_next_tcp_reassembled(tcp_reassemble_stream_t *reass, char **content, - uint16_t *len); + uint16_t *len, libtrace_packet_t ***packets, int *pkt_cnt); ipfrag_reassembler_t *create_new_ipfrag_reassembler(void); diff --git a/src/collector/sipparsing.c b/src/collector/sipparsing.c index d7c9a872..ed840cb2 100644 --- a/src/collector/sipparsing.c +++ b/src/collector/sipparsing.c @@ -36,7 +36,7 @@ #include "util.h" #include "location.h" -static int parse_tcp_sip_packet(openli_sip_parser_t *p, +static int parse_tcp_sip_packet(openli_sip_parser_t *p, libtrace_packet_t *pkt, libtrace_tcp_t *tcp, uint32_t tcprem, tcp_streamid_t *tcpid, struct timeval *tv) { @@ -70,7 +70,7 @@ static int parse_tcp_sip_packet(openli_sip_parser_t *p, } ret = update_tcp_reassemble_stream(stream, (uint8_t *)payload, tcprem, - ntohl(tcp->seq)); + ntohl(tcp->seq), pkt); return ret; @@ -138,9 +138,9 @@ int parse_sip_content(openli_sip_parser_t *p, uint8_t *sipcontent, } int parse_next_sip_message(openli_sip_parser_t *p, - libtrace_packet_t *packet) { + libtrace_packet_t ***packets, int *pkt_cnt) { - int ret; + int ret, i; if (p->osip) { osip_message_free(p->osip); @@ -152,7 +152,7 @@ int parse_next_sip_message(openli_sip_parser_t *p, p->sdp = NULL; } - if (!packet) { + if (packets != NULL && (*packets) == NULL) { if (!p->sipalloced) { p->sipmessage = NULL; @@ -160,7 +160,7 @@ int parse_next_sip_message(openli_sip_parser_t *p, if (p->thisstream) { ret = get_next_tcp_reassembled(p->thisstream, &(p->sipmessage), - &(p->siplen)); + &(p->siplen), packets, pkt_cnt); if (p->sipmessage != NULL) { p->sipalloced = 1; } @@ -240,7 +240,7 @@ static int _add_sip_packet(openli_sip_parser_t *p, libtrace_packet_t *packet, if (plen + (tcp->doff * 4) < rem) { rem = plen + (tcp->doff * 4); } - ret = parse_tcp_sip_packet(p, tcp, rem, &tcpid, tv); + ret = parse_tcp_sip_packet(p, packet, tcp, rem, &tcpid, tv); if (ret == -1) { return SIP_ACTION_IGNORE; } else if (ret == 0) { @@ -298,7 +298,7 @@ static int _add_sip_fragment(openli_sip_parser_t *p, memcpy(tcpid.srcip, stream->streamid.srcip, 16); memcpy(tcpid.destip, stream->streamid.destip, 16); - ret = parse_tcp_sip_packet(p, tcp, fraglen, &tcpid, tv); + ret = parse_tcp_sip_packet(p, NULL, tcp, fraglen, &tcpid, tv); if (ret == -1) { return SIP_ACTION_IGNORE; } else if (ret == 0) { diff --git a/src/collector/sipparsing.h b/src/collector/sipparsing.h index 8aabc5e9..de25ae33 100644 --- a/src/collector/sipparsing.h +++ b/src/collector/sipparsing.h @@ -58,6 +58,13 @@ enum { SIP_ACTION_REASSEMBLE_IPFRAG, }; +enum { + SIP_PROCESSING_PARSING, + SIP_PROCESSING_UPDATING_STATE, + SIP_PROCESSING_EXTRACTING_IPS, + SIP_PROCESSING_ADD_PARSER, +}; + typedef struct sipserverdetails { sipproto_t proto; union { @@ -104,7 +111,7 @@ int add_sip_packet_to_parser(openli_sip_parser_t **parser, int parse_sip_content(openli_sip_parser_t *parser, uint8_t *sipcontent, uint16_t siplen); int parse_next_sip_message(openli_sip_parser_t *parser, - libtrace_packet_t *packet); + libtrace_packet_t ***packets, int *pkt_cnt); void release_sip_parser(openli_sip_parser_t *parser); char *get_sip_contents(openli_sip_parser_t *parser, uint16_t *siplen); diff --git a/src/configparser.c b/src/configparser.c index bbd3c40e..531c0927 100644 --- a/src/configparser.c +++ b/src/configparser.c @@ -462,6 +462,8 @@ static int parse_core_server_list(coreserver_t **servlist, uint8_t cstype, cs->info = NULL; cs->ipstr = NULL; cs->portstr = NULL; + cs->lower_portstr = NULL; + cs->upper_portstr = NULL; cs->servertype = cstype; cs->awaitingconfirm = 1; @@ -483,6 +485,18 @@ static int parse_core_server_list(coreserver_t **servlist, uint8_t cstype, strcmp((char *)key->data.scalar.value, "port") == 0) { SET_CONFIG_STRING_OPTION(cs->portstr, value); } + + if (key->type == YAML_SCALAR_NODE && + value->type == YAML_SCALAR_NODE && + strcmp((char *)key->data.scalar.value, "port_lower") == 0) { + SET_CONFIG_STRING_OPTION(cs->lower_portstr, value); + } + + if (key->type == YAML_SCALAR_NODE && + value->type == YAML_SCALAR_NODE && + strcmp((char *)key->data.scalar.value, "port_upper") == 0) { + SET_CONFIG_STRING_OPTION(cs->upper_portstr, value); + } } if (construct_coreserver_key(cs) != NULL) { @@ -963,6 +977,7 @@ static int parse_ipintercept_list(ipintercept_t **ipints, yaml_document_t *doc, newcept->username_len = 0; newcept->vendmirrorid = OPENLI_VENDOR_MIRROR_NONE; newcept->accesstype = INTERNET_ACCESS_TYPE_UNDEFINED; + newcept->mobileident = OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; newcept->statics = NULL; newcept->options = 0; @@ -1035,6 +1050,14 @@ static int parse_ipintercept_list(ipintercept_t **ipints, yaml_document_t *doc, } } + + if (key->type == YAML_SCALAR_NODE && + value->type == YAML_SCALAR_NODE && + strcmp((char *)key->data.scalar.value, "mobileident") + == 0) { + newcept->mobileident = map_mobile_ident_string( + (char *)value->data.scalar.value); + } } if (newcept->common.encryptkey == NULL && diff --git a/src/coreserver.c b/src/coreserver.c index ddea646b..5ca8ff55 100644 --- a/src/coreserver.c +++ b/src/coreserver.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "coreserver.h" #include "logger.h" #include "util.h" @@ -66,9 +67,20 @@ coreserver_t *deep_copy_coreserver(coreserver_t *cs) { } else { cscopy->portstr = NULL; } + if (cs->lower_portstr) { + cscopy->lower_portstr = strdup(cs->lower_portstr); + } else { + cscopy->lower_portstr = NULL; + } + if (cs->upper_portstr) { + cscopy->upper_portstr = strdup(cs->upper_portstr); + } else { + cscopy->upper_portstr = NULL; + } cscopy->servertype = cs->servertype; cscopy->info = NULL; - cscopy->portswapped = cs->portswapped; + cscopy->lower_portnumeric = cs->lower_portnumeric; + cscopy->upper_portnumeric = cs->upper_portnumeric; cscopy->awaitingconfirm = 0; return cscopy; } @@ -80,6 +92,12 @@ void free_single_coreserver(coreserver_t *cs) { if (cs->portstr) { free(cs->portstr); } + if (cs->lower_portstr) { + free(cs->lower_portstr); + } + if (cs->upper_portstr) { + free(cs->upper_portstr); + } if (cs->info) { freeaddrinfo(cs->info); } @@ -95,13 +113,19 @@ char *construct_coreserver_key(coreserver_t *cs) { return NULL; } - if (cs->portstr == NULL) { - snprintf(keyspace, 256, "%s-default-%s", cs->ipstr, + if (cs->lower_portstr || cs->upper_portstr) { + snprintf(keyspace, 256, "%s-%s-%s-%s", cs->ipstr, + cs->lower_portstr ? cs->lower_portstr : "1", + cs->upper_portstr ? cs->upper_portstr : "65535", coreserver_type_to_string(cs->servertype)); + } else if (cs->portstr) { + snprintf(keyspace, 256, "%s-%s-%s-%s", cs->ipstr, cs->portstr, + cs->portstr, coreserver_type_to_string(cs->servertype)); } else { - snprintf(keyspace, 256, "%s-%s-%s", cs->ipstr, cs->portstr, + snprintf(keyspace, 256, "%s-1-65535-%s", cs->ipstr, coreserver_type_to_string(cs->servertype)); } + cs->serverkey = strdup(keyspace); return cs->serverkey; } @@ -117,31 +141,114 @@ void free_coreserver_list(coreserver_t *cslist) { } +static inline int portstr_to_numericport(const char *portstr, uint16_t *res) { + + uint16_t port16; + uint64_t toul; + + errno = 0; + toul = strtoul(portstr, NULL, 10); + if (errno) { + logger(LOG_INFO, + "OpenLI: unable to convert '%s' to a valid port number: %s", + portstr, strerror(errno)); + return -1; + } + + if (toul > 65535) { + logger(LOG_INFO, + "OpenLI: invalid port number '%lu' -- must be 65535 or below", + toul); + return -1; + } + + /* Don't actually need to swap because already in host byte order... */ + port16 = (uint16_t)toul; + *res = port16; + return 0; +} + +int prepare_coreserver(coreserver_t *cs) { + + cs->info = populate_addrinfo(cs->ipstr, NULL, SOCK_DGRAM); + if (!cs->info) { + logger(LOG_INFO, + "Removing %s from core server list due to getaddrinfo error", + cs->serverkey); + return -1; + } + + if (cs->lower_portstr && cs->upper_portstr) { + if (portstr_to_numericport(cs->lower_portstr, + &(cs->lower_portnumeric)) < 0) { + return -1; + } + if (portstr_to_numericport(cs->upper_portstr, + &(cs->upper_portnumeric)) < 0) { + return -1; + } + } else if (cs->lower_portstr) { + if (portstr_to_numericport(cs->lower_portstr, + &(cs->lower_portnumeric)) < 0) { + return -1; + } + if (portstr_to_numericport("65535", + &(cs->upper_portnumeric)) < 0) { + return -1; + } + } else if (cs->upper_portstr) { + if (portstr_to_numericport("1", + &(cs->lower_portnumeric)) < 0) { + return -1; + } + if (portstr_to_numericport(cs->upper_portstr, + &(cs->upper_portnumeric)) < 0) { + return -1; + } + } else if (cs->portstr) { + if (portstr_to_numericport(cs->portstr, + &(cs->lower_portnumeric)) < 0) { + return -1; + } + if (portstr_to_numericport(cs->portstr, + &(cs->upper_portnumeric)) < 0) { + return -1; + } + } else { + logger(LOG_INFO, + "Removing %s from core server list due to missing port information", + cs->serverkey); + return -1; + } + + if (cs->lower_portnumeric > cs->upper_portnumeric) { + logger(LOG_INFO, + "Invalid port range: %s : %s - %s (check the ordering!)\n", + cs->ipstr, cs->lower_portstr, cs->upper_portstr); + return -1; + } + + return 0; +} + coreserver_t *match_packet_to_coreserver(coreserver_t *serverlist, - packet_info_t *pinfo) { + packet_info_t *pinfo, uint8_t just_dest) { coreserver_t *cs, *tmp; if (pinfo->destport == 0) { return NULL; } + if (pinfo->srcport == 0 && just_dest == 0) { + return NULL; + } HASH_ITER(hh, serverlist, cs, tmp) { if (cs->info == NULL) { - cs->info = populate_addrinfo(cs->ipstr, cs->portstr, SOCK_DGRAM); - if (!cs->info) { - logger(LOG_INFO, - "Removing %s:%s from %s core server list due to getaddrinfo error", - cs->ipstr, cs->portstr, - coreserver_type_to_string(cs->servertype)); + if (prepare_coreserver(cs) < 0) { HASH_DELETE(hh, serverlist, cs); continue; } - if (cs->info->ai_family == AF_INET) { - cs->portswapped = ntohs(CS_TO_V4(cs)->sin_port); - } else if (cs->info->ai_family == AF_INET6) { - cs->portswapped = ntohs(CS_TO_V6(cs)->sin6_port); - } } if (cs->info->ai_family == AF_INET) { @@ -150,12 +257,24 @@ coreserver_t *match_packet_to_coreserver(coreserver_t *serverlist, if (CORESERVER_MATCH_V4(cs, sa, pinfo->destport)) { return cs; } + if (!just_dest) { + sa = (struct sockaddr_in *)(&(pinfo->srcip)); + if (CORESERVER_MATCH_V4(cs, sa, pinfo->srcport)) { + return cs; + } + } } else if (cs->info->ai_family == AF_INET6) { struct sockaddr_in6 *sa6; sa6 = (struct sockaddr_in6 *)(&(pinfo->destip)); if (CORESERVER_MATCH_V6(cs, sa6, pinfo->destport)) { return cs; } + if (!just_dest) { + sa6 = (struct sockaddr_in6 *)(&(pinfo->srcip)); + if (CORESERVER_MATCH_V6(cs, sa6, pinfo->srcport)) { + return cs; + } + } } } diff --git a/src/coreserver.h b/src/coreserver.h index c7359de3..302e28a8 100644 --- a/src/coreserver.h +++ b/src/coreserver.h @@ -65,9 +65,12 @@ typedef struct coreserver { char *serverkey; uint8_t servertype; char *ipstr; - char *portstr; + char *portstr; // kinda deprecated, but need for backwards compat + char *lower_portstr; + char *upper_portstr; struct addrinfo *info; - uint16_t portswapped; + uint16_t lower_portnumeric; + uint16_t upper_portnumeric; uint8_t awaitingconfirm; UT_hash_handle hh; @@ -80,18 +83,18 @@ const char *coreserver_type_to_string(uint8_t cstype); coreserver_t *deep_copy_coreserver(coreserver_t *cs); coreserver_t *match_packet_to_coreserver(coreserver_t *serverlist, - packet_info_t *pinfo); + packet_info_t *pinfo, uint8_t just_dest); #define CS_TO_V4(cs) ((struct sockaddr_in *)(cs->info->ai_addr)) #define CS_TO_V6(cs) ((struct sockaddr_in6 *)(cs->info->ai_addr)) #define CORESERVER_MATCH_V4(cs, sa, port) \ - ((port == cs->portswapped) && \ + ((port >= cs->lower_portnumeric && port <= cs->upper_portnumeric) && \ (memcmp(&(sa->sin_addr), &(CS_TO_V4(cs)->sin_addr), \ sizeof(struct in_addr)) == 0)) #define CORESERVER_MATCH_V6(cs, sa, port) \ - ((port == cs->portswapped) && \ + ((port >= cs->lower_portnumeric && port <= cs->upper_portnumeric) && \ (memcmp(&(sa->sin6_addr), &(CS_TO_V6(cs)->sin6_addr), \ sizeof(struct in6_addr)) == 0)) diff --git a/src/etsili_core.c b/src/etsili_core.c index 4ff3cd99..7d5c21b2 100644 --- a/src/etsili_core.c +++ b/src/etsili_core.c @@ -254,7 +254,7 @@ static inline void encode_ipiri_id(wandder_encoder_t *encoder, static inline void encode_email_recipients(wandder_encoder_t *encoder, etsili_email_recipients_t *recipients) { - int i; + size_t i; for (i = 0; i < recipients->count; i++) { wandder_encode_next(encoder, WANDDER_TAG_UTF8STR, diff --git a/src/export_buffer.c b/src/export_buffer.c index 6662bedf..c26c460e 100644 --- a/src/export_buffer.c +++ b/src/export_buffer.c @@ -222,6 +222,44 @@ uint64_t append_message_to_buffer(export_buffer_t *buf, return (buf->buftail - buf->bufhead); } +uint64_t append_heartbeat_to_buffer(export_buffer_t *buf) { + ii_header_t hbeat; + + uint32_t enclen = sizeof(hbeat); + uint64_t bufused = buf->buftail - buf->bufhead; + uint64_t spaceleft = buf->alloced - bufused; + uint32_t added = 0; + int rcint; + + hbeat.magic = htonl(OPENLI_PROTO_MAGIC); + hbeat.bodylen = 0; + hbeat.intercepttype = htons((uint16_t)OPENLI_PROTO_HEARTBEAT); + hbeat.internalid = 0; + + if (bufused == 0) { + buf->partialfront = 0; + } + + while (spaceleft < sizeof(hbeat)) { + /* Add some space to the buffer */ + spaceleft = extend_buffer(buf); + if (spaceleft == 0) { + return 0; + } + } + + memcpy(buf->buftail, &hbeat, sizeof(hbeat)); + buf->buftail += sizeof(hbeat); + added += sizeof(hbeat); + + if (buf->since_last_saved_offset + added >= BUF_OFFSET_FREQUENCY) { + J1S(rcint, buf->record_offsets, bufused); + buf->since_last_saved_offset = 0; + } + buf->since_last_saved_offset += added; + return (buf->buftail - buf->bufhead); +} + int transmit_heartbeat(int fd, SSL *ssl) { ii_header_t hbeat; char *ptr; @@ -354,7 +392,7 @@ int transmit_buffered_records(export_buffer_t *buf, int fd, return -1; } return 0; - } else if (ret < sent) { + } else if ((uint64_t)ret < sent) { /* Partial send, move partialfront ahead by whatever we did send. */ buf->partialfront += (uint32_t)ret; buf->partialrem -= (uint32_t)ret; @@ -367,14 +405,76 @@ int transmit_buffered_records(export_buffer_t *buf, int fd, return sent; } +int check_rmq_connection_block_status(amqp_connection_state_t amqp_state, + uint8_t *is_blocked) { + + amqp_frame_t frame; + struct timeval tv; + int x, ret; + + tv.tv_sec = tv.tv_usec = 0; + x = amqp_simple_wait_frame_noblock(amqp_state, &frame, &tv); + + if (x != AMQP_STATUS_OK && x != AMQP_STATUS_TIMEOUT) { + logger(LOG_INFO, + "OpenLI: unable to check status of collector RMQ publishing socket"); + return -1; + } + + if (*is_blocked) { + ret = 0; + } else { + ret = 1; + } + + if (x == AMQP_STATUS_TIMEOUT) { + return ret; + } + + if (AMQP_FRAME_METHOD == frame.frame_type) { + switch(frame.payload.method.id) { + case AMQP_CONNECTION_BLOCKED_METHOD: + if ((*is_blocked) == 0) { + logger(LOG_INFO, + "OpenLI: collector RMQ is unable to handle any more published ETSI records!"); + logger(LOG_INFO, + "OpenLI: this is a SERIOUS problem -- OpenLI will buffer in memory for now, but this will only buy you a little time"); + } + *is_blocked = 1; + ret = 0; + break; + case AMQP_CONNECTION_UNBLOCKED_METHOD: + if ((*is_blocked) == 1) { + logger(LOG_INFO, + "OpenLI: collector RMQ has become unblocked and will resume publishing ETSI records."); + ret = 0; + } else { + ret = 1; + } + *is_blocked = 0; + break; + case AMQP_CONNECTION_CLOSE_METHOD: + logger(LOG_INFO, + "OpenLI: 'close' exception occurred on the collector RMQ connection -- must restart connection"); + return -1; + case AMQP_CHANNEL_CLOSE_METHOD: + logger(LOG_INFO, + "OpenLI: channel exception occurred on the collector RMQ connection -- going to reset connection"); + return -1; + } + } + return ret; +} + + int transmit_buffered_records_RMQ(export_buffer_t *buf, amqp_connection_state_t amqp_state, amqp_channel_t channel, amqp_bytes_t exchange, amqp_bytes_t routing_key, - uint64_t bytelimit) { + uint64_t bytelimit, uint8_t *is_blocked) { uint64_t sent = 0; uint8_t *bhead = buf->bufhead + buf->deadfront; - int ret; + int ret, x; sent = (buf->buftail - (bhead)); @@ -390,26 +490,36 @@ int transmit_buffered_records_RMQ(export_buffer_t *buf, props._flags = AMQP_BASIC_DELIVERY_MODE_FLAG; props.delivery_mode = 2; /* persistent mode */ + ret = 0; + + if ((*is_blocked) == 0) { + int pub_ret = amqp_basic_publish( + amqp_state, + channel, + exchange, + routing_key, + 0, + 0, + &props, + message_bytes); + + if ( pub_ret != 0 ){ + logger(LOG_INFO, + "OpenLI: RMQ publish error %d", pub_ret); + return -1; + } else { + ret = sent; + } + } - int pub_ret = amqp_basic_publish( - amqp_state, - channel, - exchange, - routing_key, - 0, - 0, - &props, - message_bytes); - - if ( pub_ret != 0 ){ - logger(LOG_INFO, - "OpenLI: RMQ publish error %d", pub_ret); - ret = 0; - } else { - ret = sent; + if ((x = check_rmq_connection_block_status(amqp_state, + is_blocked)) < 0) { + return -1; } - buf->deadfront += ((uint32_t)ret); + if (x > 0) { + buf->deadfront += ((uint32_t)ret); + } } post_transmit(buf); diff --git a/src/export_buffer.h b/src/export_buffer.h index 9789faf8..9969a446 100644 --- a/src/export_buffer.h +++ b/src/export_buffer.h @@ -77,6 +77,7 @@ void release_export_buffer(export_buffer_t *buf); uint64_t get_buffered_amount(export_buffer_t *buf); uint64_t append_message_to_buffer(export_buffer_t *buf, openli_encoded_result_t *msg, uint32_t beensent); +uint64_t append_heartbeat_to_buffer(export_buffer_t *buf); uint64_t append_etsipdu_to_buffer(export_buffer_t *buf, uint8_t *pdustart, uint32_t pdulen, uint32_t beensent); int transmit_buffered_records(export_buffer_t *buf, int fd, @@ -84,7 +85,9 @@ int transmit_buffered_records(export_buffer_t *buf, int fd, int transmit_buffered_records_RMQ(export_buffer_t *buf, amqp_connection_state_t amqp_state, amqp_channel_t channel, amqp_bytes_t exchange, amqp_bytes_t routing_key, - uint64_t bytelimit); + uint64_t bytelimit, uint8_t *is_blocked); +int check_rmq_connection_block_status(amqp_connection_state_t amqp_state, + uint8_t *is_blocked); int transmit_heartbeat(int fd, SSL *ssl); int advance_export_buffer_head(export_buffer_t *buf, uint64_t amount); uint8_t *get_buffered_head(export_buffer_t *buf, uint64_t *rem); diff --git a/src/intercept.c b/src/intercept.c index e16bee2b..c0d82100 100644 --- a/src/intercept.c +++ b/src/intercept.c @@ -1027,11 +1027,45 @@ int add_intercept_to_email_user_intercept_list( return 0; } +int generate_ipint_userkey(ipintercept_t *ipint, char *space, + size_t spacelen) { + + char *ptr = space; + int used = 0; + + memset(space, 0, spacelen); + if (ipint->accesstype == INTERNET_ACCESS_TYPE_MOBILE) { + if (ipint->mobileident == OPENLI_MOBILE_IDENTIFIER_MSISDN || + ipint->mobileident == OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED) { + memcpy(ptr, "msisdn:", strlen("msisdn:")); + ptr += strlen("msisdn:"); + } else if (ipint->mobileident == OPENLI_MOBILE_IDENTIFIER_IMSI) { + memcpy(ptr, "imsi:", strlen("imsi:")); + ptr += strlen("imsi:"); + } else if (ipint->mobileident == OPENLI_MOBILE_IDENTIFIER_IMEI) { + memcpy(ptr, "imei:", strlen("imei:")); + ptr += strlen("imei:"); + } + } + + used = ptr - space; + + if (strlen(ipint->username) + used + 1 > spacelen) { + logger(LOG_INFO, "OpenLI: username is too long to fit in a key?"); + return -1; + } + + memcpy(ptr, ipint->username, ipint->username_len); + return used + ipint->username_len; +} + int add_intercept_to_user_intercept_list(user_intercept_list_t **ulist, ipintercept_t *ipint) { user_intercept_list_t *found; ipintercept_t *check; + char taggeduser[2048]; + if (ipint->username == NULL) { logger(LOG_INFO, @@ -1039,7 +1073,14 @@ int add_intercept_to_user_intercept_list(user_intercept_list_t **ulist, return -1; } - HASH_FIND(hh, *ulist, ipint->username, ipint->username_len, found); + if (generate_ipint_userkey(ipint, taggeduser, 2048) < 0) { + logger(LOG_INFO, + "OpenLI: error while constructing user key for IP intercept %s", + ipint->common.liid); + return -1; + } + + HASH_FIND(hh, *ulist, taggeduser, strlen(taggeduser), found); if (!found) { found = (user_intercept_list_t *)malloc(sizeof(user_intercept_list_t)); if (!found) { @@ -1047,7 +1088,7 @@ int add_intercept_to_user_intercept_list(user_intercept_list_t **ulist, "OpenLI: out of memory in add_intercept_to_userlist()"); return -1; } - found->username = strdup(ipint->username); + found->username = strdup(taggeduser); if (!found->username) { free(found); logger(LOG_INFO, @@ -1055,7 +1096,7 @@ int add_intercept_to_user_intercept_list(user_intercept_list_t **ulist, return -1; } found->intlist = NULL; - HASH_ADD_KEYPTR(hh, *ulist, found->username, ipint->username_len, + HASH_ADD_KEYPTR(hh, *ulist, found->username, strlen(found->username), found); } @@ -1149,6 +1190,7 @@ int remove_intercept_from_user_intercept_list(user_intercept_list_t **ulist, user_intercept_list_t *found; ipintercept_t *existing; + char taggeduser[2048]; if (ipint->username == NULL) { logger(LOG_INFO, @@ -1156,10 +1198,17 @@ int remove_intercept_from_user_intercept_list(user_intercept_list_t **ulist, return -1; } - HASH_FIND(hh, *ulist, ipint->username, ipint->username_len, found); + if (generate_ipint_userkey(ipint, taggeduser, 2048) < 0) { + logger(LOG_INFO, + "OpenLI: error while generating user key for intercept %s", + ipint->common.liid); + return -1; + } + + HASH_FIND(hh, *ulist, taggeduser, strlen(taggeduser), found); if (!found) { - printf("!found: %s\n", ipint->username); + printf("!found: %s\n", taggeduser); return 0; } @@ -1259,6 +1308,22 @@ const char *get_radius_ident_string(uint32_t radoptions) { return "any"; } +const char *get_mobile_identifier_string(openli_mobile_identifier_t idtype) { + switch(idtype) { + case OPENLI_MOBILE_IDENTIFIER_MSISDN: + return "MSISDN"; + case OPENLI_MOBILE_IDENTIFIER_IMSI: + return "IMSI"; + case OPENLI_MOBILE_IDENTIFIER_IMEI: + return "IMEI"; + case OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED: + return "Unspecified"; + default: + break; + } + return "Unknown"; +} + const char *get_access_type_string(internet_access_method_t method) { switch(method) { @@ -1297,6 +1362,22 @@ payload_encryption_method_t map_encrypt_method_string(char *encstr) { return OPENLI_PAYLOAD_ENCRYPTION_NONE; } +openli_mobile_identifier_t map_mobile_ident_string(char *idstr) { + if (idstr == NULL) { + return OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; + } + if (strcasecmp(idstr, "IMSI") == 0) { + return OPENLI_MOBILE_IDENTIFIER_IMSI; + } else if (strcasecmp(idstr, "MSISDN") == 0) { + return OPENLI_MOBILE_IDENTIFIER_MSISDN; + } else if (strcasecmp(idstr, "IMEI") == 0) { + return OPENLI_MOBILE_IDENTIFIER_IMEI; + } + logger(LOG_INFO, "OpenLI: unexpected mobile identifier type: %s", + idstr); + return OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; +} + uint8_t map_email_decompress_option_string(char *decstr) { if (strcasecmp(decstr, "as-is") == 0) { return OPENLI_EMAILINT_DELIVER_COMPRESSED_ASIS; diff --git a/src/intercept.h b/src/intercept.h index 0893a51e..0634f3f0 100644 --- a/src/intercept.h +++ b/src/intercept.h @@ -58,6 +58,13 @@ typedef enum { INTERNET_ACCESS_TYPE_MOBILE = 32, /* Not a "real" value */ } internet_access_method_t; +typedef enum { + OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED = 0, + OPENLI_MOBILE_IDENTIFIER_MSISDN = 1, + OPENLI_MOBILE_IDENTIFIER_IMSI = 2, + OPENLI_MOBILE_IDENTIFIER_IMEI = 3, +} openli_mobile_identifier_t; + typedef enum { OPENLI_PAYLOAD_ENCRYPTION_NOT_SPECIFIED = 0, OPENLI_PAYLOAD_ENCRYPTION_NONE = 1, @@ -117,8 +124,8 @@ typedef struct intercept_common { char *targetagency; int seqtrackerid; uint32_t hi1_seqno; - uint64_t tostart_time; - uint64_t toend_time; + time_t tostart_time; + time_t toend_time; intercept_outputs_t tomediate; payload_encryption_method_t encrypt; char *encryptkey; @@ -157,6 +164,7 @@ typedef struct ipintercept { static_ipranges_t *statics; + openli_mobile_identifier_t mobileident; uint8_t awaitingconfirm; uint32_t options; UT_hash_handle hh_liid; @@ -317,7 +325,7 @@ struct emailsession { uint32_t client_octets; uint64_t login_time; uint8_t login_sent; - uint64_t event_time; + time_t event_time; char *ingest_target_id; uint8_t ingest_direction; @@ -333,8 +341,8 @@ struct emailsession { void *proto_state; void **held_captured; - int held_captured_size; - int next_expected_captured; + uint32_t held_captured_size; + uint32_t next_expected_captured; uint8_t sender_validated_etsivalue; Pvoid_t ccs_sent; @@ -518,12 +526,17 @@ int add_intercept_to_email_user_intercept_list( email_user_intercept_list_t *ulist, emailintercept_t *em, email_target_t *tgt); +int generate_ipint_userkey(ipintercept_t *ipint, char *space, + size_t spacelen); + +const char *get_mobile_identifier_string(openli_mobile_identifier_t idtype); const char *get_access_type_string(internet_access_method_t method); const char *get_radius_ident_string(uint32_t radoptions); internet_access_method_t map_access_type_string(char *confstr); uint32_t map_radius_ident_string(char *confstr); payload_encryption_method_t map_encrypt_method_string(char *encstr); uint8_t map_email_decompress_option_string(char *decstr); +openli_mobile_identifier_t map_mobile_ident_string(char *idstr); void intercept_mediation_mode_as_string(intercept_outputs_t mode, char *space, int spacelen); diff --git a/src/mediator/coll_recv_thread.c b/src/mediator/coll_recv_thread.c index 8894dc43..f46fbff2 100644 --- a/src/mediator/coll_recv_thread.c +++ b/src/mediator/coll_recv_thread.c @@ -148,8 +148,11 @@ static void remove_expired_liid_queues(coll_recv_t *col) { if (tv.tv_sec - known->lastseen < LIID_QUEUE_EXPIRY_THRESH) { /* Not expired yet, so redeclare the queue to keep rabbitmq * from deleting it accidentally */ - declare_mediator_liid_RMQ_queue(col->amqp_producer_state, - known->liid, known->liidlen); + if (declare_mediator_liid_RMQ_queue(col->amqp_producer_state, + known->liid, known->liidlen, + &(col->rmq_blocked)) > 0) { + known->declared_int_rmq = 1; + } continue; } @@ -213,6 +216,7 @@ static int start_collector_ssl(coll_recv_t *col) { "OpenLI Mediator: SSL handshake failed for collector %s", col->ipaddr); } + col->lastsslerror = r; return -1; } @@ -382,6 +386,16 @@ static int continue_collector_handshake(coll_recv_t *col, med_epoll_ev_t *mev) { return 1; } +static void increment_col_drop_counter(coll_recv_t *col) { + + col->dropped_recs ++; + if (col->dropped_recs == 10 || col->dropped_recs % 1000 == 0) { + logger(LOG_INFO, + "OpenLI mediator: dropped %lu records from collector %s so far", + col->dropped_recs, col->ipaddr); + } +} + /** Processes an intercept record received from a collector and inserts * it into the appropriate mediator-internal LIID queue. * @@ -428,6 +442,7 @@ static int process_received_data(coll_recv_t *col, uint8_t *msgbody, found->liidlen = strlen(found->liid); found->lastseen = 0; found->declared_raw_rmq = 0; + found->declared_int_rmq = 0; snprintf(qname, 1024, "%s-iri", found->liid); found->queuenames[0] = strdup(qname); @@ -440,12 +455,19 @@ static int process_received_data(coll_recv_t *col, uint8_t *msgbody, found); logger(LOG_INFO, "OpenLI Mediator: LIID %s has been seen coming from collector %s", found->liid, col->ipaddr); + } + + if (found->declared_int_rmq == 0) { /* declare amqp queue for this LIID */ - if (declare_mediator_liid_RMQ_queue(col->amqp_producer_state, - found->liid, found->liidlen) < 0) { + r = declare_mediator_liid_RMQ_queue(col->amqp_producer_state, + found->liid, found->liidlen, &(col->rmq_blocked)); + if (r < 0) { logger(LOG_INFO, "OpenLI Mediator: failed to create internal RMQ queues for LIID %s in collector thread %s", found->liid, col->ipaddr); return -1; } + if (r > 0) { + found->declared_int_rmq = 1; + } } gettimeofday(&tv, NULL); @@ -453,31 +475,75 @@ static int process_received_data(coll_recv_t *col, uint8_t *msgbody, /* Hand off to publishing methods defined in mediator_rmq.c */ if (msgtype == OPENLI_PROTO_ETSI_CC) { - r = publish_cc_on_mediator_liid_RMQ_queue(col->amqp_producer_state, - msgbody + (liidlen + 2), msglen - (liidlen + 2), found->liid, - found->queuenames[1]); + if (found->declared_int_rmq) { + r = publish_cc_on_mediator_liid_RMQ_queue(col->amqp_producer_state, + msgbody + (liidlen + 2), msglen - (liidlen + 2), + found->liid, found->queuenames[1], &(col->rmq_blocked)); + if (r <= 0) { + increment_col_drop_counter(col); + } + if (r < 0) { + amqp_destroy_connection(col->amqp_producer_state); + col->amqp_producer_state = NULL; + } + } else { + increment_col_drop_counter(col); + r = 0; + } return r; } if (msgtype == OPENLI_PROTO_ETSI_IRI) { - return publish_iri_on_mediator_liid_RMQ_queue(col->amqp_producer_state, - msgbody + (liidlen + 2), msglen - (liidlen + 2), found->liid, - found->queuenames[0]); + if (found->declared_int_rmq) { + r = publish_iri_on_mediator_liid_RMQ_queue( + col->amqp_producer_state, + msgbody + (liidlen + 2), msglen - (liidlen + 2), + found->liid, found->queuenames[0], &(col->rmq_blocked)); + if (r <= 0) { + increment_col_drop_counter(col); + } + if (r < 0) { + amqp_destroy_connection(col->amqp_producer_state); + col->amqp_producer_state = NULL; + } + } else { + increment_col_drop_counter(col); + r = 0; + } + return r; } if (msgtype == OPENLI_PROTO_RAWIP_SYNC || - msgtype == OPENLI_PROTO_RAWIP_CC) { + msgtype == OPENLI_PROTO_RAWIP_CC || + msgtype == OPENLI_PROTO_RAWIP_IRI) { /* declare a queue for raw IP */ if (!found->declared_raw_rmq) { - declare_mediator_rawip_RMQ_queue(col->amqp_producer_state, - found->liid, found->liidlen); - found->declared_raw_rmq = 1; + r = declare_mediator_rawip_RMQ_queue(col->amqp_producer_state, + found->liid, found->liidlen, &(col->rmq_blocked)); + if (r < 0) { + return -1; + } else if (r > 0) { + found->declared_raw_rmq = 1; + } } /* publish to raw IP queue */ - return publish_rawip_on_mediator_liid_RMQ_queue( - col->amqp_producer_state, msgbody, msglen, found->liid, - found->queuenames[2]); + if (found->declared_raw_rmq) { + r = publish_rawip_on_mediator_liid_RMQ_queue( + col->amqp_producer_state, msgbody, msglen, found->liid, + found->queuenames[2], &(col->rmq_blocked)); + if (r <= 0) { + increment_col_drop_counter(col); + } + if (r < 0) { + amqp_destroy_connection(col->amqp_producer_state); + col->amqp_producer_state = NULL; + } + } else { + increment_col_drop_counter(col); + r = 0; + } + return r; } return 1; @@ -538,6 +604,7 @@ static int receive_collector(coll_recv_t *col, med_epoll_ev_t *mev) { break; case OPENLI_PROTO_RAWIP_SYNC: case OPENLI_PROTO_RAWIP_CC: + case OPENLI_PROTO_RAWIP_IRI: case OPENLI_PROTO_ETSI_CC: case OPENLI_PROTO_ETSI_IRI: /* Intercept record -- process it appropriately */ @@ -673,6 +740,10 @@ static void cleanup_collector_thread(coll_recv_t *col) { if (col->ipaddr) { logger(LOG_INFO, "OpenLI mediator: exiting collector thread for %s", col->ipaddr); + logger(LOG_INFO, + "OpenLI mediator: dropped %lu records from collector %s", + col->dropped_recs, col->ipaddr); + free(col->ipaddr); } @@ -957,6 +1028,7 @@ int mediator_accept_collector_connection(mediator_collector_t *medcol, newcol->ipaddr = strdup(strbuf); newcol->iplen = strlen(strbuf); newcol->col_fd = newfd; + newcol->rmq_blocked = 0; HASH_ADD_KEYPTR(hh, medcol->threads, newcol->ipaddr, newcol->iplen, newcol); diff --git a/src/mediator/coll_recv_thread.h b/src/mediator/coll_recv_thread.h index 3871a1b3..fd6a3042 100644 --- a/src/mediator/coll_recv_thread.h +++ b/src/mediator/coll_recv_thread.h @@ -101,6 +101,11 @@ typedef struct col_known_liid { */ uint8_t declared_raw_rmq; + /** Flag indicating whether we have declared the RMQs for publishing + * IRIs and CCs internally + */ + uint8_t declared_int_rmq; + const char *queuenames[3]; UT_hash_handle hh; @@ -214,6 +219,16 @@ typedef struct single_coll_receiver { */ libtrace_message_queue_t in_main; + /** Flag that indicates whether RMQ has told us that it is "connection + * blocked, i.e. no longer able to accept published messages + */ + uint8_t rmq_blocked; + + /** Number of received records that we have been unable to publish + * to the internal RMQ + */ + uint64_t dropped_recs; + UT_hash_handle hh; } coll_recv_t; diff --git a/src/mediator/handover.c b/src/mediator/handover.c index efb68e3a..b1f488a1 100644 --- a/src/mediator/handover.c +++ b/src/mediator/handover.c @@ -44,13 +44,13 @@ */ int xmit_handover_keepalive(handover_t *ho) { - /* We don't lock the handover mutex here, because we're going to be + /* We don't lock the handover mutex here, because we're going to be * doing this a lot and the mutex is mostly protecting logging-related - * members (e.g. disconnect_msg). A few bogus messages are a small + * members (e.g. disconnect_msg). A few bogus messages are a small * price to pay compared with the performance impact of locking a mutex * everytime we want to send a record to a client. */ - int ret = 0; + int ret = 0; if (!ho->ho_state->pending_ka) { return 0; @@ -74,7 +74,7 @@ int xmit_handover_keepalive(handover_t *ho) { if (ret == 0) { return -1; } - if (ret == ho->ho_state->pending_ka->len) { + if ((unsigned int)ret == ho->ho_state->pending_ka->len) { /* Sent the whole thing successfully */ wandder_release_encoded_result(NULL, ho->ho_state->pending_ka); ho->ho_state->pending_ka = NULL; diff --git a/src/mediator/mediator_prov.c b/src/mediator/mediator_prov.c index b5c478c3..860983a2 100644 --- a/src/mediator/mediator_prov.c +++ b/src/mediator/mediator_prov.c @@ -353,7 +353,7 @@ int attempt_provisioner_connect(mediator_prov_t *prov, int provfail) { int transmit_provisioner(mediator_prov_t *prov, med_epoll_ev_t *mev) { int ret; - openli_proto_msgtype_t err; + openli_proto_msgtype_t err = OPENLI_PROTO_NO_MESSAGE; /* Try to send whatever we've got in the netcomms buffer */ ret = transmit_net_buffer(prov->outgoing, &err); diff --git a/src/mediator/mediator_rmq.c b/src/mediator/mediator_rmq.c index 3bc44338..e8688de3 100644 --- a/src/mediator/mediator_rmq.c +++ b/src/mediator/mediator_rmq.c @@ -43,7 +43,8 @@ * @param queueid The name to assign to the queue * @param channel The channel to declare the queue on * - * @return -1 if an error occurs, 0 otherwise + * @return -1 if an error occurs, 0 if the queue cannot be declared right + * now, 1 if the queue was declared successfully. */ static int declare_RMQ_queue(amqp_connection_state_t state, char *queueid, int channel) { @@ -75,7 +76,7 @@ static int declare_RMQ_queue(amqp_connection_state_t state, return -1; } - return 0; + return 1; } /** Checks if a particular RabbitMQ queue is empty (i.e. contains zero @@ -147,6 +148,70 @@ static int register_RMQ_consumer(amqp_connection_state_t state, return 0; } +static int update_mediator_rmq_connection_block_status( + amqp_connection_state_t state, uint8_t *is_blocked) { + + /* copy of code from export_buffer.c, but with different log + * messages when things go awry + */ + amqp_frame_t frame; + struct timeval tv; + int x, ret; + + tv.tv_sec = tv.tv_usec = 0; + x = amqp_simple_wait_frame_noblock(state, &frame, &tv); + + if (x != AMQP_STATUS_OK && x != AMQP_STATUS_TIMEOUT) { + logger(LOG_INFO, + "OpenLI mediator: unable to check status of an internal RMQ publishing socket"); + return -1; + } + + if (*is_blocked) { + ret = 0; + } else { + ret = 1; + } + + if (x == AMQP_STATUS_TIMEOUT) { + return ret; + } + + if (AMQP_FRAME_METHOD == frame.frame_type) { + switch(frame.payload.method.id) { + case AMQP_CONNECTION_BLOCKED_METHOD: + if ((*is_blocked) == 0) { + logger(LOG_INFO, + "OpenLI mediator: RMQ is unable to handle any more published ETSI records!"); + logger(LOG_INFO, + "OpenLI mediator: this is a SERIOUS problem -- received ETSI records are going to be dropped!"); + } + *is_blocked = 1; + ret = 0; + break; + case AMQP_CONNECTION_UNBLOCKED_METHOD: + if ((*is_blocked) == 1) { + logger(LOG_INFO, + "OpenLI mediator: RMQ has become unblocked and will resume publishing ETSI records."); + ret = 0; + } else { + ret = 1; + } + *is_blocked = 0; + break; + case AMQP_CONNECTION_CLOSE_METHOD: + logger(LOG_INFO, + "OpenLI mediator: 'close' exception occurred on an internal RMQ connection -- must restart connection"); + return -1; + case AMQP_CHANNEL_CLOSE_METHOD: + logger(LOG_INFO, + "OpenLI mediator: channel exception occurred on an internal RMQ connection -- must reset connection"); + return -1; + } + } + return ret; +} + /** Disables consumption from a RabbitMQ queue by an existing connection * * @param state The RMQ connection to disassociate the queue from @@ -177,14 +242,23 @@ static int cancel_RMQ_consumer(amqp_connection_state_t state, * @param liid The LIID to declare queues for * @param liidlen The length of the LIID (in bytes) * - * @return -1 if an error occurs, 0 otherwise. + * @return -1 if an error occurs, 0 if the queue cannot be declared right + * now, 1 if the queue was declared successfully. */ int declare_mediator_liid_RMQ_queue(amqp_connection_state_t state, - char *liid, int liidlen) { + char *liid, int liidlen, uint8_t *is_blocked) { char cc_queuename[1024]; char iri_queuename[1024]; + /* + if (update_mediator_rmq_connection_block_status(state, is_blocked) < 0) { + return -1; + } + */ + if (*is_blocked) { + return 0; + } snprintf(cc_queuename, 1024, "%s-%s", liid, "cc"); snprintf(iri_queuename, 1024, "%s-%s", liid, "iri"); @@ -202,14 +276,23 @@ int declare_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param liid The LIID to declare a raw IP queue for * @param liidlen The length of the LIID (in bytes) * - * @return -1 if an error occurs, 0 otherwise. + * @return -1 if an error occurs, 0 if the queue cannot be declared right + * now, 1 if the queue was declared successfully. */ int declare_mediator_rawip_RMQ_queue(amqp_connection_state_t state, - char *liid, int liidlen) { + char *liid, int liidlen, uint8_t *is_blocked) { char queuename[1024]; - snprintf(queuename, 1024, "%s-rawip", liid); - return declare_RMQ_queue(state, queuename, 4); + /* + if (update_mediator_rmq_connection_block_status(state, is_blocked) < 0) { + return -1; + } + */ + if (*is_blocked == 0) { + snprintf(queuename, 1024, "%s-rawip", liid); + return declare_RMQ_queue(state, queuename, 4); + } + return 0; } /** Publishes a message onto a mediator RMQ queue. @@ -224,12 +307,13 @@ int declare_mediator_rawip_RMQ_queue(amqp_connection_state_t state, * @param queuename THe name of the queue to publish to * @param expiry The TTL of the message in seconds -- if set to 0, * the message will not be expired by RMQ + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ static int produce_mediator_RMQ(amqp_connection_state_t state, uint8_t *msg, uint16_t msglen, char *liid, int channel, - const char *queuename, uint32_t expiry) { + const char *queuename, uint32_t expiry, uint8_t *is_blocked) { amqp_bytes_t message_bytes; amqp_basic_properties_t props; int pub_ret; @@ -246,10 +330,18 @@ static int produce_mediator_RMQ(amqp_connection_state_t state, props.expiration = amqp_cstring_bytes(expirystr); } - pub_ret = amqp_basic_publish(state, channel, amqp_cstring_bytes(""), - amqp_cstring_bytes(queuename), 0, 0, &props, message_bytes); - if (pub_ret != 0) { - logger(LOG_INFO, "OpenLI Mediator: error publishing to internal RMQ for LIID %s: %d", liid, pub_ret); + if (update_mediator_rmq_connection_block_status(state, is_blocked) < 0) { + return -1; + } + + if ((*is_blocked) == 0) { + pub_ret = amqp_basic_publish(state, channel, amqp_cstring_bytes(""), + amqp_cstring_bytes(queuename), 0, 0, &props, message_bytes); + if (pub_ret != 0) { + logger(LOG_INFO, "OpenLI Mediator: error publishing to internal RMQ for LIID %s: %d", liid, pub_ret); + return -1; + } + } else { return 0; } @@ -263,12 +355,15 @@ static int produce_mediator_RMQ(amqp_connection_state_t state, * @param msglen The length of the packet body, in bytes * @param liid The LIID that the message belongs to * @param queuename THe name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_rawip_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename) { - return produce_mediator_RMQ(state, msg, msglen, liid, 4, queuename, 0); + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked) { + return produce_mediator_RMQ(state, msg, msglen, liid, 4, queuename, 0, + is_blocked); } /** Publishes an encoded IRI onto a mediator RMQ queue. @@ -278,13 +373,16 @@ int publish_rawip_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param msglen The length of the encoded IRI, in bytes * @param liid The LIID that the message belongs to * @param queuename THe name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_iri_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename) { + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked) { - return produce_mediator_RMQ(state, msg, msglen, liid, 2, queuename, 0); + return produce_mediator_RMQ(state, msg, msglen, liid, 2, queuename, 0, + is_blocked); } /** Publishes an encoded CC onto a mediator RMQ queue. @@ -293,14 +391,17 @@ int publish_iri_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param msg A pointer to the start of the encoded CC * @param msglen The length of the encoded CC, in bytes * @param liid The LIID that the message belongs to - * @param queuename THe name of the queue to publish to + * @param queuename The name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_cc_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename) { + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked) { - return produce_mediator_RMQ(state, msg, msglen, liid, 3, queuename, 0); + return produce_mediator_RMQ(state, msg, msglen, liid, 3, queuename, 0, + is_blocked); } void remove_mediator_liid_RMQ_queue(amqp_connection_state_t state, @@ -433,6 +534,12 @@ amqp_connection_state_t join_mediator_RMQ_as_consumer(char *agencyid, */ amqp_connection_state_t join_mediator_RMQ_as_producer(coll_recv_t *col) { + amqp_table_entry_t login_properties[1]; + amqp_table_t login_properties_table; + + amqp_table_entry_t client_capabilities[1]; + amqp_table_t client_capabilities_table; + if (col->amqp_producer_state) { return col->amqp_producer_state; } @@ -452,10 +559,25 @@ amqp_connection_state_t join_mediator_RMQ_as_producer(coll_recv_t *col) { goto prodfailed; } + client_capabilities[0].key = amqp_cstring_bytes("connection.blocked"); + client_capabilities[0].value.kind = AMQP_FIELD_KIND_BOOLEAN; + client_capabilities[0].value.value.boolean = 1; + + client_capabilities_table.entries = client_capabilities; + client_capabilities_table.num_entries = 1; + + login_properties[0].key = amqp_cstring_bytes("capabilities"); + login_properties[0].value.kind = AMQP_FIELD_KIND_TABLE; + login_properties[0].value.value.table = client_capabilities_table; + + login_properties_table.entries = login_properties; + login_properties_table.num_entries = 1; + /* Hard-coded username and password -- not ideal, but the RMQ instance * should only be accessible via localhost. */ - if ((amqp_login(col->amqp_producer_state, "OpenLI-med", 0, 131072, 0, + if ((amqp_login_with_properties(col->amqp_producer_state, "OpenLI-med", 0, + 131072, 0, &login_properties_table, AMQP_SASL_METHOD_PLAIN, "openli.nz", col->internalpass)) .reply_type != AMQP_RESPONSE_NORMAL) { if (col->disabled_log == 0) { diff --git a/src/mediator/mediator_rmq.h b/src/mediator/mediator_rmq.h index 15eed492..e1f668ba 100644 --- a/src/mediator/mediator_rmq.h +++ b/src/mediator/mediator_rmq.h @@ -153,11 +153,12 @@ int deregister_mediator_rawip_RMQ_consumer(amqp_connection_state_t state, * @param state The RMQ connection to use to declare the queues * @param liid The LIID to declare queues for * @param liidlen The length of the LIID (in bytes) + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return -1 if an error occurs, 0 otherwise. */ int declare_mediator_liid_RMQ_queue(amqp_connection_state_t state, - char *liid, int liidlen); + char *liid, int liidlen, uint8_t *is_blocked); /** Declares the Raw IP queue in RabbitMQ for a particular LIID * @@ -166,11 +167,12 @@ int declare_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param state The RMQ connection to use to declare the queue * @param liid The LIID to declare a raw IP queue for * @param liidlen The length of the LIID (in bytes) + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return -1 if an error occurs, 0 otherwise. */ int declare_mediator_rawip_RMQ_queue(amqp_connection_state_t state, - char *liid, int liidlen); + char *liid, int liidlen, uint8_t *is_blocked); void remove_mediator_liid_RMQ_queue(amqp_connection_state_t state, @@ -185,11 +187,13 @@ void remove_mediator_rawip_RMQ_queue(amqp_connection_state_t state, * @param msglen The length of the encoded CC, in bytes * @param liid The LIID that the message belongs to * @param queuename THe name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_iri_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename); + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked); /** Publishes an encoded CC onto a mediator RMQ queue. * @@ -198,11 +202,13 @@ int publish_iri_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param msglen The length of the encoded CC, in bytes * @param liid The LIID that the message belongs to * @param queuename THe name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_cc_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename); + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked); /** Publishes an encoded CC onto a mediator RMQ queue. * @@ -211,11 +217,13 @@ int publish_cc_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, * @param msglen The length of the encoded CC, in bytes * @param liid The LIID that the message belongs to * @param queuename THe name of the queue to publish to + * @param is_blocked [in|out] Is the RMQ broker accepting publishes? * * @return 0 if an error occurs, 1 if the message is published successfully */ int publish_rawip_on_mediator_liid_RMQ_queue(amqp_connection_state_t state, - uint8_t *msg, uint16_t msglen, char *liid, const char *queuename); + uint8_t *msg, uint16_t msglen, char *liid, const char *queuename, + uint8_t *is_blocked); /** Consumes CC records using an RMQ connection, writing them into the * provided export buffer. diff --git a/src/mediator/pcapthread.c b/src/mediator/pcapthread.c index ba84b365..8106f3a7 100644 --- a/src/mediator/pcapthread.c +++ b/src/mediator/pcapthread.c @@ -297,6 +297,9 @@ static int open_pcap_output_file(lea_thread_state_t *state, goto pcaptraceerr; } + if (act->uri) { + free(act->uri); + } act->uri = strdup(uri); act->pktwritten = 0; diff --git a/src/netcomms.c b/src/netcomms.c index 30e9dbf1..02e3ec25 100644 --- a/src/netcomms.c +++ b/src/netcomms.c @@ -376,12 +376,13 @@ int push_lea_withdrawal_onto_net_buffer(net_buffer_t *nb, liagency_t *lea) { #define VENDMIRROR_IPINTERCEPT_MODIFY_BODY_LEN(ipint) \ (INTERCEPT_COMMON_LEN(ipint->common) + \ ipint->username_len + sizeof(ipint->accesstype) + \ - sizeof(ipint->options) + sizeof(ipint->vendmirrorid) + (4 * 4)) + sizeof(ipint->options) + sizeof(ipint->vendmirrorid) + \ + sizeof(ipint->mobileident) + (5 * 4)) #define IPINTERCEPT_MODIFY_BODY_LEN(ipint) \ (INTERCEPT_COMMON_LEN(ipint->common) + \ ipint->username_len + sizeof(ipint->accesstype) + \ - sizeof(ipint->options) + (3 * 4)) + sizeof(ipint->options) + sizeof(ipint->mobileident) + (4 * 4)) static int _push_intercept_common_fields(net_buffer_t *nb, intercept_common_t *common) { @@ -482,6 +483,12 @@ static int _push_ipintercept_modify(net_buffer_t *nb, ipintercept_t *ipint) { goto pushmodfail; } + if (push_tlv(nb, OPENLI_PROTO_FIELD_MOBILEIDENT, + (uint8_t *)(&(ipint->mobileident)), + sizeof(ipint->mobileident)) == -1) { + goto pushmodfail; + } + if (push_tlv(nb, OPENLI_PROTO_FIELD_INTOPTIONS, (uint8_t *)(&(ipint->options)), sizeof(ipint->options)) == -1) { @@ -967,12 +974,13 @@ int push_static_ipranges_onto_net_buffer(net_buffer_t *nb, #define IPINTERCEPT_BODY_LEN(ipint) \ (INTERCEPT_COMMON_LEN(ipint->common) + \ ipint->username_len + sizeof(ipint->options) + \ - sizeof(ipint->accesstype) + (3 * 4)) + sizeof(ipint->accesstype) + sizeof(ipint->mobileident) + (4 * 4)) #define VENDMIRROR_IPINTERCEPT_BODY_LEN(ipint) \ (INTERCEPT_COMMON_LEN(ipint->common) + \ ipint->username_len + sizeof(ipint->vendmirrorid) + \ - sizeof(ipint->options) + sizeof(ipint->accesstype) + (4 * 4)) + sizeof(ipint->options) + sizeof(ipint->accesstype) + \ + sizeof(ipint->mobileident) + (5 * 4)) int push_ipintercept_onto_net_buffer(net_buffer_t *nb, void *data) { @@ -1020,6 +1028,12 @@ int push_ipintercept_onto_net_buffer(net_buffer_t *nb, void *data) { goto pushipintfail; } + if ((ret = push_tlv(nb, OPENLI_PROTO_FIELD_MOBILEIDENT, + (uint8_t *)(&ipint->mobileident), + sizeof(ipint->mobileident))) == -1) { + goto pushipintfail; + } + if ((ret = push_tlv(nb, OPENLI_PROTO_FIELD_INTOPTIONS, (uint8_t *)(&ipint->options), sizeof(ipint->options))) == -1) { @@ -1249,9 +1263,29 @@ int push_default_email_compression_onto_net_buffer(net_buffer_t *nb, return totallen; } -#define CORESERVER_BODY_LEN(cs) \ - (sizeof(uint8_t) + strlen(cs->ipstr) + \ - (cs->portstr ? (strlen(cs->portstr) + (3 * 4)) : (2 * 4))) +static inline uint16_t get_coreserver_body_len(coreserver_t *cs) { + uint16_t len = 0; + size_t fcount = 2; + len += (sizeof(uint8_t) + strlen(cs->ipstr)); + + if (cs->portstr) { + len += strlen(cs->portstr); + fcount ++; + } + + if (cs->upper_portstr) { + len += strlen(cs->upper_portstr); + fcount ++; + } + + if (cs->lower_portstr) { + len += strlen(cs->lower_portstr); + fcount ++; + } + + len += (fcount * 4); + return len; +} static int push_coreserver_msg_onto_net_buffer(net_buffer_t *nb, coreserver_t *cs, uint8_t cstype, openli_proto_msgtype_t type) { @@ -1261,7 +1295,7 @@ static int push_coreserver_msg_onto_net_buffer(net_buffer_t *nb, int ret; /* Pre-compute our body length so we can write it in the header */ - totallen = CORESERVER_BODY_LEN(cs); + totallen = get_coreserver_body_len(cs); /* Push on header */ populate_header(&hdr, type, totallen, 0); @@ -1288,6 +1322,22 @@ static int push_coreserver_msg_onto_net_buffer(net_buffer_t *nb, } } + if (cs->upper_portstr) { + if ((ret = push_tlv(nb, OPENLI_PROTO_FIELD_CORESERVER_UPPER_PORT, + (uint8_t *)cs->upper_portstr, + strlen(cs->upper_portstr))) == -1) { + return -1; + } + } + + if (cs->lower_portstr) { + if ((ret = push_tlv(nb, OPENLI_PROTO_FIELD_CORESERVER_LOWER_PORT, + (uint8_t *)cs->lower_portstr, + strlen(cs->lower_portstr))) == -1) { + return -1; + } + } + return (int)totallen; } @@ -1641,6 +1691,7 @@ int decode_ipintercept_start(uint8_t *msgbody, uint16_t len, ipint->accesstype = INTERNET_ACCESS_TYPE_UNDEFINED; ipint->statics = NULL; ipint->options = 0; + ipint->mobileident = OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; ipint->common.liid_len = 0; ipint->common.authcc_len = 0; @@ -1667,6 +1718,8 @@ int decode_ipintercept_start(uint8_t *msgbody, uint16_t len, ipint->vendmirrorid = *((uint32_t *)valptr); } else if (f == OPENLI_PROTO_FIELD_ACCESSTYPE) { ipint->accesstype = *((internet_access_method_t *)valptr); + } else if (f == OPENLI_PROTO_FIELD_MOBILEIDENT) { + ipint->mobileident = *((openli_mobile_identifier_t *)valptr); } else if (f == OPENLI_PROTO_FIELD_LIID) { DECODE_STRING_FIELD(ipint->common.liid, valptr, vallen); ipint->common.liid_len = vallen; @@ -1870,6 +1923,8 @@ int decode_coreserver_announcement(uint8_t *msgbody, uint16_t len, cs->servertype = OPENLI_CORE_SERVER_UNKNOWN; cs->ipstr = NULL; cs->portstr = NULL; + cs->upper_portstr = NULL; + cs->lower_portstr = NULL; cs->info = NULL; cs->awaitingconfirm = 0; cs->serverkey = NULL; @@ -1889,6 +1944,10 @@ int decode_coreserver_announcement(uint8_t *msgbody, uint16_t len, DECODE_STRING_FIELD(cs->ipstr, valptr, vallen); } else if (f == OPENLI_PROTO_FIELD_CORESERVER_PORT) { DECODE_STRING_FIELD(cs->portstr, valptr, vallen); + } else if (f == OPENLI_PROTO_FIELD_CORESERVER_UPPER_PORT) { + DECODE_STRING_FIELD(cs->upper_portstr, valptr, vallen); + } else if (f == OPENLI_PROTO_FIELD_CORESERVER_LOWER_PORT) { + DECODE_STRING_FIELD(cs->lower_portstr, valptr, vallen); } else { dump_buffer_contents(msgbody, len); logger(LOG_INFO, @@ -2286,16 +2345,15 @@ openli_proto_msgtype_t receive_RMQ_buffer(net_buffer_t *nb, if (AMQP_FRAME_METHOD == frame.frame_type) { switch (frame.payload.method.id) { case AMQP_BASIC_ACK_METHOD: - /* if we've turned publisher confirms on, and we've published a - * message here is a message being confirmed. - */ - logger(LOG_INFO, "basic ack"); - //break; + /* if we've turned publisher confirms on, and + * we've published a message here, then this is a + * message being confirmed. + */ return OPENLI_PROTO_NO_MESSAGE; case AMQP_BASIC_RETURN_METHOD: - /* if a published message couldn't be routed and the mandatory - * flag was set this is what would be returned. The message then - * needs to be read. + /* if a published message couldn't be routed and the + * mandatory flag was set this is what would be + * returned. The message then needs to be read. */ { amqp_message_t message; @@ -2306,28 +2364,28 @@ openli_proto_msgtype_t receive_RMQ_buffer(net_buffer_t *nb, amqp_destroy_message(&message); } - //break; return OPENLI_PROTO_NO_MESSAGE; case AMQP_CHANNEL_CLOSE_METHOD: - /* a channel.close method happens when a channel exception occurs, - * this can happen by publishing to an exchange that doesn't exist - * for example. - * - * In this case you would need to open another channel redeclare - * any queues that were declared auto-delete, and restart any - * consumers that were attached to the previous channel. - */ + /* a channel.close method happens when a channel + * exception occurs, this can happen by publishing to + * an exchange that doesn't exist (for example). + * + * In this case you would need to open another channel, + * redeclare any queues that were declared auto-delete, + * and restart any consumers that were attached to the + * previous channel. + */ logger(LOG_INFO, "OpenLI: RMQ Channel closed"); return OPENLI_PROTO_RECV_ERROR; case AMQP_CONNECTION_CLOSE_METHOD: - /* a connection.close method happens when a connection exception - * occurs, this can happen by trying to use a channel that isn't - * open for example. - * - * In this case the whole connection must be restarted. - */ + /* a connection.close method happens when a connection + * exception occurs, this can happen by trying to use + * a channel that isn't open (for example). + * + * In this case the whole connection must be restarted. + */ return OPENLI_PROTO_PEER_DISCONNECTED; default: @@ -2425,6 +2483,10 @@ void nb_log_receive_error(openli_proto_msgtype_t err) { logger(LOG_INFO, "OpenLI: received invalid protocol message."); break; + case OPENLI_PROTO_NO_MESSAGE: + logger(LOG_INFO, + "OpenLI: error cause not recorded by OpenLI :("); + break; default: logger(LOG_DEBUG, "OpenLI: unrecognised receive net buffer error %d.", err); diff --git a/src/netcomms.h b/src/netcomms.h index 44fbc3dc..c5b91671 100644 --- a/src/netcomms.h +++ b/src/netcomms.h @@ -127,6 +127,7 @@ typedef enum { OPENLI_PROTO_WITHDRAW_EMAIL_TARGET, OPENLI_PROTO_ANNOUNCE_DEFAULT_EMAIL_COMPRESSION, OPENLI_PROTO_RAWIP_CC, + OPENLI_PROTO_RAWIP_IRI, } openli_proto_msgtype_t; typedef struct net_buffer { @@ -180,6 +181,9 @@ typedef enum { OPENLI_PROTO_FIELD_PAYLOAD_ENCRYPTION, OPENLI_PROTO_FIELD_ENCRYPTION_KEY, OPENLI_PROTO_FIELD_DELIVER_COMPRESSED, + OPENLI_PROTO_FIELD_MOBILEIDENT, + OPENLI_PROTO_FIELD_CORESERVER_UPPER_PORT, + OPENLI_PROTO_FIELD_CORESERVER_LOWER_PORT, } openli_proto_fieldtype_t; net_buffer_t *create_net_buffer(net_buffer_type_t buftype, int fd, SSL *ssl); diff --git a/src/provisioner/clientupdates.c b/src/provisioner/clientupdates.c index b9773557..6edbb4cb 100644 --- a/src/provisioner/clientupdates.c +++ b/src/provisioner/clientupdates.c @@ -405,8 +405,7 @@ int announce_hi1_notification_to_mediators(provision_state_t *state, if (ceptdata->start_hi1_sent) { return 0; } - if (tv.tv_sec >= 0 && intcomm->tostart_time > - (unsigned long)tv.tv_sec) { + if (tv.tv_sec >= 0 && intcomm->tostart_time > tv.tv_sec) { return 0; } ceptdata->start_hi1_sent = 1; @@ -425,8 +424,7 @@ int announce_hi1_notification_to_mediators(provision_state_t *state, return 0; } if (!ceptdata->start_hi1_sent) { - if (tv.tv_sec >= 0 && intcomm->tostart_time > - (unsigned long)tv.tv_sec) { + if (tv.tv_sec >= 0 && intcomm->tostart_time > tv.tv_sec) { return 0; } else { /* shouldn't get here ideally, but just in case we do then diff --git a/src/provisioner/configwriter.c b/src/provisioner/configwriter.c index af8fcbf2..a7038620 100644 --- a/src/provisioner/configwriter.c +++ b/src/provisioner/configwriter.c @@ -131,15 +131,49 @@ static int emit_core_server_list(coreserver_t *servers, const char *label, YAML_PLAIN_SCALAR_STYLE); if (!yaml_emitter_emit(emitter, &event)) return -1; - yaml_scalar_event_initialize(&event, NULL, (yaml_char_t *)YAML_STR_TAG, - (yaml_char_t *)"port", strlen("port"), 1, 0, - YAML_PLAIN_SCALAR_STYLE); - if (!yaml_emitter_emit(emitter, &event)) return -1; + if (cs->portstr) { + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"port", strlen("port"), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; - yaml_scalar_event_initialize(&event, NULL, (yaml_char_t *)YAML_STR_TAG, - (yaml_char_t *)cs->portstr, strlen(cs->portstr), 1, 0, - YAML_PLAIN_SCALAR_STYLE); - if (!yaml_emitter_emit(emitter, &event)) return -1; + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)cs->portstr, strlen(cs->portstr), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + } + + if (cs->lower_portstr) { + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"port_lower", strlen("port_lower"), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)cs->lower_portstr, + strlen(cs->lower_portstr), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + } + + if (cs->upper_portstr) { + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"port_upper", strlen("port_upper"), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)cs->upper_portstr, + strlen(cs->upper_portstr), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + } yaml_mapping_end_event_initialize(&event); if (!yaml_emitter_emit(emitter, &event)) return -1; @@ -659,6 +693,23 @@ static int emit_ipintercepts(ipintercept_t *ipints, yaml_emitter_t *emitter) { if (!yaml_emitter_emit(emitter, &event)) return -1; } + if (ipint->mobileident != OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED) { + const char *mobtype = NULL; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"mobileident", strlen("mobileident"), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + mobtype = get_mobile_identifier_string(ipint->mobileident); + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)mobtype, strlen(mobtype), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + if (!yaml_emitter_emit(emitter, &event)) return -1; + } + if (ipint->options & (1<options & (1<accesstype == INTERNET_ACCESS_TYPE_MOBILE && + b->accesstype == INTERNET_ACCESS_TYPE_MOBILE && + a->mobileident != b->mobileident) { + return 0; + } + return 1; } diff --git a/src/provisioner/updateserver_jsoncreation.c b/src/provisioner/updateserver_jsoncreation.c index cc24723f..3e1ea5b1 100644 --- a/src/provisioner/updateserver_jsoncreation.c +++ b/src/provisioner/updateserver_jsoncreation.c @@ -122,7 +122,7 @@ static void convert_commonintercept_to_json(json_object *jobj, static json_object *convert_ipintercept_to_json(ipintercept_t *ipint) { json_object *jobj; json_object *vendmirrorid, *user, *accesstype, *radiusident; - json_object *staticips; + json_object *staticips, *mobileident; jobj = json_object_new_object(); convert_commonintercept_to_json(jobj, &(ipint->common)); @@ -135,7 +135,12 @@ static json_object *convert_ipintercept_to_json(ipintercept_t *ipint) { json_object_object_add(jobj, "user", user); json_object_object_add(jobj, "accesstype", accesstype); - json_object_object_add(jobj, "radiusident", radiusident); + + if (ipint->mobileident != OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED) { + mobileident = json_object_new_string( + get_mobile_identifier_string(ipint->mobileident)); + json_object_object_add(jobj, "mobileident", mobileident); + } if (ipint->vendmirrorid != 0xFFFFFFFF) { vendmirrorid = json_object_new_int(ipint->vendmirrorid); @@ -240,15 +245,29 @@ static json_object *convert_voipintercept_to_json(voipintercept_t *vint) { static json_object *convert_coreserver_to_json(coreserver_t *cs) { json_object *jobj; - json_object *ipaddr, *port; + json_object *ipaddr, *port, *upper_port, *lower_port; + + port = upper_port = lower_port = NULL; jobj = json_object_new_object(); ipaddr = json_object_new_string(cs->ipstr); - port = json_object_new_string(cs->portstr); json_object_object_add(jobj, "ipaddress", ipaddr); - json_object_object_add(jobj, "port", port); + if (cs->upper_portstr) { + upper_port = json_object_new_string(cs->upper_portstr); + json_object_object_add(jobj, "port_upper", upper_port); + } + + if (cs->lower_portstr) { + lower_port = json_object_new_string(cs->lower_portstr); + json_object_object_add(jobj, "port_lower", lower_port); + } + + if (cs->portstr) { + port = json_object_new_string(cs->portstr); + json_object_object_add(jobj, "port", port); + } return jobj; } diff --git a/src/provisioner/updateserver_jsonparsing.c b/src/provisioner/updateserver_jsonparsing.c index fc73d429..486fa2fa 100644 --- a/src/provisioner/updateserver_jsonparsing.c +++ b/src/provisioner/updateserver_jsonparsing.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "provisioner.h" #include "updateserver.h" @@ -54,6 +55,7 @@ struct json_intercept { struct json_object *accesstype; struct json_object *user; struct json_object *radiusident; + struct json_object *mobileident; struct json_object *vendmirrorid; struct json_object *starttime; struct json_object *endtime; @@ -201,6 +203,7 @@ static inline void extract_intercept_json_objects( json_object_object_get_ex(parsed, "user", &(ipjson->user)); json_object_object_get_ex(parsed, "accesstype", &(ipjson->accesstype)); json_object_object_get_ex(parsed, "radiusident", &(ipjson->radiusident)); + json_object_object_get_ex(parsed, "mobileident", &(ipjson->mobileident)); json_object_object_get_ex(parsed, "starttime", &(ipjson->starttime)); json_object_object_get_ex(parsed, "endtime", &(ipjson->endtime)); json_object_object_get_ex(parsed, "outputhandovers", &(ipjson->tomediate)); @@ -218,21 +221,20 @@ static inline int compare_intercept_times(intercept_common_t *latest, int changed = 0; - if (latest->tostart_time == (uint64_t)-1 && - latest->toend_time == (uint64_t)-1) { + if (latest->tostart_time == -1 && latest->toend_time == -1) { /* No new times were provided in the JSON object */ return 0; } - if (latest->tostart_time != (uint64_t)-1) { + if (latest->tostart_time != -1) { if (latest->tostart_time != current->tostart_time) { current->tostart_time = latest->tostart_time; changed = 1; } } - if (latest->toend_time != (uint64_t)-1) { + if (latest->toend_time != -1) { if (latest->toend_time != current->toend_time) { current->toend_time = latest->toend_time; changed = 1; @@ -359,7 +361,7 @@ static int parse_intercept_common_json(struct json_intercept *jsonp, } if (common->tostart_time > 0 && tv.tv_sec >= 0 && - common->tostart_time > (unsigned long)tv.tv_sec) { + common->tostart_time > tv.tv_sec) { if (add_intercept_timer(epoll_fd, common->tostart_time, tv.tv_sec, timers, PROV_EPOLL_INTERCEPT_START) < 0) { snprintf(cinfo->answerstring, 4096, "unable to create a 'intercept start' timer for intercept %s", common->liid); @@ -368,7 +370,7 @@ static int parse_intercept_common_json(struct json_intercept *jsonp, } if (common->toend_time > 0 && tv.tv_sec >= 0 && - common->toend_time > (unsigned long)tv.tv_sec) { + common->toend_time > tv.tv_sec) { if (add_intercept_timer(epoll_fd, common->toend_time, tv.tv_sec, timers, PROV_EPOLL_INTERCEPT_HALT) < 0) { snprintf(cinfo->answerstring, 4096, "unable to create a 'intercept end' timer for intercept %s", common->liid); @@ -574,11 +576,42 @@ int remove_coreserver(update_con_info_t *cinfo UNUSED, provision_state_t *state, const char *idstr, uint8_t srvtype) { char search[1024]; + char addendum[64]; coreserver_t *found = NULL; coreserver_t **src; + char *tok, *saved, *copy; - snprintf(search, 1024, "%s-%s", idstr, coreserver_type_to_string(srvtype)); + copy = strdup(idstr); + /* check for case where the user has provided only one port in the key */ + tok = strtok(copy, "-"); + if (!tok) { + logger(LOG_INFO, + "OpenLI: unable to remove %s server %s via update socket.", + coreserver_type_to_string(srvtype), idstr); + free(copy); + return 0; + } + tok = strtok(NULL, "-"); + if (!tok) { + logger(LOG_INFO, + "OpenLI: unable to remove %s server %s via update socket.", + coreserver_type_to_string(srvtype), idstr); + free(copy); + return 0; + } + saved = tok; + + tok = strtok(NULL, "-"); + if (tok == NULL) { + snprintf(addendum, 64, "-%s", saved); + } else { + addendum[0] = '\0'; + } + + snprintf(search, 1024, "%s%s-%s", idstr, addendum, + coreserver_type_to_string(srvtype)); + free(copy); if (srvtype == OPENLI_CORE_SERVER_SIP) { HASH_FIND(hh, state->interceptconf.sipservers, search, strlen(search), found); @@ -701,6 +734,8 @@ int add_new_coreserver(update_con_info_t *cinfo, provision_state_t *state, coreserver_t *new_cs = NULL; struct json_object *ipaddr; struct json_object *port; + struct json_object *upper_port; + struct json_object *lower_port; char srvstring[1024]; @@ -726,6 +761,8 @@ int add_new_coreserver(update_con_info_t *cinfo, provision_state_t *state, json_object_object_get_ex(parsed, "ipaddress", &(ipaddr)); json_object_object_get_ex(parsed, "port", &(port)); + json_object_object_get_ex(parsed, "port_upper", &(upper_port)); + json_object_object_get_ex(parsed, "port_lower", &(lower_port)); snprintf(srvstring, 1024, "%s server", coreserver_type_to_string(srvtype)); @@ -733,7 +770,11 @@ int add_new_coreserver(update_con_info_t *cinfo, provision_state_t *state, EXTRACT_JSON_STRING_PARAM("ipaddress", srvstring, ipaddr, new_cs->ipstr, &parseerr, true); EXTRACT_JSON_STRING_PARAM("port", srvstring, port, - new_cs->portstr, &parseerr, true); + new_cs->portstr, &parseerr, false); + EXTRACT_JSON_STRING_PARAM("port_upper", srvstring, upper_port, + new_cs->upper_portstr, &parseerr, false); + EXTRACT_JSON_STRING_PARAM("port_lower", srvstring, lower_port, + new_cs->lower_portstr, &parseerr, false); if (parseerr) { goto cserr; @@ -795,8 +836,8 @@ int add_new_coreserver(update_con_info_t *cinfo, provision_state_t *state, } announce_coreserver_change(state, new_cs, true); - logger(LOG_INFO, "OpenLI: added %s '%s:%s' via update socket.", - srvstring, new_cs->ipstr, new_cs->portstr); + logger(LOG_INFO, "OpenLI: added %s '%s' via update socket.", + srvstring, new_cs->serverkey); } if (parsed) { @@ -1271,6 +1312,7 @@ int add_new_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { ipintercept_t *found = NULL; int parseerr = 0; char *accessstring = NULL; + char *mobileidentstring = NULL; char *radiusidentstring = NULL; ipintercept_t *ipint = NULL; prov_intercept_data_t *timers = NULL; @@ -1282,6 +1324,7 @@ int add_new_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { ipint->awaitingconfirm = 1; ipint->vendmirrorid = OPENLI_VENDOR_MIRROR_NONE; ipint->accesstype = INTERNET_ACCESS_TYPE_UNDEFINED; + ipint->mobileident = OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; ipint->options = 0; if (parse_intercept_common_json(&ipjson, &(ipint->common), @@ -1300,6 +1343,8 @@ int add_new_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { accessstring, &parseerr, false); EXTRACT_JSON_STRING_PARAM("radiusident", "IP intercept", ipjson.radiusident, radiusidentstring, &parseerr, false); + EXTRACT_JSON_STRING_PARAM("mobileident", "IP intercept", ipjson.mobileident, + mobileidentstring, &parseerr, false); if (parseerr) { goto cepterr; @@ -1332,6 +1377,16 @@ int add_new_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { (1 << OPENLI_IPINT_OPTION_RADIUS_IDENT_USER); } + if (ipint->accesstype != INTERNET_ACCESS_TYPE_MOBILE) { + ipint->mobileident = OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; + } else { + ipint->mobileident = map_mobile_ident_string(mobileidentstring); + } + if (mobileidentstring) { + free(mobileidentstring); + mobileidentstring = NULL; + } + HASH_FIND(hh_liid, state->interceptconf.ipintercepts, ipint->common.liid, ipint->common.liid_len, found); @@ -1385,6 +1440,9 @@ int add_new_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { if (radiusidentstring) { free(radiusidentstring); } + if (mobileidentstring) { + free(mobileidentstring); + } if (parsed) { json_object_put(parsed); } @@ -1728,6 +1786,7 @@ int modify_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { char *liidstr = NULL; char *accessstring = NULL; char *radiusidentstring = NULL; + char *mobileidentstring = NULL; int parseerr = 0, changed = 0, agencychanged = 0; int timeschanged = 0; @@ -1775,6 +1834,8 @@ int modify_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { accessstring, &parseerr, false); EXTRACT_JSON_STRING_PARAM("radiusident", "IP intercept", ipjson.radiusident, radiusidentstring, &parseerr, false); + EXTRACT_JSON_STRING_PARAM("mobileident", "IP intercept", ipjson.mobileident, + mobileidentstring, &parseerr, false); EXTRACT_JSON_INT_PARAM("vendmirrorid", "IP intercept", ipjson.vendmirrorid, ipint->vendmirrorid, &parseerr, false); @@ -1836,6 +1897,11 @@ int modify_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { (1 << OPENLI_IPINT_OPTION_RADIUS_IDENT_CSID); } + if (ipint->accesstype != INTERNET_ACCESS_TYPE_MOBILE) { + ipint->mobileident = OPENLI_MOBILE_IDENTIFIER_NOT_SPECIFIED; + } else { + ipint->mobileident = map_mobile_ident_string(mobileidentstring); + } /* TODO: warn if user tries to change fields that we don't support * changing (e.g. mediator) ? * @@ -1843,6 +1909,11 @@ int modify_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { MODIFY_STRING_MEMBER(ipint->username, found->username, &changed); found->username_len = strlen(found->username); + if (mobileidentstring && ipint->mobileident != found->mobileident) { + changed = 1; + found->mobileident = ipint->mobileident; + } + if (accessstring && ipint->accesstype != found->accesstype) { changed = 1; found->accesstype = ipint->accesstype; @@ -1886,6 +1957,9 @@ int modify_ipintercept(update_con_info_t *cinfo, provision_state_t *state) { if (radiusidentstring) { free(radiusidentstring); } + if (mobileidentstring) { + free(mobileidentstring); + } if (ipint) { free_single_ipintercept(ipint); diff --git a/src/util.c b/src/util.c index ccf7f25a..955454b4 100644 --- a/src/util.c +++ b/src/util.c @@ -627,5 +627,42 @@ char *extract_liid_from_exported_msg(uint8_t *etsimsg, return (char *)space; } +libtrace_packet_t *openli_copy_packet(libtrace_packet_t *pkt) { + libtrace_packet_t *copy; + int caplen = trace_get_capture_length(pkt); + int framelen = trace_get_framing_length(pkt); + + if (caplen == -1 || framelen == -1) { + return NULL; + } + + copy = (libtrace_packet_t *)calloc((size_t)1, sizeof(libtrace_packet_t)); + if (!copy) { + logger(LOG_INFO, "OpenLI: out of memory while copying libtrace packet"); + exit(1); + } + + copy->trace = pkt->trace; + copy->buf_control = TRACE_CTRL_PACKET; + copy->buffer = malloc(framelen + caplen); + copy->type = pkt->type; + copy->header = copy->buffer; + copy->payload = ((char *)copy->buffer) + framelen; + copy->order = pkt->order; + copy->hash = pkt->hash; + copy->error = pkt->error; + copy->which_trace_start = pkt->which_trace_start; + copy->cached.capture_length = caplen; + copy->cached.framing_length = framelen; + copy->cached.wire_length = -1; + copy->cached.payload_length = -1; + /* everything else in cache should be 0 or NULL due to our earlier + * calloc() */ + memcpy(copy->header, pkt->header, framelen); + memcpy(copy->payload, pkt->payload, caplen); + + return copy; +} + // vim: set sw=4 tabstop=4 softtabstop=4 expandtab : diff --git a/src/util.h b/src/util.h index 38b4b825..7744a3bb 100644 --- a/src/util.h +++ b/src/util.h @@ -66,6 +66,7 @@ void openli_copy_ipcontent(libtrace_packet_t *pkt, uint8_t **ipc, char *extract_liid_from_exported_msg(uint8_t *etsimsg, uint64_t msglen, unsigned char *space, int maxspace, uint16_t *liidlen); +libtrace_packet_t *openli_copy_packet(libtrace_packet_t *pkt); /* string set methods */ int remove_from_string_set(string_set_t **set, char *term);