Merge branch 'develop'

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [email protected].nz. All
+reported by contacting the project team at hello@openli.nz. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

README.md

@@ -1,6 +1,6 @@
 OpenLI -- open source ETSI-compliant Lawful Intercept software
 
-Version: 1.1.1
+Version: 1.1.2
 
 ---------------------------------------------------------------------------
 
@@ -50,7 +50,7 @@ obligations to assist law enforcement to prevent criminal or terrorist activity.
 Any use of this software to assist with the violation of human rights or the
 oppression of a populace is forbidden. If you are unsure as to whether your use
 of this software may violate these conditions, please contact us as
-<[email protected].nz> and explain your situation to receive our advice
+<hello@openli.nz> and explain your situation to receive our advice
 on whether you may use OpenLI or not.
 
 ## Software Packages
@@ -73,7 +73,7 @@ API, DPDK with OpenLI, etc.). The wiki tends to be updated more often than
 the in-code documentation (e.g. the `doc/` directory) as well.
 
 If there is content that you would like to contribute to the OpenLI wiki,
-please feel free to reach out to us at <[email protected].nz> and we
+please feel free to reach out to us at <hello@openli.nz> and we
 will be more than happy to accept your contribution.
 
 ## Dependencies for building from source
@@ -297,7 +297,7 @@ A. Unfortunately there are plenty of reasons why this might happen. Here are
 * Try installing the latest 'develop' branch of openli itself from
   https://github.com/OpenLI-NZ/openli
 
-  If all else fails, send us an email at openli-support@waikato.ac.nz and
+  If all else fails, send us an email at support@openli.nz and
   someone will try to help you.
 
 ---

configure.ac

@@ -1,6 +1,6 @@
 # Super primitive configure script
 
-AC_INIT([openli],[1.1.1],[[email protected]])
+AC_INIT([openli],[1.1.2],[[email protected]])
 
 AM_INIT_AUTOMAKE([subdir-objects])
 AC_CONFIG_SRCDIR(src/collector/collector.c)
@@ -47,6 +47,7 @@ AC_CHECK_LIB([ssl], [OPENSSL_init_ssl],libssl11_found=1,)
 AC_CHECK_LIB([crypto], [ERR_load_crypto_strings],libcrypto_found=1,)
 AC_CHECK_LIB([crypto], [EVP_CIPHER_CTX_new],libcrypto_found=1,)
 AC_CHECK_LIB([rabbitmq], [amqp_new_connection],rabbit_found=1,rabbit_found=0)
+AC_CHECK_LIB([Judy], [JudySLGet],,libjudy_found=0)
 
 if test "x$libzmq_found" = "x1"; then
         COLLECTOR_LIBS="$COLLECTOR_LIBS -lzmq"
@@ -70,6 +71,10 @@ AS_IF([test "x$uthash_avail" != "xyes"],
 
 if test "x$enable_collector" != "xno"; then
         AC_CHECK_LIB([tcmalloc], [tc_version],,libtcmalloc_found=0)
+        AC_CHECK_LIB([z], [inflateEnd],,libz_found=0)
+        if test "$libz_found" = 0; then
+                AC_MSG_ERROR(Required library libz not found; use LDFLAGS to specify library location)
+        fi
         AC_CHECK_LIB([osipparser2], [osip_message_init],libosip2_found=1,libosip2_found=0)
         if test "$libosip2_found" = 0; then
                 AC_MSG_ERROR(Required library libosipparser2 not found; use LDFLAGS to specify library location)
@@ -78,7 +83,7 @@ if test "x$enable_collector" != "xno"; then
         if test "$libb64_found" = 0; then
                 AC_MSG_ERROR(Required library libb64 not found; use LDFLAGS to specify library location)
         fi
-        COLLECTOR_LIBS="$COLLECTOR_LIBS -losipparser2 -lb64"
+        COLLECTOR_LIBS="$COLLECTOR_LIBS -losipparser2 -lb64 -lz"
 fi
 
 if test "x$enable_provisioner" != "xno" -o "x$enable_collector" != "xno"; then
@@ -120,13 +125,9 @@ if test "x$enable_collector" != "xno" -o "x$enable_mediator" != "xno"; then
                 AC_MSG_ERROR(Required library libwandder 2.0.6 or later not found; use LDFLAGS to specify library location)
         fi
 
-        AC_CHECK_LIB([Judy], [JudySLGet],libjudy_found=1,libjudy_found=0)
-        if test "$libjudy_found" = 0; then
-                AC_MSG_ERROR(Required library libjudy not found; use LDFLAGS to specify library location)
-        fi
 
-        COLLECTOR_LIBS="$COLLECTOR_LIBS -lJudy -lwandder"
-        MEDIATOR_LIBS="$MEDIATOR_LIBS -lJudy -lwandder"
+        COLLECTOR_LIBS="$COLLECTOR_LIBS -lwandder"
+        MEDIATOR_LIBS="$MEDIATOR_LIBS -lwandder"
 fi
 
 if test "$libtrace_found" = 0; then
@@ -149,6 +150,9 @@ else
         PROVISIONER_LIBS="$PROVISIONER_LIBS -lcrypto"
 fi
 
+if test "$libjudy_found" = 0; then
+        AC_MSG_ERROR(Required library libjudy not found; use LDFLAGS to specify library location)
+fi
 
 if test "$rabbit_found" != 1; then
     AC_MSG_ERROR(Required library librabbitmq not found; use LDFLAGS to specify library location)

debian/changelog

@@ -1,3 +1,37 @@
+openli (1.1.2-1) unstable; urgency=medium
+
+  * Email ingestion: add support for PART_ID field and attempt
+    to reorder received messages based on PART_ID value.
+  * Email ingestion: fix segfault that can occur if an incomplete
+    multi-part message is received by the ingestion socket.
+  * Email ingestion: fix parsing failure caused by packet segmentation.
+  * IMAP: fix interception failures when the COMPRESS extension
+    is used by a session.
+  * REST API: fix segfaults when using PUT to modify existing
+    intercepts / agencies.
+  * IMAP: fix bad regex that was causing problems when intercepting
+    mail content containing unbalanced parentheses.
+  * REST API: added new openliversion/ endpoint which returns the
+    version of the provisioner software.
+  * SMTP: emit all SMTP messages and replies as CCs, not just the
+    DATA message.
+  * SMTP: produce "email-receive" IRI if a recipient is an intercept
+    target, i.e. if the email comes from an external mail provider.
+  * SMTP: if present, use AUTH to derive the sender identity. Include
+    sender validity info in IRIs.
+  * SMTP: improve handling of unexpected or bogus commands, so
+    now interception won't cease if we see a command we don't understand.
+  * SIP: don't crash if an authorization field has an empty username.
+  * SIP: fix issue where RTP would not be intercepted if the "c="
+    field in the SDP announcement is after the first "m=" field.
+  * SIP: CANCEL will now correctly move an intercepted call into the
+    end state.
+  * HI1 Operations: send HI1 operations messages for intercepts with
+    specific start and end times at the expected time, not when the
+    intercept is configured.
+
+ -- Shane Alcock <[email protected]>  Tue, 10 Oct 2023 16:19:41 +1300
+
 openli (1.1.1-1) unstable; urgency=medium
 
   * Add ability to encrypt CC and IRI payload, as per Annex G of

debian/control

@@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 12), dh-autoreconf, dh-systemd (>=1.5),
  libtrace4-dev (>= 4.0.19), libyaml-dev, uthash-dev, libwandder2-dev (>=2.0.4),
  libjudy-dev, libzmq3-dev, libgoogle-perftools-dev, libosip2-dev,
  libssl1.0-dev (>=1.0.2r) | libssl-dev, librabbitmq-dev, libb64-dev,
- libmicrohttpd-dev, libjson-c-dev, libsqlcipher-dev
+ libmicrohttpd-dev, libjson-c-dev, libsqlcipher-dev, zlib1g-dev
 Standards-Version: 4.1.3
 Homepage: https://openli.nz
 

doc/CollectorDoc.md

@@ -215,6 +215,10 @@ The basic option keys are:
                        credentials in IMAP traffic that have been
                        intercepted using an email intercept with "XXX".
                        Defaults to "yes".
+* defaultemaildomain -- during email interception, any authenticated email
+                        users that do not explicitly include their domain
+                        in their username will be assumed to be using the
+                        address'<username>@<this domain>'.
 
 Be aware that increasing the number of threads used for sequence number
 tracking, encoding or forwarding can actually decrease OpenLI's performance,

doc/ProvisionerDoc.md

@@ -535,6 +535,18 @@ An email target is a JSON object that contains just a single field:
 
 * `address`               -- the email address of the target
 
+Optional key-value elements for an email intercept are:
+
+* `delivercompressed`     -- if email content is compressed (e.g. via the
+                             IMAP COMPRESS extension), should OpenLI create
+                             CC records using the compressed or decompressed
+                             version of the content? Set to `as-is` to emit
+                             CC records with compressed content, or
+                             `decompressed` to emit CC records using
+                             decompressed content. If not set, the approach
+                             described by the `email-defaultdelivercompressed`
+                             option will be used.
+
 
 ---
 
@@ -567,6 +579,22 @@ All intercept types also support the following optional key-value elements:
                              ideal key length is 24 characters. Shorter keys
                              will be padded with null bytes, longer keys will be
                              truncated to 24 characters.
+---
+
+The default approach for delivering compressed email content to the agencies
+can be set by adding a key-value pair to the top level of the intercept
+configuration. The key should be `email-defaultdelivercompressed` and the value
+should be either `as-is` (to deliver compressed content in its original
+compressed form) or `decompressed` (to deliver the decompressed version of the
+content instead).
+
+The approach described in the `email-defaultdelivercompressed` option will be
+applied to all email intercepts, but can be overridden for specific
+email intercepts by including the `delivercompressed` config option in the
+individual intercept configuration.
+
+If `email-defaultdelivercompressed` is not set, `as-is` will be used as the
+default approach for handling compressed email content.
 
 
 ### SIP Target Specifics

doc/exampleconfigs/collector-example.yaml

@@ -45,6 +45,12 @@ maskimapcreds: yes
 # POP3 traffic captured using an email intercept, not an IP intercept.
 maskpop3creds: yes
 
+# Email users who supply a domain-less username during authentication will
+# be assumed to have an email address of '<username>@<defaultemaildomain>'.
+# Obviously, you would want to replace this config option with the domain
+# that your email addresses are assigned to.
+defaultemaildomain: example.org
+
 # Number of minutes between each dump of performance statistics to the
 # logger. Set to zero to disable this extra logging altogether.
 logstatfrequency: 5

doc/exampleconfigs/running-intercept-example.yaml

@@ -248,3 +248,21 @@ emailintercepts:
    targets:
      - address: [email protected]          # email address for the target
      - address: [email protected]             # multiple addresses are allowed
+   delivercompressed: decompressed      # If content is compressed, send the
+                                        # decompressed version to the agency.
+                                        # Other valid options are `as-is` (i.e.
+                                        # send the compressed version) or
+                                        # `default` (i.e. do whatever the
+                                        # `email-defaultdelivercompressed` below
+                                        # says). If this option is not set,
+                                        # the default is `default`.
+
+
+# Setting this option to `as-is` will cause any intercepted email content
+# that is compressed to be sent to the agencies in its original compressed
+# form. Setting this option to `decompressed` will instead have
+# OpenLI decompress the message contents before encoding it into CC records.
+# Individual intercepts can override this default by including a corresponding
+# `delivercompressed` option in the intercept configuration (see above).
+# Defaults to `as-is` if not set.
+email-defaultdelivercompressed: as-is

rpm/openli.spec

@@ -1,5 +1,5 @@
 Name:           openli
-Version:        1.1.1
+Version:        1.1.2
 Release:        1%{?dist}
 Summary:        Software for performing ETSI-compliant lawful intercept
 
@@ -28,6 +28,7 @@ BuildRequires: systemd
 BuildRequires: sqlcipher-devel
 BuildRequires: librabbitmq-devel
 BuildRequires: libb64-devel
+BuildRequires: zlib-devel
 
 %description
 Software for performing ETSI-compliant lawful intercept
@@ -282,6 +283,9 @@ fi
 
 
 %changelog
+* Tue Oct 10 2023 Shane Alcock <[email protected]> - 1.1.2-1
+- Updated for 1.1.2 release
+
 * Mon Jul 31 2023 Shane Alcock <[email protected]> - 1.1.1-1
 - Updated for 1.1.1 release
 

src/Makefile.am

@@ -15,7 +15,8 @@ openliprovisioner_SOURCES=provisioner/provisioner.c provisioner/provisioner.h \
                 provisioner/updateserver.h \
                 provisioner/updateserver_jsonparsing.c \
                 provisioner/updateserver_jsoncreation.c \
-                provisioner/hup_reload.c
+                provisioner/hup_reload.c \
+                provisioner/intercept_timers.c provisioner/intercept_timers.h
 
 openliprovisioner_LDFLAGS = -lpthread @PROVISIONER_LIBS@
 openliprovisioner_LDADD = @ADD_LIBS@

src/collector/collector.c

@@ -1228,6 +1228,10 @@ static void destroy_collector_state(collector_global_t *glob) {
         free(glob->collocals);
     }
 
+    if (glob->default_email_domain) {
+        free(glob->default_email_domain);
+    }
+
     pthread_mutex_destroy(&(glob->stats_mutex));
     pthread_rwlock_destroy(&(glob->email_config_mutex));
     pthread_rwlock_destroy(&glob->config_mutex);
@@ -1455,7 +1459,7 @@ static void init_collector_global(collector_global_t *glob) {
     glob->email_timeouts.imap = 30;
     glob->mask_imap_creds = 1;      // defaults to "enabled"
     glob->mask_pop3_creds = 1;      // defaults to "enabled"
-
+    glob->default_email_domain = NULL;
 }
 
 static collector_global_t *parse_global_config(char *configfile) {
@@ -1527,6 +1531,11 @@ static collector_global_t *parse_global_config(char *configfile) {
         logger(LOG_INFO, "Email interception: rewriting POP3 plain text passwords to avoid leaking passwords to agencies");
     }
 
+    if (glob->default_email_domain) {
+        logger(LOG_INFO, "Using '%s' as the default email domain",
+                glob->default_email_domain);
+    }
+
     logger(LOG_DEBUG, "OpenLI: session idle timeout for SMTP sessions: %u minutes", glob->email_timeouts.smtp);
     logger(LOG_DEBUG, "OpenLI: session idle timeout for IMAP sessions: %u minutes", glob->email_timeouts.imap);
     logger(LOG_DEBUG, "OpenLI: session idle timeout for POP3 sessions: %u minutes", glob->email_timeouts.pop3);
@@ -1669,6 +1678,28 @@ static int reload_collector_config(collector_global_t *glob,
         }
     }
 
+    if (glob->default_email_domain) {
+        if (!newstate.default_email_domain) {
+            logger(LOG_INFO, "OpenLI: default email domain has been unset.");
+            free(glob->default_email_domain);
+            glob->default_email_domain = NULL;
+        } else if (strcmp(glob->default_email_domain,
+                newstate.default_email_domain) != 0) {
+            logger(LOG_INFO,
+                    "OpenLI: changing default email domain from '%s' to '%s'",
+                    glob->default_email_domain, newstate.default_email_domain);
+            free(glob->default_email_domain);
+            glob->default_email_domain = newstate.default_email_domain;
+            newstate.default_email_domain = NULL;
+        }
+    } else if (newstate.default_email_domain) {
+        logger(LOG_INFO,
+                "OpenLI: setting default email domain to be '%s'",
+                newstate.default_email_domain);
+        glob->default_email_domain = newstate.default_email_domain;
+        newstate.default_email_domain = NULL;
+    }
+
     glob->mask_imap_creds = newstate.mask_imap_creds;
     glob->mask_pop3_creds = newstate.mask_pop3_creds;
     glob->email_timeouts.smtp = newstate.email_timeouts.smtp;
@@ -1949,7 +1980,10 @@ int main(int argc, char *argv[]) {
         glob->emailworkers[i].glob_config_mutex = &(glob->email_config_mutex);
         glob->emailworkers[i].mask_imap_creds = &(glob->mask_imap_creds);
         glob->emailworkers[i].mask_pop3_creds = &(glob->mask_pop3_creds);
+        glob->emailworkers[i].defaultdomain = &(glob->default_email_domain);
         glob->emailworkers[i].timeout_thresholds = &(glob->email_timeouts);
+        glob->emailworkers[i].default_compress_delivery =
+                OPENLI_EMAILINT_DELIVER_COMPRESSED_ASIS;
 
         pthread_create(&(glob->emailworkers[i].threadid), NULL,
                 start_email_worker_thread, (void *)&(glob->emailworkers[i]));

src/collector/collector.h

@@ -315,7 +315,7 @@ typedef struct collector_global {
     openli_email_timeouts_t email_timeouts;
     uint8_t mask_imap_creds;
     uint8_t mask_pop3_creds;
-
+    char *default_email_domain;
     int emailsockfd;
     email_ingestor_state_t *email_ingestor;
 

src/collector/collector_sync.c

@@ -1025,6 +1025,8 @@ static int new_mediator(collector_sync_t *sync, uint8_t *provmsg,
         return -1;
     }
 
+    logger(LOG_INFO, "OpenLI: new mediator announcement for %s:%s",
+            med.ipstr, med.portstr);
     for (i = 0; i < sync->forwardcount; i++) {
         expmsg = (openli_export_recv_t *)calloc(1,
                 sizeof(openli_export_recv_t));
@@ -1036,8 +1038,6 @@ static int new_mediator(collector_sync_t *sync, uint8_t *provmsg,
         publish_openli_msg(sync->zmq_fwdctrlsocks[i], expmsg);
     }
 
-    logger(LOG_INFO, "OpenLI: new mediator announcement for %s:%s",
-            med.ipstr, med.portstr);
     return 1;
 }
 
@@ -1832,6 +1832,7 @@ static int recv_from_provisioner(collector_sync_t *sync) {
             case OPENLI_PROTO_MODIFY_EMAILINTERCEPT:
             case OPENLI_PROTO_ANNOUNCE_EMAIL_TARGET:
             case OPENLI_PROTO_WITHDRAW_EMAIL_TARGET:
+            case OPENLI_PROTO_ANNOUNCE_DEFAULT_EMAIL_COMPRESSION:
                 ret = forward_provmsg_to_email_workers(sync, provmsg, msglen,
                         msgtype);
                 if (ret == -1) {