auto jinja filter; syslog-ng fixes; typo affecting the Observe functi…

…onality; expose websockets IP address improvement

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change Log
 
+## [1.5.6] - 2024-09-01
+
+### Added
+- The `auto jinja filter` option under `features`.
+
+### Fixed
+- Adapted `syslog-ng` to Ubuntu 24.04.
+
 ## [1.5.5] - 2024-08-20
 
 ### Fixed

Docker/smart-multi-line.fsm

@@ -0,0 +1,30 @@
+#
+# Copyright 2023 Balazs Scheidler
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# The regular expressions were extracted from
+# https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions
+# and converted into a TSV format by Balazs Scheidler.
+#
+# List of tab separated fields
+#
+# comma-separated-states /regexp/ new_state
+#
+
+# python
+start_state	/^Traceback \(most recent call last\):$/	python
+python	/^[\t ]*File /	python_code
+python_code	/[^\t ]/	python
+python	/^(?:[^\s.():]+\.)*[^\s.():]+:/	start_state

Docker/syslog-ng-docker.conf

@@ -1,12 +1,12 @@
-@version: 3.35
+@version: 4.3
 @include "scl.conf"
 
 # Syslog-ng configuration file, compatible with default Debian syslogd
 # installation.
 
 # First, set some global options.
 options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
-	  owner("root"); group("adm"); perm(0640); stats_freq(0);
+	  owner("root"); group("adm"); perm(0640); stats(freq(0));
 	  bad_hostname("^gconfd$"); dns_cache(no);
 };
 
@@ -34,6 +34,7 @@ source s_src {
 destination d_auth { file("/var/log/auth.log"); };
 destination d_cron { file("/var/log/cron.log"); };
 destination d_daemon { file("/var/log/daemon.log"); };
+destination d_kern { file("/var/log/kern.log"); };
 destination d_lpr { file("/var/log/lpr.log"); };
 destination d_mail { file("/var/log/mail.log"); };
 destination d_syslog { file("/var/log/syslog"); };
@@ -71,7 +72,7 @@ destination d_console { usertty("root"); };
 #
 #    $ xconsole -file /dev/xconsole [...]
 #
-destination d_xconsole { pipe("/dev/xconsole"); };
+# destination d_xconsole { pipe("/dev/xconsole"); };
 
 # Debian only
 destination d_ppp { file("/var/log/ppp.log"); };
@@ -97,6 +98,7 @@ filter f_messages { level(info,notice,warn) and
 filter f_auth { facility(auth, authpriv) and not filter(f_debug); };
 filter f_cron { facility(cron) and not filter(f_debug); };
 filter f_daemon { facility(daemon) and not filter(f_debug); };
+filter f_kern { facility(kern) and not filter(f_debug); };
 filter f_lpr { facility(lpr) and not filter(f_debug); };
 filter f_local { facility(local0, local1, local3, local4, local5,
                         local6, local7) and not filter(f_debug); };
@@ -118,6 +120,7 @@ filter f_console { level(warn .. emerg); };
 log { source(s_src); filter(f_auth); destination(d_auth); };
 log { source(s_src); filter(f_cron); destination(d_cron); };
 log { source(s_src); filter(f_daemon); destination(d_daemon); };
+log { source(s_src); filter(f_kern); destination(d_kern); };
 log { source(s_src); filter(f_lpr); destination(d_lpr); };
 log { source(s_src); filter(f_syslog3); destination(d_syslog); };
 log { source(s_src); filter(f_user); destination(d_user); };

Docker/syslog-ng.conf

@@ -1,13 +1,12 @@
-@version: 3.35
+@version: 4.3
 @include "scl.conf"
-@include "`scl-root`/system/tty10.conf"
 
 # Syslog-ng configuration file, compatible with default Debian syslogd
 # installation.
 
 # First, set some global options.
 options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
-          owner("root"); group("adm"); perm(0640); stats_freq(0);
+          owner("root"); group("adm"); perm(0640); stats(freq(0));
           bad_hostname("^gconfd$"); dns_cache(no);
 };
 
@@ -78,7 +77,7 @@ destination d_console { usertty("root"); };
 
 # Virtual console.
 #
-destination d_console_all { file(`tty10`); };
+# destination d_console_all { file(`tty10`); };
 
 # The named pipe /dev/xconsole is for the nsole' utility.  To use it,
 # you must invoke nsole' with the -file' option:
@@ -176,7 +175,7 @@ log { source(s_network); filter(f_debug); destination(d_debug); };
 log { source(s_network); filter(f_error); destination(d_error); };
 log { source(s_network); filter(f_messages); destination(d_messages); };
 
-log { source(s_network); filter(f_console); destination(d_console_all); };
+# log { source(s_network); filter(f_console); destination(d_console_all); };
 log { source(s_network); filter(f_crit); destination(d_console); };
 
 # All messages send to a remote site

Dockerfile

@@ -12,6 +12,7 @@ bash -c \
 && cp /tmp/docassemble/Docker/cgi-bin/index.sh /usr/lib/cgi-bin/ \
 && cp /tmp/docassemble/Docker/syslog-ng.conf /usr/share/docassemble/webapp/syslog-ng.conf \
 && cp /tmp/docassemble/Docker/syslog-ng-docker.conf /usr/share/docassemble/webapp/syslog-ng-docker.conf \
+&& cp /tmp/docassemble/Docker/smart-multi-line.fsm /usr/share/syslog-ng/smart-multi-line.fsm \
 && cp /tmp/docassemble/Docker/docassemble-syslog-ng.conf /usr/share/docassemble/webapp/docassemble-syslog-ng.conf \
 && cp /tmp/docassemble/Docker/apache.logrotate /etc/logrotate.d/apache2 \
 && cp /tmp/docassemble/Docker/nginx.logrotate /etc/logrotate.d/nginx \

docassemble_base/docassemble/base/functions.py

@@ -4245,7 +4245,7 @@ def package_template_filename(the_file, **kwargs):
         if package is not None:
             parts = [package, the_file]
     if len(parts) == 2:
-        m = re.search(r'^docassemble.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*)$', parts[0])
+        m = re.search(r'^docassemble\.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*)$', parts[0])
         if m:
             parts[1] = re.sub(r'^data/templates/', '', parts[1])
             abs_file = server.absolute_filename("/playgroundtemplate/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1]))  # pylint: disable=assignment-from-none
@@ -4291,7 +4291,7 @@ def package_data_filename(the_file, return_nonexistent=False):
     if len(parts) == 2:
         if filename_invalid(parts[1]) or package_name_invalid(parts[0]):
             return None
-        m = re.search(r'^docassemble.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*)$', parts[0])
+        m = re.search(r'^docassemble\.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*)$', parts[0])
         if m:
             if re.search(r'^data/sources/', parts[1]):
                 parts[1] = re.sub(r'^data/sources/', '', parts[1])

docassemble_base/docassemble/base/parse.py

@@ -2281,6 +2281,19 @@ def __init__(self, orig_data, caller, **kwargs):
                 self.interview.options['checkin interval'] = data['features']['checkin interval']
             if 'hide corner interface' in data['features']:
                 self.interview.options['hide corner interface'] = data['features']['hide corner interface']
+            if 'auto jinja filter' in data['features']:
+                if isinstance(data['features']['auto jinja filter'], list):
+                    the_list = data['features']['auto jinja filter']
+                elif not isinstance(data['features']['auto jinja filter'], str):
+                    raise DASourceError("A features section auto jinja filter entry must be a list or plain text." + self.idebug(data))
+                else:
+                    the_list = [data['features']['auto jinja filter']]
+                if 'auto jinja filter' not in self.interview.options:
+                    self.interview.options['auto jinja filter'] = []
+                for expression in the_list:
+                    if not isinstance(expression, str):
+                        raise DASourceError("A features section auto jinja filter entry must be plain text." + self.idebug(data))
+                    self.interview.options['auto jinja filter'].append(compile(expression, '<auto jinja filter>', 'eval'))
             for key in ('javascript', 'css'):
                 if key in data['features']:
                     if isinstance(data['features'][key], list):
@@ -7090,6 +7103,10 @@ def finalize_attachment(self, attachment, result, the_user_dict):
                                 result['template'].current_rendering_part = result['template'].docx._part
                             docassemble.base.functions.set_context('docx', template=result['template'])
                             docassemble.base.functions.this_thread.misc['docx_subdocs'] = []
+                            docassemble.base.functions.this_thread.misc['auto jinja filter'] = []
+                            if 'auto jinja filter' in self.interview.options:
+                                for item in self.interview.options['auto jinja filter']:
+                                    docassemble.base.functions.this_thread.misc['auto jinja filter'].append(eval(item, the_user_dict))
                             try:
                                 the_template = result['template']
                                 template_loop_count = 0
@@ -10500,6 +10517,8 @@ def ampersand_filter(value):
     value = docassemble.base.file_docx.sanitize_xml(value)
     if '<w:r>' in value or '</w:t>' in value:
         return re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&amp;', value)
+    for auto_filter in docassemble.base.functions.this_thread.misc.get('auto jinja filter', []):
+        value = auto_filter(value)
     return re.sub(r'>', '&gt;', re.sub(r'<', '&lt;', re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&amp;', value)))
 
 

docassemble_base/docassemble/base/standardformatter.py

@@ -3187,6 +3187,8 @@ def input_for(status, field, embedded=False, floating_label=None):
             if hasattr(field, 'inputtype') and field.inputtype == 'combobox' and not embedded:
                 if placeholdertext == '':
                     first_option += '<option value="">' + option_escape(word('Select one')) + '</option>'
+                elif floating_label is not None:
+                    first_option += '<option value=""></option>'
                 else:
                     first_option += '<option value="">' + option_escape(str(status.hints[field.number].replace('\n', ' '))) + '</option>'
             else:

docassemble_webapp/docassemble/webapp/backend.py

@@ -183,7 +183,7 @@ def user_is_developer(user_id):
 
 
 def absolute_filename(the_file):
-    match = re.match(r'^docassemble.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*):(.*)', the_file)
+    match = re.match(r'^docassemble\.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*):(.*)', the_file)
     # logmessage("absolute_filename call: " + the_file)
     if match:
         if not user_is_developer(match.group(1)):

docassemble_webapp/docassemble/webapp/file_access.py

@@ -33,7 +33,7 @@ def url_if_exists(file_reference, **kwargs):
     base_url = url_for('rootindex', _external=kwargs.get('_external', False)).rstrip('/')
     if len(parts) == 2:
         if cloud and docassemble.base.config.daconfig.get('use cloud urls', False):
-            m = re.search(r'^docassemble.playground([0-9]+)(.*)$', parts[0])
+            m = re.search(r'^docassemble\.playground([0-9]+)(.*)$', parts[0])
             if m:
                 user_id = m.group(1)
                 project = m.group(2)

docassemble_webapp/docassemble/webapp/server.py

@@ -13238,7 +13238,7 @@ def index(action_argument=None, refer=None):
     pipe.execute()
     if user_dict['_internal']['livehelp']['availability'] != 'unavailable':
         inputkey = 'da:input:uid:' + str(user_code) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
-        r.publish(inputkey, json.dumps({'message': 'newpage', key: key}))
+        r.publish(inputkey, json.dumps({'message': 'newpage', 'key': key}))
     if is_json:
         data = {'browser_title': interview_status.tabtitle, 'lang': interview_language, 'csrf_token': generate_csrf(), 'steps': steps, 'allow_going_back': allow_going_back, 'message_log': docassemble.base.functions.get_message_log(), 'id_dict': question_id_dict}
         data.update(interview_status.as_data(user_dict))
@@ -17512,7 +17512,7 @@ def create_playground_package():
                     break
     file_list = {}
     the_directory = directory_for(area['playgroundpackages'], current_project)
-    file_list['playgroundpackages'] = sorted([re.sub(r'^docassemble.', r'', f) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
+    file_list['playgroundpackages'] = sorted([re.sub(r'^docassemble\.', r'', f) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
     the_choices = []
     for file_option in file_list['playgroundpackages']:
         the_choices.append((file_option, file_option))
@@ -20905,16 +20905,16 @@ def playground_packages():
     files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
     editable_files = []
     for a_file in files:
-        editable_files.append({'name': re.sub(r'^docassemble.', r'', a_file), 'modtime': os.path.getmtime(os.path.join(the_directory, a_file))})
+        editable_files.append({'name': re.sub(r'^docassemble\.', r'', a_file), 'modtime': os.path.getmtime(os.path.join(the_directory, a_file))})
     assign_opacity(editable_files)
     editable_file_listing = [x['name'] for x in editable_files]
     if request.method == 'GET' and not the_file and not is_new:
         current_file = get_current_file(current_project, 'packages')
         if not current_file.startswith('docassemble.'):
             current_file = 'docassemble.' + current_file
             set_current_file(current_project, 'packages', current_file)
-        if re.sub(r'^docassemble.', r'', current_file) in editable_file_listing:
-            the_file = re.sub(r'^docassemble.', r'', current_file)
+        if re.sub(r'^docassemble\.', r'', current_file) in editable_file_listing:
+            the_file = re.sub(r'^docassemble\.', r'', current_file)
         else:
             delete_current_file(current_project, 'packages')
             if len(editable_files) > 0:
@@ -20928,7 +20928,7 @@ def playground_packages():
         set_current_file(current_project, 'packages', 'docassemble.' + the_file)
     if the_file == '' and len(file_list['playgroundpackages']) and not is_new:
         the_file = file_list['playgroundpackages'][0]
-        the_file = re.sub(r'^docassemble.', r'', the_file)
+        the_file = re.sub(r'^docassemble\.', r'', the_file)
     old_info = {}
     branch_info = []
     github_http = None
@@ -24913,7 +24913,7 @@ def user_interviews(user_id=None, secret=None, exclude_invalid=True, action=None
                 metadata = {}
                 tags = set()
             if include_dict:
-                if dictionary['_internal']['starttime']:
+                if dictionary['_internal']['starttime'] and isinstance(dictionary['_internal']['starttime'], datetime.datetime):
                     utc_starttime = dictionary['_internal']['starttime']
                     starttime = nice_date_from_utc(dictionary['_internal']['starttime'], timezone=the_timezone)
                 else:

docassemble_webapp/docassemble/webapp/socketserver.py

@@ -22,14 +22,14 @@
 from docassemble.webapp.backend import nice_utc_date, fetch_user_dict, get_chat_log, encrypt_phrase, pack_phrase, fix_pickle_obj, get_session  # noqa: E402
 from docassemble.webapp.daredis import r as rr, redis_host, redis_port, redis_offset  # noqa: E402
 from docassemble.webapp.users.models import UserModel, ChatLog  # noqa: E402
-from docassemblekvsession import KVSessionExtension  # noqa: E402 # pylint: disable=wrong-import-error
-from flask import session, request  # noqa: E402 # pylint: disable=wrong-import-error
-from flask_socketio import join_room  # noqa: E402 # pylint: disable=wrong-import-error
-from simplekv.memory.redisstore import RedisStore  # noqa: E402 # pylint: disable=wrong-import-error
-from sqlalchemy import select  # noqa: E402 # pylint: disable=wrong-import-error
-from sqlalchemy.orm import sessionmaker, joinedload  # noqa: E402 # pylint: disable=wrong-import-error
-import netifaces as ni  # noqa: E402 # pylint: disable=import-error # pylint: disable=wrong-import-error
-import redis  # noqa: E402 # pylint: disable=wrong-import-error
+from docassemblekvsession import KVSessionExtension  # noqa: E402
+from flask import session, request  # noqa: E402
+from flask_socketio import join_room  # noqa: E402
+from simplekv.memory.redisstore import RedisStore  # noqa: E402
+from sqlalchemy import select  # noqa: E402
+from sqlalchemy.orm import sessionmaker, joinedload  # noqa: E402
+import netifaces as ni  # noqa: E402 # pylint: disable=import-error
+import redis  # noqa: E402
 
 store = RedisStore(docassemble.webapp.daredis.r_store)
 kv_session = KVSessionExtension(store, app)
@@ -100,9 +100,8 @@ def background_thread(sid=None, user_id=None, temp_user_id=None):
     else:
         with session_scope() as dbsession:
             person = dbsession.execute(select(UserModel).options(joinedload(UserModel.roles)).filter_by(id=user_id)).scalar()
-        user_is_temp = False
-    if not (person is not None and person.timezone is not None):
-        r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
+        user_is_temp = person is not None
+    r = redis.StrictRedis(host=redis_host, port=redis_port, db=redis_offset)
 
     partners = set()
     pubsub = r.pubsub()
@@ -1084,7 +1083,13 @@ def terminate_observer_connection():
                 host = daconfig['websockets ip']
             else:
                 ifaces = [iface for iface in ni.interfaces() if iface != 'lo']
-                host = ni.ifaddresses(ifaces[0])[ni.AF_INET][0]['addr']
+                host = None
+                for interface in ifaces:
+                    info = ni.ifaddresses(interface)
+                    if ni.AF_INET in info and len(info[ni.AF_INET]) > 0 and 'addr' in info[ni.AF_INET][0]:
+                        host = info[ni.AF_INET][0]['addr']
+                        break
+                assert host is not None
             socketio.run(app, host=host, port=daconfig.get('websockets port', 5000))
         except:
             logmessage("Could not find the external IP address")