Update docs

domainaware · Mar 4, 2024 · 10ef6fc · 10ef6fc
1 parent 51fd81a
commit 10ef6fc
Show file tree

Hide file tree

Showing 29 changed files with 1,408 additions and 100 deletions.
diff --git a/_modules/index.html b/_modules/index.html
@@ -3,7 +3,7 @@
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Overview: module code &mdash; parsedmarc 8.7.0 documentation</title>
+  <title>Overview: module code &mdash; parsedmarc 8.8.0 documentation</title>
       <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
       <link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
 
@@ -35,7 +35,7 @@
             parsedmarc
           </a>
               <div class="version">
-                8.7.0
+                8.8.0
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -51,6 +51,7 @@
 <li class="toctree-l1"><a class="reference internal" href="../usage.html">Using parsedmarc</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../output.html">Sample outputs</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../opensearch.html">OpenSearch and Grafana</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../kibana.html">Using the Kibana dashboards</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../splunk.html">Splunk</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../davmail.html">Accessing an inbox using OWA/EWS</a></li>
@@ -85,6 +86,7 @@
   <h1>All modules for which code is available</h1>
 <ul><li><a href="parsedmarc.html">parsedmarc</a></li>
 <ul><li><a href="parsedmarc/elastic.html">parsedmarc.elastic</a></li>
+<li><a href="parsedmarc/opensearch.html">parsedmarc.opensearch</a></li>
 <li><a href="parsedmarc/splunk.html">parsedmarc.splunk</a></li>
 <li><a href="parsedmarc/utils.html">parsedmarc.utils</a></li>
 </ul></ul>

diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html
@@ -3,7 +3,7 @@
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>parsedmarc &mdash; parsedmarc 8.7.0 documentation</title>
+  <title>parsedmarc &mdash; parsedmarc 8.8.0 documentation</title>
       <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
       <link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
 
@@ -35,7 +35,7 @@
             parsedmarc
           </a>
               <div class="version">
-                8.7.0
+                8.8.0
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -51,6 +51,7 @@
 <li class="toctree-l1"><a class="reference internal" href="../usage.html">Using parsedmarc</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../output.html">Sample outputs</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../opensearch.html">OpenSearch and Grafana</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../kibana.html">Using the Kibana dashboards</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../splunk.html">Splunk</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../davmail.html">Accessing an inbox using OWA/EWS</a></li>
@@ -120,7 +121,7 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
 <span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
 <span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
 
-<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.7.0&quot;</span>
+<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.8.0&quot;</span>
 
 <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;parsedmarc v</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
 
@@ -300,10 +301,14 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
         <span class="n">new_failure_details</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
             <span class="n">result_type</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;result-type&quot;</span><span class="p">],</span>
             <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;failed-session-count&quot;</span><span class="p">],</span>
-            <span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;sending-mta-ip&quot;</span><span class="p">],</span>
-            <span class="n">receiving_ip</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;receiving-ip&quot;</span><span class="p">]</span>
         <span class="p">)</span>
 
+        <span class="k">if</span> <span class="s2">&quot;sending-mta-ip&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
+            <span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
+                <span class="s2">&quot;sending-mta-ip&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="s2">&quot;receiving-ip&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
+            <span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_ip&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
+                <span class="s2">&quot;receiving-ip&quot;</span><span class="p">]</span>
         <span class="k">if</span> <span class="s2">&quot;receiving-mx-hostname&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
             <span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
                 <span class="s2">&quot;receiving-mx-hostname&quot;</span><span class="p">]</span>
@@ -389,6 +394,7 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
             <span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization-name&quot;</span><span class="p">],</span>
             <span class="n">begin_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date-range&quot;</span><span class="p">][</span><span class="s2">&quot;start-datetime&quot;</span><span class="p">],</span>
             <span class="n">end_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date-range&quot;</span><span class="p">][</span><span class="s2">&quot;end-datetime&quot;</span><span class="p">],</span>
+            <span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;contact-info&quot;</span><span class="p">],</span>
             <span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report-id&quot;</span><span class="p">],</span>
             <span class="n">policies</span><span class="o">=</span><span class="n">policies</span>
         <span class="p">)</span>
@@ -449,10 +455,10 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">,</span> <span class="s2">&quot;begin_date&quot;</span><span class="p">,</span> <span class="s2">&quot;end_date&quot;</span><span class="p">,</span> <span class="s2">&quot;report_id&quot;</span><span class="p">,</span>
-              <span class="s2">&quot;successful_session_count&quot;</span><span class="p">,</span> <span class="s2">&quot;failed_session_count&quot;</span><span class="p">,</span>
-              <span class="s2">&quot;policy_domain&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_type&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_strings&quot;</span><span class="p">,</span>
-              <span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">,</span> <span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_ip&quot;</span><span class="p">,</span>
-              <span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">,</span>
+              <span class="s2">&quot;result_type&quot;</span><span class="p">,</span> <span class="s2">&quot;successful_session_count&quot;</span><span class="p">,</span>
+              <span class="s2">&quot;failed_session_count&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_domain&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_type&quot;</span><span class="p">,</span>
+              <span class="s2">&quot;policy_strings&quot;</span><span class="p">,</span> <span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">,</span> <span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">,</span>
+              <span class="s2">&quot;receiving_ip&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">,</span>
               <span class="s2">&quot;additional_info_uri&quot;</span><span class="p">,</span> <span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span>
 
     <span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">newline</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
@@ -546,7 +552,7 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
         <span class="n">new_report_metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report_id</span>
         <span class="n">date_range</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">][</span><span class="s2">&quot;date_range&quot;</span><span class="p">]</span>
         <span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">2</span><span class="o">*</span><span class="mi">86400</span><span class="p">:</span>
-            <span class="n">_error</span> <span class="o">=</span> <span class="s2">&quot;Timespan &gt; 24 hours - RFC 7489 section 7.2&quot;</span>
+            <span class="n">_error</span> <span class="o">=</span> <span class="s2">&quot;Time span &gt; 24 hours - RFC 7489 section 7.2&quot;</span>
             <span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_error</span><span class="p">)</span>
         <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;begin&quot;</span><span class="p">])</span>
         <span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp_to_human</span><span class="p">(</span><span class="n">date_range</span><span class="p">[</span><span class="s2">&quot;end&quot;</span><span class="p">])</span>
@@ -1539,7 +1545,7 @@ <h1>Source code for parsedmarc</h1><div class="highlight"><pre>
                     <span class="n">message</span> <span class="o">=</span> <span class="s2">&quot;Moving message&quot;</span>
                     <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                         <span class="n">message</span><span class="p">,</span>
-                        <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">smtp_tls_msg_uids</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
+                        <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">number_of_smtp_tls_uids</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
                     <span class="k">try</span><span class="p">:</span>
                         <span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span>
                                                 <span class="n">smtp_tls_reports_folder</span><span class="p">)</span>