Skip to content

Commit

Permalink
Merge pull request #1300 from mailwatch/migrate-to-dbd-mariadb
Browse files Browse the repository at this point in the history 
Migrate MailScanner perl scripts to DBD::MariaDB
  • Loading branch information
@endelwar
endelwar authored Jul 31, 2024
2 parents 5e41f3a + 2bafb80 commit 0b58f3e
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 120 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
- Update htmlpurifier library to 4.17.0
- Avoid usage of deprecated utf8_encode function in viewpart

### Compatibility
- Migrate MailScanner perl script to DBD:MariaDB

## 1.2.23
### Added
- Support for handling the `uopz` extension to prevent the application from breaking due to disabled `exit` calls by `uopz`.
Expand Down
43 changes: 16 additions & 27 deletions MailScanner_perl_scripts/MailWatch.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
# MailWatch for MailScanner
# Copyright (C) 2003-2011 Steve Freegard ([email protected])
# Copyright (C) 2011 Garrod Alwood ([email protected])
# Copyright (C) 2014-2021 MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
# Copyright (C) 2014-2024 MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
#
# Custom Module MailWatch
#
# Version 1.61
# Version 1.7
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
Expand All @@ -31,6 +31,7 @@ package MailScanner::CustomConfig;

use strict;
use DBI;
use DBD::MariaDB;
use utf8;
use Sys::Hostname;
use Storable(qw[freeze thaw]);
Expand All @@ -46,7 +47,7 @@ use Sys::Syslog;
use vars qw($VERSION);

### The package version, both in 1.23 style *and* usable by MakeMaker:
$VERSION = substr q$Revision: 1.5 $, 10;
$VERSION = '1.7';

# Trace settings - uncomment this to debug
#DBI->trace(2,'/tmp/dbitrace.log');
Expand Down Expand Up @@ -119,20 +120,20 @@ sub InitMailWatchLogging {
sub CheckSQLVersion {
# Prevent Logger from dying if connection fails
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
LogMessage("warn", "Unable to initialise database connection: $DBI::errstr");
close(SERVER);
return 1;
}
$SQLversion = $dbh->{mysql_serverversion};
$SQLversion = $dbh->{mariadb_serverversion};
$dbh->disconnect;
return $SQLversion;

return $SQLversion;
}

sub LogMessage {
Expand Down Expand Up @@ -177,29 +178,17 @@ sub InitDB {
return 1;
}

if (CheckSQLVersion() >= 50503 ) {
eval { $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
);
};
if ($@ || !$dbh) {
LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
return 1;
}
$dbh->do('SET NAMES utf8mb4');
} else {
eval { $dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
eval { $dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
return 1;
}
$dbh->do('SET NAMES utf8');
};
if ($@ || !$dbh) {
LogMessage('warn', "Unable to initialise database connection: $DBI::errstr");
return 1;
}
$dbh->do('SET NAMES utf8mb4');

$sth = $dbh->prepare("INSERT INTO maillog (timestamp, id, size, from_address, from_domain, to_address, to_domain, subject, clientip, archive, isspam, ishighspam, issaspam, isrblspam, spamwhitelisted, spamblacklisted, sascore, spamreport, virusinfected, nameinfected, otherinfected, report, ismcp, ishighmcp, issamcp, mcpwhitelisted, mcpblacklisted, mcpsascore, mcpreport, hostname, date, time, headers, quarantined, rblspamreport, token, messageid) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
if (!$sth) {
LogMessage('warn', "Error: $DBI::errstr" );
Expand Down
53 changes: 20 additions & 33 deletions MailScanner_perl_scripts/SQLBlackWhiteList.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
# MailWatch for MailScanner
# Copyright (C) 2003-2011 Steve Freegard ([email protected])
# Copyright (C) 2011 Garrod Alwood ([email protected])
# Copyright (C) 2014-2021 MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
# Copyright (C) 2014-2024 MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
#
# Custom Module SQLBlackWhiteList
#
# Version 1.6
# Version 1.7
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
Expand Down Expand Up @@ -39,9 +39,10 @@ use vars qw($VERSION);
#use Data::Dumper;

### The package version, both in 1.23 style *and* usable by MakeMaker:
$VERSION = substr q$Revision: 1.5 $, 10;
$VERSION = '1.7';

use DBI;
use DBD::MariaDB;
my (%Whitelist, %Blacklist);
my ($wtime, $btime);
my ($dbh);
Expand All @@ -61,23 +62,23 @@ my ($db_pass) = mailwatch_get_db_password();
# Get refresh time from from 00MailWatchConf.pm
my ($bwl_refresh_time) = mailwatch_get_BWL_refresh_time();

# Check MySQL version
# Check MySQL/MariaDB version
sub CheckSQLVersion {
# Prevent dying from failed db connection
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList:: Unable to initialise database connection: %s", $DBI::errstr);
return 1;
}
$SQLversion = $dbh->{mysql_serverversion};
$SQLversion = $dbh->{mariadb_serverversion};
$dbh->disconnect;
return $SQLversion;

return $SQLversion;
}

#
Expand Down Expand Up @@ -135,39 +136,25 @@ sub CreateList {
my ($type, $BlackWhite) = @_;
my ($sql, $to_address, $from_address, $count, $filter);

# Check if MySQL is >= 5.3.3
# Check if MySQL/MariaDB is >= 5.3.3
my $version = CheckSQLVersion();

# Database connection failed, so return 0 (count) and exit early
if ($version == 1) {
return 0;
}

if ($version >= 50503 ) {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');
} else {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8');
eval {
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLBlackWhiteList::CreateList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');

# Uncommet the folloging line when debugging SQLBlackWhiteList.pm
#MailScanner::Log::WarnLog("MailWatch: DEBUG SQLBlackWhiteList: CreateList: %s", Dumper($BlackWhite));
Expand Down
88 changes: 31 additions & 57 deletions MailScanner_perl_scripts/SQLSpamSettings.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
# MailWatch for MailScanner
# Copyright (C) 2003-2011 Steve Freegard ([email protected])
# Copyright (C) 2011 Garrod Alwood ([email protected])
# Copyright (C) 2014-2021 MailWatch Team (https://github.com/mailwatch/1.2.0/graphs/contributors)
# Copyright (C) 2014-2024 MailWatch Team (https://github.com/mailwatch/MailWatch/graphs/contributors)
#
# Custom Module SQLSpamSettings
#
# Version 1.6
# Version 1.7
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later
Expand Down Expand Up @@ -41,9 +41,10 @@ no strict 'subs'; # Allow bare words for parameter %'s
use vars qw($VERSION);

### The package version, both in 1.23 style *and* usable by MakeMaker:
$VERSION = substr q$Revision: 1.5 $, 10;
$VERSION = '1.7';

use DBI;
use DBD::MariaDB;
my ($dbh);
my ($sth);
my ($SQLversion);
Expand All @@ -64,20 +65,22 @@ my ($db_pass) = mailwatch_get_db_password();
# Get refresh time from from 00MailWatchConf.pm
my ($ss_refresh_time) = mailwatch_get_SS_refresh_time();

# Check MySQL version
# Check MySQL/MariaDB version
sub CheckSQLVersion {
# Prevent dying from failed db connection
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: Unable to initialise database connection: %s", $DBI::errstr);
return 1;
}
$SQLversion = $dbh->{mysql_serverversion};
$SQLversion = $dbh->{mariadb_serverversion};
$dbh->disconnect;

return $SQLversion;
}

Expand Down Expand Up @@ -169,39 +172,25 @@ sub CreateScoreList
my ($type, $UserList) = @_;
my ($sql, $username, $count);

# Check if MySQL is >= 5.3.3
# Check if MySQL/MariaDB is >= 5.3.3
my $version = CheckSQLVersion();

# Cannot get SQL version, bail out with count of 0
if ($version == 1) {
return 0;
}

if ($version >= 50503 ) {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');
} else {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8');
eval {
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings:: CreateScoreList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');

$sql = "SELECT username, $type FROM users WHERE $type > 0";
$sth = $dbh->prepare($sql);
Expand All @@ -228,32 +217,17 @@ sub CreateNoScanList
my ($type, $NoScanList) = @_;
my ($sql, $username, $count);

# Check if MySQL is >= 5.3.3
if (CheckSQLVersion() >= 50503 ) {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8mb4 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');
} else {
eval {
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1, mysql_enable_utf8 => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8');
eval {
$dbh = DBI->connect("DBI:MariaDB:database=$db_name;host=$db_host",
$db_user, $db_pass,
{ PrintError => 0, AutoCommit => 1, RaiseError => 1 }
);
};
if ($@ || !$dbh) {
MailScanner::Log::WarnLog("MailWatch: SQLSpamSettings::CreateNoScanList::: Unable to initialise database connection: %s", $DBI::errstr);
return 0;
}
$dbh->do('SET NAMES utf8mb4');

$sql = "SELECT username, $type FROM users WHERE $type > 0";
$sth = $dbh->prepare($sql);
Expand Down
6 changes: 3 additions & 3 deletions Remote_DB.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ This document presumes that you will have a server acting as a database with PHP
mysql> GRANT FILE ON *.* TO mailwatch IDENTIFIED BY '<password>';
mysql> flush privileges;

3) On each MailScanner gateway, you'll need to make sure that the mysql client, perl, perl DBI and perl DBD-Mysql (4.032 or higher) are installed: see https://docs.mailwatch.org/install/getting-started.html
3) On each MailScanner gateway, you'll need to make sure that the mysql client, perl, perl DBI and perl DBD-MariaDB are installed: see https://docs.mailwatch.org/install/getting-started.html

4) From one of the MailScanner gateway, verify you can connect to the db:
% mysql mailscanner -u mailwatch -h <db_hostname> -p
Expand All @@ -19,6 +19,6 @@ This document presumes that you will have a server acting as a database with PHP

5) On each MailScanner gateway continue following the install instructions at https://docs.mailwatch.org/install/installing.html#create-a-mysql-user-and-password--set-up-mailscanner-for-sql-logging

7) On each MailWatch system set RPC_ALLOWED_CLIENTS in conf.php to a list of IP addresses of each MailWatch system.
6) On each MailWatch system set RPC_ALLOWED_CLIENTS in conf.php to a list of IP addresses of each MailWatch system.

8) (Optional) If you want a combined display of the number of mails in the mail queues also set RPC_REMOTE_SERVER in conf.php
7) (Optional) If you want a combined display of the number of mails in the mail queues also set RPC_REMOTE_SERVER in conf.php

0 comments on commit 0b58f3e

Please sign in to comment.