From 5ba3e1444e4be3325a33ab2b65e2e577e0857a22 Mon Sep 17 00:00:00 2001 From: Daniel Berteaud Date: Thu, 11 Oct 2012 11:29:02 +0200 Subject: [PATCH] Premier commit --- createlinks | 8 + mail_log.sql | 32 ++ .../db/configuration/defaults/qpsmtpd/DbName | 1 + .../db/configuration/defaults/qpsmtpd/DbUser | 1 + .../configuration/migrate/qpsmtpdDBPassword | 27 ++ .../etc/e-smith/sql/init/qpsmtpdLog2Sql | 1 + .../etc/e-smith/sql/init/qpsmtpdLog2Sql | 46 +++ .../qpsmtpd/config/peers/local/03log2sql | 15 + .../usr/share/qpsmtpd/plugins/logging/log2sql | 323 ++++++++++++++++++ smeserver-qpsmtpd-log2sql.spec | 50 +++ 10 files changed, 504 insertions(+) create mode 100644 createlinks create mode 100644 mail_log.sql create mode 100644 root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbName create mode 100644 root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbUser create mode 100644 root/etc/e-smith/db/configuration/migrate/qpsmtpdDBPassword create mode 100644 root/etc/e-smith/templates.metadata/etc/e-smith/sql/init/qpsmtpdLog2Sql create mode 100644 root/etc/e-smith/templates/etc/e-smith/sql/init/qpsmtpdLog2Sql create mode 100644 root/etc/e-smith/templates/var/service/qpsmtpd/config/peers/local/03log2sql create mode 100644 root/usr/share/qpsmtpd/plugins/logging/log2sql create mode 100644 smeserver-qpsmtpd-log2sql.spec diff --git a/createlinks b/createlinks new file mode 100644 index 0000000..3df26b9 --- /dev/null +++ b/createlinks @@ -0,0 +1,8 @@ +#!/usr/bin/perl -w + +use esmith::Build::CreateLinks qw(:all); + +foreach my $event (qw/bootstrap-console-save email-update/){ + templates2events("/etc/e-smith/sql/init/qpsmtpdLog2Sql", $event); +} + diff --git a/mail_log.sql b/mail_log.sql new file mode 100644 index 0000000..e6060ee --- /dev/null +++ b/mail_log.sql @@ -0,0 +1,32 @@ +CREATE TABLE `messages` ( + `mail_id` varchar(255) NOT NULL default '', + `time` int(11) default '0', + `remote_ip` varchar(255) default NULL, + `remote_host` varchar(255) default NULL, + `remote_info` varchar(255) default NULL, + `sender` varchar(255) default NULL, + `subject` text, + `header_size` int(11) default NULL, + `body_size` int(11) default NULL, + `spam_status` float default NULL, + `header` text, + `deny` set('YES','NO') default NULL, + `deny_plugin` varchar(100) default '', + `deny_code` int(11) default '0', + `deny_msg` text, + PRIMARY KEY (`mail_id`) +) TYPE=MyISAM; + +CREATE TABLE `rcpts` ( + `rcpt_id` int(11) NOT NULL auto_increment, + `mail_id` varchar(255) NOT NULL default '', + `recipient` varchar(255), + PRIMARY KEY (`rcpt_id`) +) TYPE=MyISAM; + +CREATE TABLE `message_body` ( + `bdy_id` int(11) NOT NULL auto_increment, + `mail_id` varchar(255) NOT NULL default '', + `body` text, + PRIMARY KEY (`bdy_id`) +) TYPE=MyISAM; diff --git a/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbName b/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbName new file mode 100644 index 0000000..5faaaf4 --- /dev/null +++ b/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbName @@ -0,0 +1 @@ +mail_log diff --git a/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbUser b/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbUser new file mode 100644 index 0000000..9df0b30 --- /dev/null +++ b/root/etc/e-smith/db/configuration/defaults/qpsmtpd/DbUser @@ -0,0 +1 @@ +qpsmtpd diff --git a/root/etc/e-smith/db/configuration/migrate/qpsmtpdDBPassword b/root/etc/e-smith/db/configuration/migrate/qpsmtpdDBPassword new file mode 100644 index 0000000..1d28e71 --- /dev/null +++ b/root/etc/e-smith/db/configuration/migrate/qpsmtpdDBPassword @@ -0,0 +1,27 @@ +{ + my $rec = $DB->get('qpsmtpd') + || $DB->new_record('qpsmtpd', {type => 'service'}); + my $pw = $rec->prop('DbPassword'); + if (not $pw or length($pw) < 57){ + use MIME::Base64 qw(encode_base64); + + $pw = "not set due to error"; + if ( open( RANDOM, "/dev/urandom" ) ){ + my $buf; + # 57 bytes is a full line of Base64 coding, and contains + # 456 bits of randomness - given a perfectly random /dev/random + if ( read( RANDOM, $buf, 57 ) != 57 ){ + warn("Short read from /dev/random: $!"); + } + else{ + $pw = encode_base64($buf); + chomp $pw; + } + close RANDOM; + } + else{ + warn "Could not open /dev/urandom: $!"; + } + $rec->set_prop('DbPassword', $pw); + } +} diff --git a/root/etc/e-smith/templates.metadata/etc/e-smith/sql/init/qpsmtpdLog2Sql b/root/etc/e-smith/templates.metadata/etc/e-smith/sql/init/qpsmtpdLog2Sql new file mode 100644 index 0000000..940dcf3 --- /dev/null +++ b/root/etc/e-smith/templates.metadata/etc/e-smith/sql/init/qpsmtpdLog2Sql @@ -0,0 +1 @@ +PERMS=0750 diff --git a/root/etc/e-smith/templates/etc/e-smith/sql/init/qpsmtpdLog2Sql b/root/etc/e-smith/templates/etc/e-smith/sql/init/qpsmtpdLog2Sql new file mode 100644 index 0000000..2e817b6 --- /dev/null +++ b/root/etc/e-smith/templates/etc/e-smith/sql/init/qpsmtpdLog2Sql @@ -0,0 +1,46 @@ +{ +my $db = ${'qpsmtpd'}{'DbName'} || 'mail_log'; +my $user = ${'qpsmtpd'}{'DbUser'} || 'qpsmtpd'; +my $pass = ${'qpsmtpd'}{'DbPassword'} || 'secret'; + +my $dbstruct = `rpm -qd smeserver-qpsmtpd-log2sql | grep mail_log.sql`; + +$OUT .= <<"END"; +#! /bin/sh +if [ ! -d /var/lib/mysql/$db ]; then + /usr/bin/mysql -e 'create database $db' + /usr/bin/mysql $db < $dbstruct +fi + +/usr/bin/mysql < # +# http://www.geocities.com/tttodorov/ # +# VERSION 0.06 # +# Last modified 2005/02/22 # +##################################################### + +use DBI; +use strict; + +#plugin level variables are here since version 0.02 +my($dsn,$mail_table,$rcpt_table,$user,$passwd,$mail_id); +my($log_header); +my($log_all_body,$log_deny_body,$body_table); +my($dbh,$sth); + +sub register +{my ($self,$qp) = (shift,shift); +$self->log(LOGERROR,"Bad count of parameters in log2sql plugin.") if @_ % 2; +my(%args) = @_; + +$self->register_hook("connect", "connect_handler"); +$self->register_hook("mail", "mail_handler"); +$self->register_hook("rcpt", "rcpt_handler"); +$self->register_hook("data_post", "data_post_handler"); +$self->register_hook("queue", "queue_handler"); + +$self->register_hook("deny", "deny_handler") if $args{D}; + +$self->register_hook("disconnect", "disconnect_handler"); + +$dsn = 'DBI'; +$dsn .= ':'.($args{t} || 'mysql'); +$dsn .= ':'.($args{d} || 'qpsmtpd'); +$dsn .= ';host='.$args{h} if $args{h}; +$dsn .= ';port='.$args{P} if $args{P}; +$dsn .= ';'.$args{o} if $args{o}; + +$mail_table = $args{m} || 'messages'; +$rcpt_table = $args{r} || 'rcpts'; + +$user = $args{u} || 'root'; +$passwd = $args{p} || ''; + +$log_header = $args{H} || undef; + +$log_all_body = $args{B} || undef; +$log_deny_body = $args{DB} || undef; +$body_table = $args{b} || 'message_body'; +}#end register + +sub connect_handler +{my ($self,$transaction) = @_; + +$self->log(LOGDEBUG,"DSN:$dsn"); +$dbh = DBI->connect($dsn,$user,$passwd) + || $self->log(LOGERROR,DBI::errstr); + +# generate mail id +$mail_id = $$.'.'.time.'.'.int(rand(10000)); +# set note for other plugins +$transaction->notes('log2sql_mail_id', $mail_id); +# quote the mail_id for later use: +$mail_id = $dbh->quote($mail_id); + +my($statement) = "INSERT INTO ".$mail_table." (mail_id,". + $mail_table.".time,". + "remote_ip,remote_host,remote_info) ". + "VALUES(". + $mail_id.",". + $dbh->quote(time).",". + $dbh->quote($self->qp->connection->remote_ip).",". + $dbh->quote($self->qp->connection->remote_host).",". + $dbh->quote($self->qp->connection->remote_info).")"; + +$self->log(LOGDEBUG,"connect_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + + +return(DECLINED); +} + + +sub mail_handler +{my ($self, $transaction, $sender) =@_; + +my($statement) = "UPDATE ".$mail_table." SET sender=". + $dbh->quote($sender->user.'@'.$sender->host). + " WHERE mail_id=".$mail_id; + + +$self->log(LOGDEBUG,"mail_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + +return(DECLINED); +} + +sub rcpt_handler +{my($self,$transaction,$recipient) = @_; + +my($statement) = "INSERT INTO ".$rcpt_table." (mail_id,". + "recipient) ". + "VALUES(". + $mail_id.",". + $dbh->quote($recipient->user.'@'.$recipient->host).")"; + +$self->log(LOGDEBUG,"rcpt_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + + +return(DECLINED); +} + +sub data_post_handler +{my($self,$transaction)=@_; +my($header) = $transaction->header(); +# In data_post handler the header is not actual, +# but in queue handler somethimes there are problems with +# header, sizes, subject logs (don't know why?!?!) + +#'touch' this message :-)) +$header->add("X-Sql-Log-ID",$mail_id); + +my($statement) = "UPDATE ".$mail_table." SET subject=". + $dbh->quote(scalar $header->get("Subject")). + ",header_size=".$dbh->quote(length($header->as_string())). + ",body_size=".$dbh->quote($transaction->body_size()). + " WHERE mail_id=".$mail_id; + +$self->log(LOGDEBUG,"data_post_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + +#if $args{H} then log the message header in the main 'messages' table +if ($log_header) + {$statement = "UPDATE ".$mail_table." SET ". + "header=".$dbh->quote($header->as_string()). + "WHERE mail_id=".$mail_id; + $self->log(LOGDEBUG,"data_post_handler log header statement:".$statement); + + $dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + }#end log geader + +#if args{B} then log message body in a separated table +if ($log_all_body) + {my(@body) = ($transaction->header()->as_string()); + + $transaction->body_resetpos(); + + while(my $line = $transaction->body_getline()) + {push(@body,$line); + } + + $statement = "INSERT INTO ".$body_table." (mail_id,body)". + " VALUES(".$mail_id.",".$dbh->quote(join('',@body)).")"; + $self->log(LOGDEBUG,"data_post body log statement:".$statement); + + $dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + + }#end body log + +return(DECLINED); +} + + +sub queue_handler +{my($self,$transaction) = @_; +my($status) = $transaction->header->get('X-Spam-Status') or return (DECLINED); +my($score) = ($status =~ m/hits=(\d+\.\d+)/)[0] || "0" ; + +my($statement) = "UPDATE ".$mail_table." SET ". + "spam_status=".$dbh->quote($score). + " WHERE mail_id=".$mail_id; + +$self->log(LOGDEBUG,"queue_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + +return(DECLINED); +} + +sub deny_handler +{my($self,$transaction,$plugin,$code,$msg) = @_; + +my($statement) = "UPDATE ".$mail_table." SET ". + "deny='YES'". + ",deny_plugin=".$dbh->quote($plugin). + ",deny_code=".$dbh->quote($code). + ",deny_msg=".$dbh->quote($msg). + " WHERE mail_id=".$mail_id; + +$self->log(LOGDEBUG,"deny_handler statement:".$statement); + +$dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + +#if configured to log the body of the denied messages +if ($log_deny_body && ! $log_all_body) + {my(@body) = ($transaction->header()->as_string()); + + $transaction->body_resetpos(); + + while(my $line = $transaction->body_getline()) + {push(@body,$line); + } + + $statement = "INSERT INTO ".$body_table." (mail_id,body)". + " VALUES(".$mail_id.",".$dbh->quote(join('',@body)).")"; + $self->log(LOGDEBUG,"data_post body log statement:".$statement); + + $dbh->do($statement) + || $self->log(LOGERROR,$dbh->errstr()); + + }#end body log + + +return(DECLINED); +} + +sub disconnect_handler +{my($self,$transaction) = @_; + +$dbh->disconnect(); + +return(DECLINED); +} + +=head1 NAME + +log2sql - qmstpd plugin for logging information for mail messages in a SQL database. + +=head1 DESCRIPTION + +The plugin logs the information for mail messages: remote_ip, remote_host, remote_info, +sender, subject, header_size, body_size, spam_status (this is the 'hits' field from spamassassin 'X-Spam-Status' header line), header ,all recipients of the message, etc. The data is logged into two tables: messages table - one message per row, +recipients table - one recipient per row. One message could have more than one recipient. + +=head1 CONFIGURATION + +The plugin accepts the following parameters: + + t - type of the SQL database (mysql, postgresql, etc.) + d - database name (default qpsmtpd) + h - hostname of the database server (optional) + P - tcp port of the database server (optional) + o - dsn options (in format like DBI dsn options). For example: "o mysql_socket=/path/to/socket" + m - database table for mail messages (default messages) + r - database table for the recipients (default rcpts) + u - database user (default root) + p - database password (default '') + H - log the header of the message in the main table for mail messages (e.g. 'messages'). For example "H Yes". + +=head1 CONFIGURATION OF THE ADDITIONAL FEATURES + +You can configure the following additional features: + + D - log the deny information for denied messages in the 'messages' table. Example "D Yes". The 'deny' hook must be enabled in order to use this feature. + B - log the body of all messages in the separated table (e.g. 'message_body'). Example "B Yes". + DB - log the body of the denied messages. D parameter required. Example "DB Yes". + b - sql table for the message bodies. Default: 'message_body'. + +As default the plugin sets a note 'log2sql_mail_id'. This chunk of data is passed to other plugins. As default the plugin sets a field 'X-Sql-Log-ID' in the message header. + +=head1 SQL TABLES + +Here are the CREATE statements for the tables. Assuming the 'messages' table is the table for logging main information for messages, 'rcpts' table is the table for recipients, 'message_body' is optional table if You want to log messages bodies (of the denied messages OR of all messages). This syntax is MySQL syntax. For other databases the syntax could be slightly different. + + + CREATE TABLE `messages` ( + `mail_id` varchar(255) NOT NULL default '', + `time` int(11) default '0', + `remote_ip` varchar(255) default NULL, + `remote_host` varchar(255) default NULL, + `remote_info` varchar(255) default NULL, + `sender` varchar(255) default NULL, + `subject` text, + `header_size` int(11) default NULL, + `body_size` int(11) default NULL, + `spam_status` float default NULL, + `header` text, + `deny` set('YES','NO') default NULL, + `deny_plugin` varchar(100) default '', + `deny_code` int(11) default '0', + `deny_msg` text, + PRIMARY KEY (`mail_id`) + ) TYPE=MyISAM; + + CREATE TABLE `rcpts` ( + `rcpt_id` int(11) NOT NULL auto_increment, + `mail_id` varchar(255) NOT NULL default '', + `recipient` varchar(255), + PRIMARY KEY (`rcpt_id`) + ) TYPE=MyISAM; + + CREATE TABLE `message_body` ( + `bdy_id` int(11) NOT NULL auto_increment, + `mail_id` varchar(255) NOT NULL default '', + `body` text, + PRIMARY KEY (`bdy_id`) + ) TYPE=MyISAM; + + +=head1 AUTHOR + +Todor Todorov, Etttodorov@yahoo.comE + +L + +=cut + + diff --git a/smeserver-qpsmtpd-log2sql.spec b/smeserver-qpsmtpd-log2sql.spec new file mode 100644 index 0000000..5d6b033 --- /dev/null +++ b/smeserver-qpsmtpd-log2sql.spec @@ -0,0 +1,50 @@ +%define version 0.0.1 +%define release 1.beta0 +%define name smeserver-qpsmtpd-log2sql + + +Summary: SQL Logging plugin for qpsmtpd +Name: %{name} +Version: %{version} +Release: %{release}%{?dist} +License: GPL +Group: Networking/Daemons +Source: %{name}-%{version}.tar.gz + +BuildRoot: /var/tmp/%{name}-%{version}-%{release}-buildroot +BuildArchitectures: noarch +BuildRequires: e-smith-devtools + +Requires: smeserver-qpsmtpd +Requires: e-smith-mysql +Requires: perl(Date::Time) +Requires: perl(DBI); + +%description +Log every smtpd transaction in a database + +%changelog +* Thu Oct 11 2012 Daniel Berteaud 0.0.1 +- Initial release + +%build +perl createlinks + +%install +/bin/rm -rf $RPM_BUILD_ROOT +(cd root ; /usr/bin/find . -depth -print | /bin/cpio -dump $RPM_BUILD_ROOT) +/bin/rm -f %{name}-%{version}-filelist +/sbin/e-smith/genfilelist $RPM_BUILD_ROOT \ + > %{name}-%{version}-filelist + +echo "%doc CHANGELOG.git" >> %{name}-%{version}-filelist +echo "%doc mail_log.sql" >> %{name}-%{version}-filelist +%files -f %{name}-%{version}-filelist +%defattr(-,root,root) + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +%preun +