[syslog-ng] Integrating with snmp traps

[Evan Rempel]

We are not quite doing this, but it would not be too difficult to make this perl script produce json output, and have
syslog-ng parse the source as json input.

The attached script gets configuration details and creates its own logs via private packages, but they are
easy to replace. Be careful to provide the PID file and the IP address to listen on.

The "good stuff" is in the "sub writeBuff" where the list of name value pairs is produced.
You could split eah of these on the " = " giving you the name/value pairs themselves and then populate a json object before printing it to the
standard output.

We run this script as
source snmptrapd { program("/opt/flare/bin/flare-snmptrap" flags(expect-hostname) keep-hostname(yes) tags("snmptrap")); };

because the script currently outputs syslog formatted messasges.

Hope this is a good starting point.



On 06/09/2015 07:29 AM, Fabien Wernli wrote:
> Hi,
>
> On Tue, Jun 09, 2015 at 06:26:40AM -0700, Evan Rempel wrote:
>> When you say you "would actually like to parse all the key-values from
>> the original payload" what do you mean?
>> Do you want to process them with syslog-ng filters etc, or are you
>> feeding this to another type of structured worker thread?
> I'd like to basically have the structured SNMP message parsed in syslog-ng,
> in the same way json-parser() parses JSON payload, and makes the key-values
> available as macros in syslog-ng.
>
>
> ______________________________________________________________________________
> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
> Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
> FAQ: http://www.balabit.com/wiki/syslog-ng-faq
>


-- 
Evan Rempel                                      erempel at uvic.ca
Senior Systems Administrator                        250.721.7691
Data Centre Services, University Systems, University of Victoria

-------------- next part --------------
#!/usr/local/bin/perl
#
# Start the snmptrapd with the correct options and formatting paramaters.
# 
# accept a stream of snmptrap lines and format them into a syslog event.
#
# The separates the name value pairs by a horizontal tab character, and terminametes the complete event with
# a vertical tab.
#
# Name value pairs can contain a newline in its values, but they are translated to spaces.
# All of the name value pairs are sorted prior to the record being rewriten to the stdout.

use strict;

use lib "/opt/flare/lib";

use POSIX;
use JSON;
use Data::Dumper;

use FLARE::Logger;
use FLARE::Config;


FLARE::Logger::message("info","starting");

# ------------------------------------------------------------------- writeBuff
# Sort all of the name value pairs and write the reformatted
# line to the stdout.
sub writeBuff($) {
	# write out the line, but sort elements 2-n of line split on tab
	my ($line) = @_;

	my ($header, @nameValues) = split(/\t/, $line);
	# foreach $data (@nameValues) { print $data, "\n"; }
	# print "------------\n";
	# foreach $data (sort(@nameValues)) { print $data, "\n"; }
	print $header, "\t", join("\t", sort(@nameValues) ), "\n";
}

# ------------------------------------------------------------------- logSignal
sub logSignal($) {
my ($signal) = @_;
	FLARE::Logger::message("info","received signal $signal");
}

# ------------------------------------------------------------------- main

# syslog-ng will send a sig TERM when shutting down.
# handle signals
$SIG{HUP} = sub { logSignal("HUP") };
$SIG{INT} = sub { logSignal("INT") };
$SIG{QUIT} = sub { logSignal("QUIT") };
$SIG{SEGV} = sub { logSignal("SEGV") };
$SIG{TRAP} = sub { logSignal("TRAP") };
$SIG{ABRT} = sub { logSignal("ABRT") };
$SIG{USR1} = sub { logSignal("USR1") };
$SIG{USR2} = sub { logSignal("USR2") };
# $SIG{PIPE} = sub { logSignal("PIPE") };
$SIG{ALRM} = sub { logSignal("ALRM") };
# $SIG{TERM} = sub { logSignal("TERM") };
$SIG{CHLD} = sub { logSignal("CHLD") };
$SIG{CONT} = sub { logSignal("CONT") };
$SIG{STOP} = sub { logSignal("STOP") };
$SIG{TSTP} = sub { logSignal("TSTP") };
$SIG{TTIN} = sub { logSignal("TTIN") };
$SIG{TTOUT} = sub { logSignal("TTOUT") };
$SIG{IO} = sub { logSignal("IO} = ") };
$SIG{SYS} = sub { logSignal("SYS") };
$SIG{URG} = sub { logSignal("URG") };

my ($ExitFlag, $sibling, $name, $pid, $pidFile, $pidDir);
$pid = 0;

# if syslog-ng closes us, write out the buffer
$ExitFlag = 0;
sub signalExit() {
	$ExitFlag = 1;
	if ($pid != 0) {
		kill 'TERM', $pid;
	}
};

$SIG{PIPE} = sub {
	signalExit();
	logSignal("PIPE");
};
$SIG{TERM} = sub {
	signalExit();
	logSignal("TERM");
};

my $hostname = $ENV{"HOSTNAME"};

my $programName = $0;
$programName =~ s/^.*\///;

my $logLevel = FLARE::Config::get("flare-snmptrap.loglevel");
if ( defined $logLevel ) {
	FLARE::Logger::level("$logLevel");
} else {
	FLARE::Logger::message("err","undefined log level");
	FLARE::Logger::level("debug");
}

my $snmptrapIP = FLARE::Config::get("flare-snmptrap.ip");
if ( ! defined $snmptrapIP ) {
	# repeatedly alert that there is no IP address to bind to.
	while ( $ExitFlag == 0 ) {
		FLARE::Logger::message("err","undefined ip address to bind snmptrapd");
		sleep 60;
	}
}

$pidFile = FLARE::Config::get("flare-snmptrap.pid");
if ( ! defined $pidFile ) {
	# repeatedly alert that there is no IP address to bind to.
	while ( $ExitFlag == 0 ) {
		FLARE::Logger::message("err","undefined pid file");
		sleep 60;
	}
} else {
	$pidFile =~ m/(.*)\//;
	$pidDir = $1;
	# FLARE::Logger::message("info","reviewing pid file $pidFile");
	sleep 1;
	if ( -r "$pidFile") {
		open(PID,"<","$pidFile");
		$sibling = <PID>;
		chomp $sibling;
		close(PID);
		# FLARE::Logger::message("info","reviewing sibling pid $sibling");
		# if the pid is another instance of myself, send it a SIGTERM
		# FLARE::Logger::message("info","reviewing pid name file /proc/$sibling/comm");
		if ( -r "/proc/$sibling/comm") {
			open(NAME,"<","/proc/$sibling/comm");
			$name = <NAME>;
			#chomp $name;
			close(NAME);
			if ("$name" == "$programName") {
				FLARE::Logger::message("info","terminating sibling $name with pid $sibling");
				kill 'TERM', $sibling;
				sleep 1;
			} else {
				FLARE::Logger::message("info","unmatched sibling $name with pid $sibling");
			}
		}
	}

	while ( ($ExitFlag == 0) && (((-e "$pidFile") && (! -w "$pidFile")) || (! -w "$pidDir")) ) {
		FLARE::Logger::message("err","can not write to pid file $pidFile");
		sleep 60;
	}
	if ( $ExitFlag == 0) {
		open(PID, ">", $pidFile);
		print PID "$$\n";
		close(PID);
	}
}

if ($ExitFlag == 0) {
	FLARE::Logger::message("info","ready");
}

$| = 1;
close(STDIN);
my ($buffer, $writeFlag, $line, @date, $today, $now);

while ( $ExitFlag == 0) {
	$buffer = "";
	$writeFlag = 0;
	$pid = open(SNMP, "/usr/sbin/snmptrapd -f -Lo -Oa -M /usr/share/snmp/mibs -m ALL -p /var/run/snmptrapd.pid -F \"<030>%y-%0.2m-%0.2lT%0.2h:%0.2j:%0.2k %B snmptrap: %N %W Trap (%q) Uptime: %T seconds\t%V\t%v\013\n\" $snmptrapIP|");
	while ( $line = <SNMP> ) {
		chomp $line;
		if ($line =~ m/^<\d+>/) {
			while ( ! $line =~ m/\013$/ ) {
				$buffer .= "$line ";
				$line = <SNMP>;
				chomp $line;
			}
			chomp $line;
			$buffer .= "$line";
			# remove all characters other than tab, space and printable.
			$buffer =~ s/[^!-~\t ]//g;
			writeBuff($buffer);
			$buffer = "";
		} else {
			FLARE::Logger::message("info","$line");
		}
	}
	close(SNMP);
	if ($ExitFlag == 1) {
		FLARE::Logger::message("info","stopping");
	} else {
		FLARE::Logger::message("info","snmptrapd exited - restart in 5 seconds");
		sleep 5;
	}
}
unlink $pidFile;
FLARE::Logger::message("info","stopped");

exit 0;