Added contrib folder with file_processor utility which is a plugin framework for reading the files-json.log and processing and taking action based on the files observed.

contrib/file_processor/Action/Log.pm

@@ -0,0 +1,15 @@
+package Action::Log;
+use Moose;
+extends 'Processor';
+
+has 'data' => (is => 'rw', isa => 'HashRef', required => 1);
+
+sub name { 'log' }
+sub description { 'Log to file' }
+
+sub perform {
+	my $self = shift;
+	$self->log->info($self->json->encode($self->data));
+}
+
+1

contrib/file_processor/Processor/Anubis.pm

@@ -0,0 +1,33 @@
+package Processor::Anubis;
+use Moose;
+extends 'Processor';
+use Data::Dumper;
+use LWP::UserAgent;
+
+has 'md5' => (is => 'ro', isa => 'Str', required => 1);
+has 'ua' => (is => 'rw', isa => 'LWP::UserAgent', required => 1, default => sub { return LWP::UserAgent->new(agent => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1'); });
+has 'url_template' => (is => 'ro', isa => 'Str', required => 1, default => 'http://anubis.iseclab.org/?action=result&task_id=%s');
+sub name { 'Anubis' }
+sub description { 'Processor for anubis.iseclab.org' }
+
+sub process {
+	my $self = shift;
+	my $url = sprintf($self->url_template, $self->md5);
+	$self->log->debug('Getting url ' . $url);
+	my $response = $self->ua->get($url);
+	#$self->log->debug(Dumper($response));
+	if ($response->code eq 200){
+		if ($response->decoded_content =~ /Invalid Task ID/){
+			$self->log->debug('No result');
+			return 0;
+		}
+		$self->log->info('Got result');
+		return $url;
+	}
+	else {
+		$self->log->debug('Communications failure: ' . Dumper($response));
+		return 0;
+	}
+}
+
+1

contrib/file_processor/Processor/Malwr.pm

@@ -0,0 +1,32 @@
+package Processor::Malwr;
+use Moose;
+extends 'Processor';
+use Data::Dumper;
+use LWP::UserAgent;
+
+has 'md5' => (is => 'ro', isa => 'Str', required => 1);
+has 'ua' => (is => 'rw', isa => 'LWP::UserAgent', required => 1, default => sub { return LWP::UserAgent->new(agent => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1'); });
+has 'url_template' => (is => 'ro', isa => 'Str', required => 1, default => 'http://malwr.com/analysis/%s/');
+sub name { 'Malwr' }
+sub description { 'Processor for Malwr.com' }
+
+sub process {
+	my $self = shift;
+	my $url = sprintf($self->url_template, $self->md5);
+	$self->log->debug('Getting url ' . $url);
+	my $response = $self->ua->get($url);
+	if ($response->code eq 200){
+		if ($response->decoded_content =~ /Cannot find analysis with specified ID or MD5/){
+			$self->log->debug('No result');
+			return 0;
+		}
+		$self->log->info('Got malwr.com result');
+		return $url;
+	}
+	else {
+		$self->log->debug('Communications failure: ' . Dumper($response));
+		return 0;
+	}
+}
+
+1

contrib/file_processor/Processor/ThreatExpert.pm

@@ -0,0 +1,33 @@
+package Processor::ThreatExpert;
+use Moose;
+extends 'Processor';
+use Data::Dumper;
+use LWP::UserAgent;
+
+has 'md5' => (is => 'ro', isa => 'Str', required => 1);
+has 'ua' => (is => 'rw', isa => 'LWP::UserAgent', required => 1, default => sub { return LWP::UserAgent->new(agent => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1'); });
+has 'url_template' => (is => 'ro', isa => 'Str', required => 1, default => 'http://www.threatexpert.com/report.aspx?md5=%s');
+sub name { 'ThreatExpert' }
+sub description { 'Processor for threatexpert.com' }
+
+sub process {
+	my $self = shift;
+	my $url = sprintf($self->url_template, $self->md5);
+	$self->log->debug('Getting url ' . $url);
+	my $response = $self->ua->get($url);
+	#$self->log->debug(Dumper($response));
+	if ($response->code eq 200){
+		if ($response->decoded_content =~ /Search All Reports/){
+			$self->log->debug('No result');
+			return 0;
+		}
+		$self->log->info('Got result');
+		return $url;
+	}
+	else {
+		$self->log->debug('Communications failure: ' . Dumper($response));
+		return 0;
+	}
+}
+
+1

contrib/file_processor/Processor/VirusTotal.pm

@@ -0,0 +1,39 @@
+package Processor::VirusTotal;
+use Moose;
+extends 'Processor';
+use Data::Dumper;
+use LWP::UserAgent;
+
+has 'md5' => (is => 'ro', isa => 'Str', required => 1);
+has 'ua' => (is => 'rw', isa => 'LWP::UserAgent', required => 1, default => sub { return LWP::UserAgent->new(agent => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1'); });
+has 'url' => (is => 'ro', isa => 'Str', required => 1, default => 'https://www.virustotal.com/vtapi/v2/file/report');
+sub name { 'VirusTotal' }
+sub description { 'Processor for virustotal.com' }
+
+sub process {
+	my $self = shift;
+	unless ($self->conf->{virustotal_apikey}){
+		warn('No VirusTotal apikey configured in config file');
+		return 0;
+	}
+	$self->log->debug('Getting url ' . $self->url);
+	#$self->log->debug('md5: ' . $self->md5 . ', apikey: ' . $self->conf->{virustotal_apikey});
+	my $response = $self->ua->post($self->url, { resource => $self->md5, apikey => $self->conf->{virustotal_apikey} });
+	#$self->log->debug(Dumper($response));
+	if ($response->code eq 200){
+		my $data = $self->json->decode($response->decoded_content);
+		$self->log->debug('data: ' . Dumper($data));
+		if ($data->{positives}){
+			return $data;
+		}
+		else {
+			return 0;
+		}
+	}
+	else {
+		$self->log->debug('Communications failure: ' . Dumper($response));
+		return 0;
+	}
+}
+
+1

contrib/file_processor/README

@@ -0,0 +1,3 @@
+This directory contains what's needed for reading the JSON file /var/log/suricata/files-json.log and processing those entries against plugins.  Included are plugins for checking the MD5 of the observed file on the network against already created reports on anubis.iseclab.org, malwr.com, and threatexpert.com.  If you have a virustotal.com API key (free, though see the terms of use on virustotal.com/documentation/public-api/), you can enable the virustotal.com plugin and configure your API key so you can check the MD5 against over forty AV vendors' results.
+
+To create new plugins, use the existing modules as a guide.  Drop a new file with the .pm extension in either the Processor or Action directory, depending on what kind of plugin it is.  Processor plugins add information to the data.  Action plugins do something with the data once all of the information is available.  A simple logging demo has been included, but many different kinds of action plugins could be written to do things like submit full files to a sandbox, send an email, log to a database, send an SNMP trap, etc.

contrib/file_processor/file_processor.conf

@@ -0,0 +1,14 @@
+{
+	"logdir": "/var/log/suricata",
+	"debug_level": "INFO",
+	#"virustotal_apikey": "xxx"
+	"actions": {
+		"Action::Log": 1
+	},
+	"processors": {
+		"Processor::Anubis": 1,
+		"Processor::Malwr": 1,
+		"Processor::ThreatExpert": 1,
+		#"Processor::VirusTotal": 1
+	}
+}

contrib/file_processor/file_processor.pl

@@ -0,0 +1,135 @@
+package Processor;
+use Moose;
+use Data::Dumper;
+use Module::Pluggable search_path => qw(Processor), sub_name => 'processors';
+use Module::Pluggable search_path => qw(Action), sub_name => 'actions';
+use Log::Log4perl;
+use JSON;
+
+has 'conf' => (is => 'rw', isa => 'HashRef', required => 1);
+has 'log' => (is => 'rw', isa => 'Object', required => 1);
+has 'json' => (is => 'ro', isa => 'JSON', required => 1, default => sub { return JSON->new->pretty->allow_blessed });
+
+sub BUILD {
+	my $self = shift;
+
+	foreach my $processor_plugin ($self->processors){
+		next unless exists $self->conf->{processors}->{$processor_plugin};
+		eval qq{require $processor_plugin};
+		$self->log->info('Using processor plugin ' . $processor_plugin->description);
+	}
+
+	foreach my $action_plugin ($self->actions){
+		next unless exists $self->conf->{actions}->{$action_plugin};
+		eval qq{require $action_plugin};
+		$self->log->info('Using action plugin ' . $action_plugin->description);
+	}
+}
+
+sub process {
+	my $self = shift;
+	my $line = shift;
+	#$self->log->debug('got line ' . $line);
+	eval {
+		my $data = $self->json->decode($line);
+		return unless $data->{md5};
+		$data->{processors} = {};
+		foreach my $processor_plugin ($self->processors){
+			next unless exists $self->conf->{processors}->{$processor_plugin};
+			my $processor = $processor_plugin->new(conf => $self->conf, log => $self->log, md5 => $data->{md5});
+			$self->log->debug('processing with plugin ' . $processor->description);
+			$data->{processors}->{ $processor->name } = $processor->process();
+		}
+		#$self->log->debug('data: ' . Dumper($data));
+		foreach my $action_plugin ($self->actions){
+			next unless exists $self->conf->{actions}->{$action_plugin};
+			my $action = $action_plugin->new(conf => $self->conf, log => $self->log, data => $data);
+			$self->log->debug('performing action with plugin ' . $action->description);
+			$action->perform();
+		}
+	};
+	if ($@){
+		$self->log->error('Error: ' . $@ . ', processing line: ' . $line);
+	}
+}
+
+package main;
+use strict;
+use Getopt::Std;
+use FindBin;
+use Config::JSON;
+use File::Tail;
+
+# Include the directory this script is in
+use lib $FindBin::Bin;
+
+my %Opts;
+getopts('c:', \%Opts);
+
+my $conf_file = $Opts{c} ? $Opts{c} : '/etc/suricata/file_processor.conf';
+my $Conf = {
+	logdir => '/tmp',
+	debug_level => 'TRACE',
+	actions => {
+		'Action::Log' => 1
+	},
+	processors => {
+		'Processor::Anubis' => 1,
+		'Processor::Malwr' => 1,
+		'Processor::ThreatExpert' => 1,
+	}
+};
+if (-f $conf_file){
+	$Conf = Config::JSON->new( $conf_file );
+	$Conf = $Conf->{config}; # native hash is 10x faster than using Config::JSON->get()
+}
+
+# Setup logger
+my $logdir = $Conf->{logdir} ? $Conf->{logdir} : '/var/log/suricata';
+my $debug_level = $Conf->{debug_level} ? $Conf->{debug_level} : 'TRACE';
+my $l4pconf = qq(
+	log4perl.category.App       = $debug_level, File, Screen
+	log4perl.appender.File			 = Log::Log4perl::Appender::File
+	log4perl.appender.File.filename  = $logdir/file_processor.log
+	log4perl.appender.File.syswrite = 1
+	log4perl.appender.File.recreate = 1
+	log4perl.appender.File.layout = Log::Log4perl::Layout::PatternLayout
+	log4perl.appender.File.layout.ConversionPattern = * %p [%d] %F (%L) %M %P %m%n
+	log4perl.filter.ScreenLevel               = Log::Log4perl::Filter::LevelRange
+	log4perl.filter.ScreenLevel.LevelMin  = $debug_level
+	log4perl.filter.ScreenLevel.LevelMax  = ERROR
+	log4perl.filter.ScreenLevel.AcceptOnMatch = true
+	log4perl.appender.Screen         = Log::Log4perl::Appender::Screen
+	log4perl.appender.Screen.Filter = ScreenLevel
+	log4perl.appender.Screen.stderr  = 1
+	log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
+	log4perl.appender.Screen.layout.ConversionPattern = * %p [%d] %F (%L) %M %P %m%n
+);
+Log::Log4perl::init( \$l4pconf ) or die("Unable to init logger\n");
+my $Log = Log::Log4perl::get_logger('App') or die("Unable to init logger\n");
+
+my $processor = new Processor(conf => $Conf, log => $Log);
+
+my $file = $Conf->{file} ? $Conf->{file} : '/var/log/suricata/files-json.log';
+my $tail = new File::Tail(name => $file, maxinterval => 1);
+
+while (my $line = $tail->read){
+	$processor->process($line);
+}
+
+__END__
+Example config file /etc/suricata/file_processor.conf
+{
+	"logdir": "/var/log/suricata",
+	"debug_level": "INFO",
+	"virustotal_apikey": "xxx"
+	"actions": {
+		"Action::Log": 1
+	},
+	"processors": {
+		"Processor::Anubis": 1,
+		"Processor::Malwr": 1,
+		"Processor::ThreatExpert": 1,
+		"Processor::VirusTotal": 1
+	}
+}