mirror of https://github.com/OISF/suricata
datasets: match on lists of data
Datasets are sets/lists of data that can be accessed or added from the rule language. This patch implements 3 data types: 1. string (or buffer) 2. md5 3. sha256 The patch also implements 2 new rule keywords: 1. dataset 2. datarep The dataset keyword allows matching against a list of values to see if it exists or not. It can also add the value to the set. The set can optionally be stored to disk on exit. The datarep support matching/lookups only. With each item in the set a reputation value is stored and this value can be matched against. The reputation value is unsigned 16 bit, so values can be between 0 and 65535. Datasets can be registered in 2 ways: 1. through the yaml 2. through the rules The goal of this rules based approach is that rule writers can start using this without the need for config changes. A dataset is implemented using a thash hash table. Each dataset is its own separate thash.pull/4166/head
Victor Julien
8 years ago
parent
b286c14324
commit
317376f59d
@ -0,0 +1,64 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "conf.h"
|
||||
#include "datasets.h"
|
||||
#include "datasets-md5.h"
|
||||
#include "util-thash.h"
|
||||
#include "util-print.h"
|
||||
#include "util-crypt.h" // encode base64
|
||||
#include "util-base64.h" // decode base64
|
||||
|
||||
int Md5StrSet(void *dst, void *src)
|
||||
{
|
||||
Md5Type *src_s = src;
|
||||
Md5Type *dst_s = dst;
|
||||
memcpy(dst_s->md5, src_s->md5, sizeof(dst_s->md5));
|
||||
dst_s->rep = src_s->rep;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Md5StrCompare(void *a, void *b)
|
||||
{
|
||||
const Md5Type *as = a;
|
||||
const Md5Type *bs = b;
|
||||
|
||||
return (memcmp(as->md5, bs->md5, sizeof(as->md5)) == 0);
|
||||
}
|
||||
|
||||
uint32_t Md5StrHash(void *s)
|
||||
{
|
||||
const Md5Type *str = s;
|
||||
uint32_t hash = 5381;
|
||||
|
||||
for (int i = 0; i < (int)sizeof(str->md5); i++) {
|
||||
hash = ((hash << 5) + hash) + str->md5[i]; /* hash * 33 + c */
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
// data stays in hash
|
||||
void Md5StrFree(void *s)
|
||||
{
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DATASETS_MD5_H__
|
||||
#define __DATASETS_MD5_H__
|
||||
|
||||
#include "datasets-reputation.h"
|
||||
|
||||
typedef struct Md5Type {
|
||||
uint8_t md5[16];
|
||||
DataRepType rep;
|
||||
} Md5Type;
|
||||
|
||||
int Md5StrSet(void *dst, void *src);
|
||||
bool Md5StrCompare(void *a, void *b);
|
||||
uint32_t Md5StrHash(void *s);
|
||||
void Md5StrFree(void *s);
|
||||
|
||||
#endif /* __DATASETS_MD5_H__ */
|
||||
@ -0,0 +1,36 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DATASETS_REPUTATION_H__
|
||||
#define __DATASETS_REPUTATION_H__
|
||||
|
||||
typedef struct DataRepType {
|
||||
uint16_t value;
|
||||
} DataRepType;
|
||||
|
||||
typedef struct DataRepResultType {
|
||||
bool found;
|
||||
DataRepType rep;
|
||||
} DataRepResultType;
|
||||
|
||||
#endif /* __DATASETS_REPUTATION_H__ */
|
||||
@ -0,0 +1,65 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "conf.h"
|
||||
#include "datasets.h"
|
||||
#include "datasets-sha256.h"
|
||||
#include "util-thash.h"
|
||||
#include "util-print.h"
|
||||
#include "util-crypt.h" // encode base64
|
||||
#include "util-base64.h" // decode base64
|
||||
|
||||
int Sha256StrSet(void *dst, void *src)
|
||||
{
|
||||
Sha256Type *src_s = src;
|
||||
Sha256Type *dst_s = dst;
|
||||
memcpy(dst_s->sha256, src_s->sha256, sizeof(dst_s->sha256));
|
||||
dst_s->rep = src_s->rep;
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Sha256StrCompare(void *a, void *b)
|
||||
{
|
||||
Sha256Type *as = a;
|
||||
Sha256Type *bs = b;
|
||||
|
||||
return (memcmp(as->sha256, bs->sha256, sizeof(as->sha256)) == 0);
|
||||
}
|
||||
|
||||
uint32_t Sha256StrHash(void *s)
|
||||
{
|
||||
Sha256Type *str = s;
|
||||
uint32_t hash = 5381;
|
||||
|
||||
for (int i = 0; i < (int)sizeof(str->sha256); i++) {
|
||||
hash = ((hash << 5) + hash) + str->sha256[i]; /* hash * 33 + c */
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
// data stays in hash
|
||||
void Sha256StrFree(void *s)
|
||||
{
|
||||
// no dynamic data
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DATASETS_SHA256_H__
|
||||
#define __DATASETS_SHA256_H__
|
||||
|
||||
#include "datasets-reputation.h"
|
||||
|
||||
typedef struct Sha256Type {
|
||||
uint8_t sha256[32];
|
||||
DataRepType rep;
|
||||
} Sha256Type;
|
||||
|
||||
int Sha256StrSet(void *dst, void *src);
|
||||
bool Sha256StrCompare(void *a, void *b);
|
||||
uint32_t Sha256StrHash(void *s);
|
||||
void Sha256StrFree(void *s);
|
||||
|
||||
#endif /* __DATASETS_SHA256_H__ */
|
||||
@ -0,0 +1,104 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "conf.h"
|
||||
#include "datasets.h"
|
||||
#include "datasets-string.h"
|
||||
#include "util-thash.h"
|
||||
#include "util-print.h"
|
||||
#include "util-crypt.h" // encode base64
|
||||
#include "util-base64.h" // decode base64
|
||||
|
||||
#if 0
|
||||
static int StringAsAscii(const void *s, char *out, size_t out_size)
|
||||
{
|
||||
const StringType *str = s;
|
||||
uint32_t offset = 0;
|
||||
PrintRawUriBuf(out, &offset, out_size, str->ptr, str->len);
|
||||
if (out[0] == '\0')
|
||||
return 0;
|
||||
strlcat(out, "\n", out_size);
|
||||
return strlen(out);
|
||||
}
|
||||
#endif
|
||||
|
||||
int StringAsBase64(const void *s, char *out, size_t out_size)
|
||||
{
|
||||
const StringType *str = s;
|
||||
|
||||
unsigned long len = out_size;
|
||||
uint8_t encoded_data[str->len * 2];
|
||||
if (Base64Encode((unsigned char *)str->ptr, str->len,
|
||||
encoded_data, &len) != SC_BASE64_OK)
|
||||
return 0;
|
||||
|
||||
strlcpy(out, (const char *)encoded_data, out_size);
|
||||
strlcat(out, "\n", out_size);
|
||||
return strlen(out);
|
||||
}
|
||||
|
||||
int StringSet(void *dst, void *src)
|
||||
{
|
||||
StringType *src_s = src;
|
||||
StringType *dst_s = dst;
|
||||
SCLogDebug("dst %p src %p, src_s->ptr %p src_s->len %u", dst, src, src_s->ptr, src_s->len);
|
||||
|
||||
dst_s->len = src_s->len;
|
||||
dst_s->ptr = SCMalloc(dst_s->len);
|
||||
BUG_ON(dst_s->ptr == NULL);
|
||||
memcpy(dst_s->ptr, src_s->ptr, dst_s->len);
|
||||
|
||||
dst_s->rep = src_s->rep;
|
||||
SCLogDebug("dst %p src %p, dst_s->ptr %p dst_s->len %u", dst, src, dst_s->ptr, dst_s->len);
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool StringCompare(void *a, void *b)
|
||||
{
|
||||
const StringType *as = a;
|
||||
const StringType *bs = b;
|
||||
|
||||
if (as->len != bs->len)
|
||||
return false;
|
||||
|
||||
return (memcmp(as->ptr, bs->ptr, as->len) == 0);
|
||||
}
|
||||
|
||||
uint32_t StringHash(void *s)
|
||||
{
|
||||
uint32_t hash = 5381;
|
||||
int c;
|
||||
|
||||
while ((c = *(char *)s++))
|
||||
hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
// base data stays in hash
|
||||
void StringFree(void *s)
|
||||
{
|
||||
StringType *str = s;
|
||||
SCFree(str->ptr);
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DATASETS_STRING_H__
|
||||
#define __DATASETS_STRING_H__
|
||||
|
||||
#include "datasets-reputation.h"
|
||||
|
||||
typedef struct StringType {
|
||||
uint32_t len;
|
||||
DataRepType rep;
|
||||
uint8_t *ptr;
|
||||
} StringType;
|
||||
|
||||
int StringSet(void *dst, void *src);
|
||||
bool StringCompare(void *a, void *b);
|
||||
uint32_t StringHash(void *s);
|
||||
void StringFree(void *s);
|
||||
int StringAsBase64(const void *s, char *out, size_t out_size);
|
||||
|
||||
#endif /* __DATASETS_STRING_H__ */
|
||||
@ -0,0 +1,938 @@
|
||||
/* Copyright (C) 2017-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "conf.h"
|
||||
#include "datasets.h"
|
||||
#include "datasets-string.h"
|
||||
#include "datasets-md5.h"
|
||||
#include "datasets-sha256.h"
|
||||
#include "datasets-reputation.h"
|
||||
#include "util-thash.h"
|
||||
#include "util-print.h"
|
||||
#include "util-crypt.h" // encode base64
|
||||
#include "util-base64.h" // decode base64
|
||||
|
||||
SCMutex sets_lock = SCMUTEX_INITIALIZER;
|
||||
static Dataset *sets = NULL;
|
||||
static uint32_t set_ids = 0;
|
||||
|
||||
static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
DataRepType *rep);
|
||||
|
||||
static Dataset *DatasetAlloc(const char *name)
|
||||
{
|
||||
Dataset *set = SCCalloc(1, sizeof(*set));
|
||||
if (set) {
|
||||
set->id = set_ids++;
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
static Dataset *DatasetSearchByName(const char *name)
|
||||
{
|
||||
Dataset *set = sets;
|
||||
while (set) {
|
||||
if (strcasecmp(name, set->name) == 0) {
|
||||
return set;
|
||||
}
|
||||
set = set->next;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Dataset *DatasetGetByName(const char *name)
|
||||
{
|
||||
Dataset *set = DatasetSearchByName(name);
|
||||
if (set)
|
||||
return set;
|
||||
|
||||
return DatasetAlloc(name);
|
||||
}
|
||||
|
||||
|
||||
static int HexToRaw(const uint8_t *in, size_t ins, uint8_t *out, size_t outs)
|
||||
{
|
||||
if (ins % 2 != 0)
|
||||
return -1;
|
||||
if (outs != ins / 2)
|
||||
return -1;
|
||||
|
||||
uint8_t hash[outs];
|
||||
size_t i, x;
|
||||
for (x = 0, i = 0; i < ins; i+=2, x++) {
|
||||
char buf[3] = { 0, 0, 0 };
|
||||
buf[0] = in[i];
|
||||
buf[1] = in[i+1];
|
||||
|
||||
long value = strtol(buf, NULL, 16);
|
||||
if (value >= 0 && value <= 255)
|
||||
hash[x] = (uint8_t)value;
|
||||
else {
|
||||
SCLogError(SC_ERR_INVALID_HASH, "hash byte out of range %ld", value);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(out, hash, outs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ParseRepLine(const char *in, size_t ins, DataRepType *rep_out)
|
||||
{
|
||||
SCLogDebug("in '%s'", in);
|
||||
char raw[ins + 1];
|
||||
memcpy(raw, in, ins);
|
||||
raw[ins] = '\0';
|
||||
char *line = raw;
|
||||
|
||||
char *ptrs[1] = {NULL};
|
||||
int idx = 0;
|
||||
|
||||
size_t i = 0;
|
||||
while (i < ins + 1) {
|
||||
if (line[i] == ',' || line[i] == '\n' || line[i] == '\0') {
|
||||
line[i] = '\0';
|
||||
SCLogDebug("line '%s'", line);
|
||||
|
||||
ptrs[idx] = line;
|
||||
idx++;
|
||||
|
||||
if (idx == 1)
|
||||
break;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (idx != 1) {
|
||||
SCLogDebug("idx %d", idx);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int v = atoi(ptrs[0]);
|
||||
if (v < 0 || v > USHRT_MAX) {
|
||||
SCLogDebug("v %d", v);
|
||||
return -1;
|
||||
}
|
||||
SCLogDebug("v %d raw %s", v, ptrs[0]);
|
||||
|
||||
rep_out->value = v;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DatasetLoadMd5(Dataset *set)
|
||||
{
|
||||
if (strlen(set->load) == 0)
|
||||
return 0;
|
||||
|
||||
SCLogNotice("dataset: %s loading from '%s'", set->name, set->load);
|
||||
|
||||
FILE *fp = fopen(set->load, "r");
|
||||
if (fp == NULL) {
|
||||
SCLogError(SC_ERR_DATASET, "fopen '%s' failed: %s",
|
||||
set->load, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint32_t cnt = 0;
|
||||
char line[1024];
|
||||
while (fgets(line, (int)sizeof(line), fp) != NULL) {
|
||||
/* straight black/white list */
|
||||
if (strlen(line) == 33) {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogDebug("line: '%s'", line);
|
||||
|
||||
uint8_t hash[16];
|
||||
if (HexToRaw((const uint8_t *)line, 32, hash, sizeof(hash)) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die");
|
||||
|
||||
if (DatasetAdd(set, (const uint8_t *)hash, 16) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die");
|
||||
cnt++;
|
||||
|
||||
/* list with rep data */
|
||||
} else if (strlen(line) > 33 && line[32] == ',') {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogDebug("MD5 with REP line: '%s'", line);
|
||||
|
||||
uint8_t hash[16];
|
||||
if (HexToRaw((const uint8_t *)line, 32, hash, sizeof(hash)) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad hash");
|
||||
|
||||
DataRepType rep = { .value = 0};
|
||||
if (ParseRepLine(line+33, strlen(line)-33, &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad rep");
|
||||
|
||||
SCLogDebug("rep v:%u", rep.value);
|
||||
if (DatasetAddwRep(set, hash, 16, &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: add failed");
|
||||
|
||||
cnt++;
|
||||
}
|
||||
else {
|
||||
SCLogNotice("MD5 bad line len %u: %s", (uint32_t)strlen(line), line);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
SCLogNotice("dataset: %s loaded %u records", set->name, cnt);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DatasetLoadSha256(Dataset *set)
|
||||
{
|
||||
if (strlen(set->load) == 0)
|
||||
return 0;
|
||||
|
||||
SCLogNotice("dataset: %s loading from '%s'", set->name, set->load);
|
||||
|
||||
FILE *fp = fopen(set->load, "r");
|
||||
if (fp == NULL) {
|
||||
SCLogError(SC_ERR_DATASET, "fopen '%s' failed: %s",
|
||||
set->load, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint32_t cnt = 0;
|
||||
char line[1024];
|
||||
while (fgets(line, (int)sizeof(line), fp) != NULL) {
|
||||
/* straight black/white list */
|
||||
if (strlen(line) == 65) {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogDebug("line: '%s'", line);
|
||||
|
||||
uint8_t hash[32];
|
||||
if (HexToRaw((const uint8_t *)line, 64, hash, sizeof(hash)) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die");
|
||||
|
||||
if (DatasetAdd(set, (const uint8_t *)hash, (uint32_t)32) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die");
|
||||
cnt++;
|
||||
|
||||
/* list with rep data */
|
||||
} else if (strlen(line) > 65 && line[64] == ',') {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogNotice("SHA-256 with REP line: '%s'", line);
|
||||
|
||||
uint8_t hash[32];
|
||||
if (HexToRaw((const uint8_t *)line, 64, hash, sizeof(hash)) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad hash");
|
||||
|
||||
DataRepType rep = { .value = 0 };
|
||||
if (ParseRepLine(line+65, strlen(line)-65, &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad rep");
|
||||
SCLogNotice("rep %u", rep.value);
|
||||
|
||||
if (DatasetAddwRep(set, hash, 32, &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: add failed");
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
SCLogNotice("dataset: %s loaded %u records", set->name, cnt);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DatasetLoadString(Dataset *set)
|
||||
{
|
||||
if (strlen(set->load) == 0)
|
||||
return 0;
|
||||
|
||||
SCLogNotice("dataset: %s loading from '%s'", set->name, set->load);
|
||||
|
||||
FILE *fp = fopen(set->load, "r");
|
||||
if (fp == NULL) {
|
||||
SCLogError(SC_ERR_DATASET, "fopen '%s' failed: %s",
|
||||
set->load, strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint32_t cnt = 0;
|
||||
char line[1024];
|
||||
while (fgets(line, (int)sizeof(line), fp) != NULL) {
|
||||
if (strlen(line) <= 1)
|
||||
continue;
|
||||
|
||||
char *r = strchr(line, ',');
|
||||
if (r == NULL) {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogDebug("line: '%s'", line);
|
||||
|
||||
uint8_t decoded[strlen(line)];
|
||||
uint32_t len = DecodeBase64(decoded, (const uint8_t *)line, strlen(line), 1);
|
||||
if (len == 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad base64 encoding");
|
||||
|
||||
if (DatasetAdd(set, (const uint8_t *)decoded, len) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die");
|
||||
cnt++;
|
||||
} else {
|
||||
line[strlen(line) - 1] = '\0';
|
||||
SCLogDebug("line: '%s'", line);
|
||||
|
||||
*r = '\0';
|
||||
|
||||
uint8_t decoded[strlen(line)];
|
||||
uint32_t len = DecodeBase64(decoded, (const uint8_t *)line, strlen(line), 1);
|
||||
if (len == 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad base64 encoding");
|
||||
|
||||
r++;
|
||||
SCLogNotice("r '%s'", r);
|
||||
|
||||
DataRepType rep = { .value = 0 };
|
||||
if (ParseRepLine(r, strlen(r), &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: bad rep");
|
||||
SCLogNotice("rep %u", rep.value);
|
||||
|
||||
if (DatasetAddwRep(set, (const uint8_t *)decoded, len, &rep) < 0)
|
||||
FatalError(SC_ERR_FATAL, "die: insert failed");
|
||||
cnt++;
|
||||
|
||||
SCLogNotice("line with rep %s, %s", line, r);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
SCLogNotice("dataset: %s loaded %u records", set->name, cnt);
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern bool g_system;
|
||||
|
||||
enum DatasetGetPathType {
|
||||
TYPE_STATE,
|
||||
TYPE_LOAD,
|
||||
};
|
||||
|
||||
static void DatasetGetPath(const char *in_path,
|
||||
char *out_path, size_t out_size, enum DatasetGetPathType type)
|
||||
{
|
||||
char path[PATH_MAX];
|
||||
struct stat st;
|
||||
int ret;
|
||||
|
||||
if (PathIsAbsolute(in_path)) {
|
||||
strlcpy(path, in_path, sizeof(path));
|
||||
strlcpy(out_path, path, out_size);
|
||||
return;
|
||||
}
|
||||
|
||||
const char *data_dir = ConfigGetDataDirectory();
|
||||
if ((ret = stat(data_dir, &st)) != 0) {
|
||||
SCLogNotice("data-dir '%s': %s", data_dir, strerror(errno));
|
||||
return;
|
||||
}
|
||||
|
||||
snprintf(path, sizeof(path), "%s/%s", data_dir, in_path); // TODO WINDOWS
|
||||
|
||||
if (type == TYPE_LOAD) {
|
||||
if ((ret = stat(path, &st)) != 0) {
|
||||
SCLogNotice("path %s: %s", path, strerror(errno));
|
||||
if (!g_system) {
|
||||
snprintf(path, sizeof(path), "%s", in_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
strlcpy(out_path, path, out_size);
|
||||
SCLogNotice("in_path \'%s\' => \'%s\'", in_path, out_path);
|
||||
}
|
||||
|
||||
Dataset *DatasetGet(const char *name, enum DatasetTypes type,
|
||||
const char *save, const char *load)
|
||||
{
|
||||
SCMutexLock(&sets_lock);
|
||||
Dataset *set = DatasetSearchByName(name);
|
||||
if (set) {
|
||||
if (type != DATASET_TYPE_NOTSET && set->type != type) {
|
||||
SCLogNotice("dataset %s already exists and is of type %u",
|
||||
set->name, set->type);
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
if ((save == NULL || strlen(save) == 0) &&
|
||||
(load == NULL || strlen(load) == 0)) {
|
||||
// OK, rule keyword doesn't have to set state/load,
|
||||
// even when yaml set has set it.
|
||||
} else {
|
||||
if ((save == NULL && strlen(set->save) > 0) ||
|
||||
(save != NULL && strcmp(set->save, save) != 0)) {
|
||||
SCLogError(SC_ERR_DATASET, "dataset %s save mismatch: %s != %s",
|
||||
set->name, set->save, save);
|
||||
goto out_err;
|
||||
}
|
||||
if ((load == NULL && strlen(set->load) > 0) ||
|
||||
(load != NULL && strcmp(set->load, load) != 0)) {
|
||||
SCLogError(SC_ERR_DATASET, "dataset %s load mismatch: %s != %s",
|
||||
set->name, set->load, load);
|
||||
goto out_err;
|
||||
}
|
||||
}
|
||||
|
||||
SCMutexUnlock(&sets_lock);
|
||||
return set;
|
||||
} else {
|
||||
if (type == DATASET_TYPE_NOTSET) {
|
||||
SCLogError(SC_ERR_DATASET, "dataset %s not defined", name);
|
||||
goto out_err;
|
||||
}
|
||||
}
|
||||
|
||||
set = DatasetAlloc(name);
|
||||
if (set == NULL) {
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
strlcpy(set->name, name, sizeof(set->name));
|
||||
set->type = type;
|
||||
if (save && strlen(save)) {
|
||||
strlcpy(set->save, save, sizeof(set->save));
|
||||
SCLogDebug("name %s save '%s'", name, set->save);
|
||||
}
|
||||
if (load && strlen(load)) {
|
||||
strlcpy(set->load, load, sizeof(set->load));
|
||||
SCLogDebug("set \'%s\' loading \'%s\' from \'%s\'", set->name, load, set->load);
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case DATASET_TYPE_MD5:
|
||||
set->hash = THashInit(name, sizeof(Md5Type), Md5StrSet,
|
||||
Md5StrFree, Md5StrHash, Md5StrCompare);
|
||||
if (set->hash == NULL)
|
||||
goto out_err;
|
||||
if (DatasetLoadMd5(set) < 0)
|
||||
goto out_err;
|
||||
break;
|
||||
case DATASET_TYPE_STRING:
|
||||
set->hash = THashInit(name, sizeof(StringType), StringSet,
|
||||
StringFree, StringHash, StringCompare);
|
||||
if (set->hash == NULL)
|
||||
goto out_err;
|
||||
if (DatasetLoadString(set) < 0)
|
||||
goto out_err;
|
||||
break;
|
||||
case DATASET_TYPE_SHA256:
|
||||
set->hash = THashInit(name, sizeof(Sha256Type), Sha256StrSet,
|
||||
Sha256StrFree, Sha256StrHash, Sha256StrCompare);
|
||||
if (set->hash == NULL)
|
||||
goto out_err;
|
||||
if (DatasetLoadSha256(set) < 0)
|
||||
goto out_err;
|
||||
break;
|
||||
}
|
||||
|
||||
SCLogDebug("set %p/%s type %u save %s load %s",
|
||||
set, set->name, set->type, set->save, set->load);
|
||||
|
||||
set->next = sets;
|
||||
sets = set;
|
||||
|
||||
SCMutexUnlock(&sets_lock);
|
||||
return set;
|
||||
out_err:
|
||||
if (set) {
|
||||
if (set->hash) {
|
||||
THashShutdown(set->hash);
|
||||
}
|
||||
SCFree(set);
|
||||
}
|
||||
SCMutexUnlock(&sets_lock);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#define SETNAME_MAX 63
|
||||
|
||||
int DatasetsInit(void)
|
||||
{
|
||||
SCLogDebug("datasets start");
|
||||
int n = 0;
|
||||
ConfNode *datasets = ConfGetNode("datasets");
|
||||
if (datasets != NULL) {
|
||||
int list_pos = 0;
|
||||
ConfNode *iter = NULL;
|
||||
TAILQ_FOREACH(iter, &datasets->head, next) {
|
||||
if (iter->val == NULL) {
|
||||
list_pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
char save[PATH_MAX] = "";
|
||||
char load[PATH_MAX] = "";
|
||||
|
||||
const char *set_name = iter->val;
|
||||
if (strlen(set_name) > SETNAME_MAX) {
|
||||
FatalError(SC_ERR_CONF_NAME_TOO_LONG, "set name '%s' too long, max %d chars",
|
||||
set_name, SETNAME_MAX);
|
||||
}
|
||||
|
||||
ConfNode *set = ConfNodeLookupChild(iter, set_name);
|
||||
if (set == NULL) {
|
||||
list_pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
ConfNode *set_type =
|
||||
ConfNodeLookupChild(set, "type");
|
||||
if (set_type == NULL) {
|
||||
list_pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
ConfNode *set_save =
|
||||
ConfNodeLookupChild(set, "state");
|
||||
if (set_save) {
|
||||
DatasetGetPath(set_save->val, save, sizeof(save), TYPE_STATE);
|
||||
strlcpy(load, save, sizeof(load));
|
||||
} else {
|
||||
ConfNode *set_load =
|
||||
ConfNodeLookupChild(set, "load");
|
||||
if (set_load) {
|
||||
DatasetGetPath(set_load->val, load, sizeof(load), TYPE_LOAD);
|
||||
}
|
||||
}
|
||||
|
||||
char conf_str[1024];
|
||||
snprintf(conf_str, sizeof(conf_str), "datasets.%d.%s", list_pos, set_name);
|
||||
|
||||
SCLogNotice("(%d) set %s type %s. Conf %s", n, set_name, set_type->val, conf_str);
|
||||
|
||||
if (strcmp(set_type->val, "md5") == 0) {
|
||||
Dataset *dset = DatasetGet(set_name, DATASET_TYPE_MD5, save, load);
|
||||
if (dset == NULL)
|
||||
FatalError(SC_ERR_FATAL, "die: no dset for %s", set_name);
|
||||
SCLogNotice("dataset %s: id %d type %s", set_name, n, set_type->val);
|
||||
n++;
|
||||
|
||||
} else if (strcmp(set_type->val, "sha256") == 0) {
|
||||
Dataset *dset = DatasetGet(set_name, DATASET_TYPE_SHA256, save, load);
|
||||
if (dset == NULL)
|
||||
FatalError(SC_ERR_FATAL, "die: no dset for %s", set_name);
|
||||
SCLogNotice("dataset %s: id %d type %s", set_name, n, set_type->val);
|
||||
n++;
|
||||
|
||||
} else if (strcmp(set_type->val, "string") == 0) {
|
||||
Dataset *dset = DatasetGet(set_name, DATASET_TYPE_STRING, save, load);
|
||||
if (dset == NULL)
|
||||
FatalError(SC_ERR_FATAL, "die: no dset for %s", set_name);
|
||||
SCLogDebug("dataset %s: id %d type %s", set_name, n, set_type->val);
|
||||
n++;
|
||||
}
|
||||
|
||||
list_pos++;
|
||||
}
|
||||
}
|
||||
SCLogNotice("datasets done: %p", datasets);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void DatasetsDestroy(void)
|
||||
{
|
||||
SCLogDebug("destroying datasets: %p", sets);
|
||||
SCMutexLock(&sets_lock);
|
||||
Dataset *set = sets;
|
||||
while (set) {
|
||||
SCLogDebug("destroying set %s", set->name);
|
||||
Dataset *next = set->next;
|
||||
THashShutdown(set->hash);
|
||||
SCFree(set);
|
||||
set = next;
|
||||
}
|
||||
sets = NULL;
|
||||
SCMutexUnlock(&sets_lock);
|
||||
SCLogDebug("destroying datasets done: %p", sets);
|
||||
}
|
||||
|
||||
static int SaveCallback(void *ctx, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
FILE *fp = ctx;
|
||||
//PrintRawDataFp(fp, data, data_len);
|
||||
if (fp) {
|
||||
return fwrite(data, data_len, 1, fp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int Md5AsAscii(const void *s, char *out, size_t out_size)
|
||||
{
|
||||
const Md5Type *md5 = s;
|
||||
uint32_t x;
|
||||
int i;
|
||||
char str[256];
|
||||
for (i = 0, x = 0; x < sizeof(md5->md5); x++) {
|
||||
i += snprintf(&str[i], 255-i, "%02x", md5->md5[x]);
|
||||
}
|
||||
strlcat(out, str, out_size);
|
||||
strlcat(out, "\n", out_size);
|
||||
return strlen(out);
|
||||
}
|
||||
|
||||
static int Sha256AsAscii(const void *s, char *out, size_t out_size)
|
||||
{
|
||||
const Sha256Type *sha = s;
|
||||
uint32_t x;
|
||||
int i;
|
||||
char str[256];
|
||||
for (i = 0, x = 0; x < sizeof(sha->sha256); x++) {
|
||||
i += snprintf(&str[i], 255-i, "%02x", sha->sha256[x]);
|
||||
}
|
||||
strlcat(out, str, out_size);
|
||||
strlcat(out, "\n", out_size);
|
||||
return strlen(out);
|
||||
}
|
||||
|
||||
void DatasetsSave(void)
|
||||
{
|
||||
SCLogNotice("saving datasets: %p", sets);
|
||||
SCMutexLock(&sets_lock);
|
||||
Dataset *set = sets;
|
||||
while (set) {
|
||||
if (strlen(set->save) == 0)
|
||||
goto next;
|
||||
|
||||
FILE *fp = fopen(set->save, "w");
|
||||
if (fp == NULL)
|
||||
goto next;
|
||||
|
||||
SCLogNotice("dumping %s to %s", set->name, set->save);
|
||||
|
||||
switch (set->type) {
|
||||
case DATASET_TYPE_STRING:
|
||||
THashWalk(set->hash, StringAsBase64, SaveCallback, fp);
|
||||
break;
|
||||
case DATASET_TYPE_MD5:
|
||||
THashWalk(set->hash, Md5AsAscii, SaveCallback, fp);
|
||||
break;
|
||||
case DATASET_TYPE_SHA256:
|
||||
THashWalk(set->hash, Sha256AsAscii, SaveCallback, fp);
|
||||
break;
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
|
||||
next:
|
||||
set = set->next;
|
||||
}
|
||||
SCMutexUnlock(&sets_lock);
|
||||
}
|
||||
|
||||
static int DatasetLookupString(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
StringType lookup = { .ptr = (uint8_t *)data, .len = data_len, .rep.value = 0 };
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
THashDataUnlock(rdata);
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static DataRepResultType DatasetLookupStringwRep(Dataset *set,
|
||||
const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
|
||||
{
|
||||
DataRepResultType rrep = { .found = false, .rep = { .value = 0 }};
|
||||
|
||||
if (set == NULL)
|
||||
return rrep;
|
||||
|
||||
StringType lookup = { .ptr = (uint8_t *)data, .len = data_len, .rep = *rep };
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
StringType *found = rdata->data;
|
||||
rrep.found = true;
|
||||
rrep.rep = found->rep;
|
||||
THashDataUnlock(rdata);
|
||||
return rrep;
|
||||
}
|
||||
return rrep;
|
||||
}
|
||||
|
||||
static int DatasetLookupMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 16)
|
||||
return 0;
|
||||
|
||||
Md5Type lookup = { .rep.value = 0 };
|
||||
memcpy(lookup.md5, data, data_len);
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
THashDataUnlock(rdata);
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static DataRepResultType DatasetLookupMd5wRep(Dataset *set,
|
||||
const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
|
||||
{
|
||||
DataRepResultType rrep = { .found = false, .rep = { .value = 0 }};
|
||||
|
||||
if (set == NULL)
|
||||
return rrep;
|
||||
|
||||
if (data_len != 16)
|
||||
return rrep;
|
||||
|
||||
Md5Type lookup = { .rep.value = 0};
|
||||
memcpy(lookup.md5, data, data_len);
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
Md5Type *found = rdata->data;
|
||||
rrep.found = true;
|
||||
rrep.rep = found->rep;
|
||||
THashDataUnlock(rdata);
|
||||
return rrep;
|
||||
}
|
||||
return rrep;
|
||||
}
|
||||
|
||||
static int DatasetLookupSha256(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 32)
|
||||
return 0;
|
||||
|
||||
Sha256Type lookup = { .rep.value = 0 };
|
||||
memcpy(lookup.sha256, data, data_len);
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
THashDataUnlock(rdata);
|
||||
return 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static DataRepResultType DatasetLookupSha256wRep(Dataset *set,
|
||||
const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
|
||||
{
|
||||
DataRepResultType rrep = { .found = false, .rep = { .value = 0 }};
|
||||
|
||||
if (set == NULL)
|
||||
return rrep;
|
||||
|
||||
if (data_len != 32)
|
||||
return rrep;
|
||||
|
||||
Sha256Type lookup = { .rep.value = 0 };
|
||||
memcpy(lookup.sha256, data, data_len);
|
||||
THashData *rdata = THashLookupFromHash(set->hash, &lookup);
|
||||
if (rdata) {
|
||||
Sha256Type *found = rdata->data;
|
||||
rrep.found = true;
|
||||
rrep.rep = found->rep;
|
||||
THashDataUnlock(rdata);
|
||||
return rrep;
|
||||
}
|
||||
return rrep;
|
||||
}
|
||||
|
||||
int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
switch (set->type) {
|
||||
case DATASET_TYPE_STRING:
|
||||
return DatasetLookupString(set, data, data_len);
|
||||
case DATASET_TYPE_MD5:
|
||||
return DatasetLookupMd5(set, data, data_len);
|
||||
case DATASET_TYPE_SHA256:
|
||||
return DatasetLookupSha256(set, data, data_len);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
DataRepResultType DatasetLookupwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
const DataRepType *rep)
|
||||
{
|
||||
DataRepResultType rrep = { .found = false, .rep = { .value = 0 }};
|
||||
if (set == NULL)
|
||||
return rrep;
|
||||
|
||||
switch (set->type) {
|
||||
case DATASET_TYPE_STRING:
|
||||
return DatasetLookupStringwRep(set, data, data_len, rep);
|
||||
case DATASET_TYPE_MD5:
|
||||
return DatasetLookupMd5wRep(set, data, data_len, rep);
|
||||
case DATASET_TYPE_SHA256:
|
||||
return DatasetLookupSha256wRep(set, data, data_len, rep);
|
||||
}
|
||||
return rrep;
|
||||
}
|
||||
|
||||
/**
|
||||
* \retval 1 data was added to the hash
|
||||
* \retval 0 data was not added to the hash as it is already there
|
||||
* \retval -1 failed to add data to the hash
|
||||
*/
|
||||
static int DatasetAddString(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
StringType lookup = { .ptr = (uint8_t *)data, .len = data_len,
|
||||
.rep.value = 0 };
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* \retval 1 data was added to the hash
|
||||
* \retval 0 data was not added to the hash as it is already there
|
||||
* \retval -1 failed to add data to the hash
|
||||
*/
|
||||
static int DatasetAddStringwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
DataRepType *rep)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
StringType lookup = { .ptr = (uint8_t *)data, .len = data_len,
|
||||
.rep = *rep };
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DatasetAddMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 16)
|
||||
return -1;
|
||||
|
||||
Md5Type lookup = { .rep.value = 0 };
|
||||
memcpy(lookup.md5, data, 16);
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DatasetAddMd5wRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
DataRepType *rep)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 16)
|
||||
return -1;
|
||||
|
||||
Md5Type lookup = { .rep = *rep };
|
||||
memcpy(lookup.md5, data, 16);
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DatasetAddSha256wRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
DataRepType *rep)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 32)
|
||||
return 0;
|
||||
|
||||
Sha256Type lookup = { .rep = *rep };
|
||||
memcpy(lookup.sha256, data, 32);
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DatasetAddSha256(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
if (data_len != 32)
|
||||
return 0;
|
||||
|
||||
Sha256Type lookup = { .rep.value = 0 };
|
||||
memcpy(lookup.sha256, data, 32);
|
||||
struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
|
||||
if (res.data) {
|
||||
THashDataUnlock(res.data);
|
||||
return res.is_new ? 1 : 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
switch (set->type) {
|
||||
case DATASET_TYPE_STRING:
|
||||
return DatasetAddString(set, data, data_len);
|
||||
case DATASET_TYPE_MD5:
|
||||
return DatasetAddMd5(set, data, data_len);
|
||||
case DATASET_TYPE_SHA256:
|
||||
return DatasetAddSha256(set, data, data_len);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
DataRepType *rep)
|
||||
{
|
||||
if (set == NULL)
|
||||
return -1;
|
||||
|
||||
switch (set->type) {
|
||||
case DATASET_TYPE_STRING:
|
||||
return DatasetAddStringwRep(set, data, data_len, rep);
|
||||
case DATASET_TYPE_MD5:
|
||||
return DatasetAddMd5wRep(set, data, data_len, rep);
|
||||
case DATASET_TYPE_SHA256:
|
||||
return DatasetAddSha256wRep(set, data, data_len, rep);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
/* Copyright (C) 2017 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef __DATASETS_H__
|
||||
#define __DATASETS_H__
|
||||
|
||||
#include "util-thash.h"
|
||||
#include "datasets-reputation.h"
|
||||
|
||||
int DatasetsInit(void);
|
||||
void DatasetsDestroy(void);
|
||||
void DatasetsSave(void);
|
||||
|
||||
enum DatasetTypes {
|
||||
#define DATASET_TYPE_NOTSET 0
|
||||
DATASET_TYPE_STRING = 1,
|
||||
DATASET_TYPE_MD5,
|
||||
DATASET_TYPE_SHA256,
|
||||
};
|
||||
|
||||
typedef struct Dataset {
|
||||
char name[64];
|
||||
enum DatasetTypes type;
|
||||
uint32_t id;
|
||||
|
||||
THashTableContext *hash;
|
||||
|
||||
char load[PATH_MAX];
|
||||
char save[PATH_MAX];
|
||||
|
||||
struct Dataset *next;
|
||||
} Dataset;
|
||||
|
||||
Dataset *DatasetGetByName(const char *name);
|
||||
Dataset *DatasetGet(const char *name, enum DatasetTypes type,
|
||||
const char *save, const char *load);
|
||||
int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len);
|
||||
int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len);
|
||||
DataRepResultType DatasetLookupwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
|
||||
const DataRepType *rep);
|
||||
|
||||
#endif /* __DATASETS_H__ */
|
||||
@ -0,0 +1,365 @@
|
||||
/* Copyright (C) 2018-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*
|
||||
* Implements the datarep keyword
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "decode.h"
|
||||
#include "detect.h"
|
||||
#include "threads.h"
|
||||
#include "datasets.h"
|
||||
#include "detect-datarep.h"
|
||||
|
||||
#include "detect-parse.h"
|
||||
#include "detect-engine.h"
|
||||
#include "detect-engine-mpm.h"
|
||||
#include "detect-engine-state.h"
|
||||
|
||||
#include "util-byte.h"
|
||||
#include "util-debug.h"
|
||||
#include "util-print.h"
|
||||
|
||||
#define PARSE_REGEX "([a-z]+)(?:,\\s*([\\-_A-z0-9\\s\\.]+)){1,4}"
|
||||
static pcre *parse_regex;
|
||||
static pcre_extra *parse_regex_study;
|
||||
|
||||
int DetectDatarepMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *,
|
||||
const Signature *, const SigMatchCtx *);
|
||||
static int DetectDatarepSetup (DetectEngineCtx *, Signature *, const char *);
|
||||
void DetectDatarepFree (void *);
|
||||
|
||||
void DetectDatarepRegister (void)
|
||||
{
|
||||
sigmatch_table[DETECT_DATAREP].name = "datarep";
|
||||
sigmatch_table[DETECT_DATAREP].desc = "operate on datasets";
|
||||
sigmatch_table[DETECT_DATAREP].url = DOC_URL DOC_VERSION "/rules/dataset-keywords.html#datarep";
|
||||
sigmatch_table[DETECT_DATAREP].Setup = DetectDatarepSetup;
|
||||
sigmatch_table[DETECT_DATAREP].Free = DetectDatarepFree;
|
||||
|
||||
DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
|
||||
}
|
||||
|
||||
/*
|
||||
1 match
|
||||
0 no match
|
||||
-1 can't match
|
||||
*/
|
||||
int DetectDatarepBufferMatch(DetectEngineThreadCtx *det_ctx,
|
||||
const DetectDatarepData *sd,
|
||||
const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (data == NULL || data_len == 0)
|
||||
return 0;
|
||||
|
||||
DataRepResultType r = DatasetLookupwRep(sd->set, data, data_len, &sd->rep);
|
||||
if (!r.found)
|
||||
return 0;
|
||||
|
||||
switch (sd->op) {
|
||||
case DATAREP_OP_GT:
|
||||
if (r.rep.value > sd->rep.value)
|
||||
return 1;
|
||||
break;
|
||||
case DATAREP_OP_LT:
|
||||
if (r.rep.value < sd->rep.value)
|
||||
return 1;
|
||||
break;
|
||||
case DATAREP_OP_EQ:
|
||||
if (r.rep.value == sd->rep.value)
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DetectDatarepParse(const char *str,
|
||||
char *cmd, int cmd_len,
|
||||
char *name, int name_len,
|
||||
enum DatasetTypes *type,
|
||||
char *load, size_t load_size,
|
||||
uint16_t *rep_value)
|
||||
{
|
||||
bool cmd_set = false;
|
||||
bool name_set = false;
|
||||
bool value_set = false;
|
||||
|
||||
char copy[strlen(str)+1];
|
||||
strlcpy(copy, str, sizeof(copy));
|
||||
char *xsaveptr = NULL;
|
||||
char *key = strtok_r(copy, ",", &xsaveptr);
|
||||
while (key != NULL) {
|
||||
while (*key != '\0' && isblank(*key)) {
|
||||
key++;
|
||||
}
|
||||
char *val = strchr(key, ' ');
|
||||
if (val != NULL) {
|
||||
*val++ = '\0';
|
||||
while (*val != '\0' && isblank(*val)) {
|
||||
val++;
|
||||
SCLogDebug("cmd %s val %s", key, val);
|
||||
}
|
||||
} else {
|
||||
SCLogDebug("cmd %s", key);
|
||||
}
|
||||
|
||||
if (strlen(key) == 0) {
|
||||
goto next;
|
||||
}
|
||||
|
||||
if (!name_set) {
|
||||
if (val) {
|
||||
return -1;
|
||||
}
|
||||
strlcpy(name, key, name_len);
|
||||
name_set = true;
|
||||
} else if (!cmd_set) {
|
||||
if (val) {
|
||||
return -1;
|
||||
}
|
||||
strlcpy(cmd, key, cmd_len);
|
||||
cmd_set = true;
|
||||
} else if (!value_set) {
|
||||
if (val) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (ByteExtractStringUint16(rep_value, 10, 0, key) != (int)strlen(key))
|
||||
return -1;
|
||||
|
||||
value_set = true;
|
||||
} else {
|
||||
if (val == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strcmp(key, "type") == 0) {
|
||||
SCLogDebug("type %s", val);
|
||||
|
||||
if (strcmp(val, "md5") == 0) {
|
||||
*type = DATASET_TYPE_MD5;
|
||||
} else if (strcmp(val, "sha256") == 0) {
|
||||
*type = DATASET_TYPE_SHA256;
|
||||
} else if (strcmp(val, "string") == 0) {
|
||||
*type = DATASET_TYPE_STRING;
|
||||
} else {
|
||||
SCLogDebug("bad type %s", val);
|
||||
return -1;
|
||||
}
|
||||
|
||||
} else if (strcmp(key, "load") == 0) {
|
||||
SCLogDebug("load %s", val);
|
||||
strlcpy(load, val, load_size);
|
||||
}
|
||||
}
|
||||
|
||||
SCLogDebug("key: %s, value: %s", key, val);
|
||||
|
||||
next:
|
||||
key = strtok_r(NULL, ",", &xsaveptr);
|
||||
}
|
||||
|
||||
if (strlen(load) > 0 && *type == DATASET_TYPE_NOTSET) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"if load is used type must be set as well");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!name_set || !cmd_set || !value_set) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"missing values");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Trim trailing whitespace. */
|
||||
while (strlen(name) > 0 && isblank(name[strlen(name) - 1])) {
|
||||
name[strlen(name) - 1] = '\0';
|
||||
}
|
||||
|
||||
/* Validate name, spaces are not allowed. */
|
||||
for (size_t i = 0; i < strlen(name); i++) {
|
||||
if (isblank(name[i])) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"spaces not allowed in dataset names");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/** \brief wrapper around dirname that does leave input untouched */
|
||||
static void GetDirName(const char *in, char *out, size_t outs)
|
||||
{
|
||||
if (strlen(in) == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t size = strlen(in) + 1;
|
||||
char tmp[size];
|
||||
strlcpy(tmp, in, size);
|
||||
|
||||
char *dir = dirname(tmp);
|
||||
BUG_ON(dir == NULL);
|
||||
strlcpy(out, dir, outs);
|
||||
return;
|
||||
}
|
||||
|
||||
static int SetupLoadPath(const DetectEngineCtx *de_ctx,
|
||||
char *load, size_t load_size)
|
||||
{
|
||||
SCLogDebug("load %s", load);
|
||||
|
||||
if (PathIsAbsolute(load)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool done = false;
|
||||
#ifdef HAVE_LIBGEN_H
|
||||
BUG_ON(de_ctx->rule_file == NULL);
|
||||
|
||||
char dir[PATH_MAX] = "";
|
||||
GetDirName(de_ctx->rule_file, dir, sizeof(dir));
|
||||
|
||||
SCLogDebug("rule_file %s dir %s", de_ctx->rule_file, dir);
|
||||
char path[PATH_MAX];
|
||||
if (snprintf(path, sizeof(path), "%s/%s", dir, load) >= (int)sizeof(path)) // TODO windows path
|
||||
return -1;
|
||||
|
||||
if (SCPathExists(path)) {
|
||||
done = true;
|
||||
strlcpy(load, path, load_size);
|
||||
SCLogNotice("using path '%s' (HAVE_LIBGEN_H)", load);
|
||||
} else {
|
||||
SCLogNotice("path '%s' does not exist (HAVE_LIBGEN_H)", path);
|
||||
}
|
||||
#endif
|
||||
if (!done) {
|
||||
char *loadp = DetectLoadCompleteSigPath(de_ctx, load);
|
||||
if (loadp == NULL) {
|
||||
return -1;
|
||||
}
|
||||
SCLogNotice("loadp %s", loadp);
|
||||
|
||||
if (SCPathExists(loadp)) {
|
||||
strlcpy(load, loadp, load_size);
|
||||
SCLogNotice("using path '%s' (non-HAVE_LIBGEN_H)", load);
|
||||
} else {
|
||||
SCLogNotice("path '%s' does not exist (non-HAVE_LIBGEN_H)", loadp);
|
||||
}
|
||||
SCFree(loadp);
|
||||
|
||||
// TODO try data-dir as well?
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DetectDatarepSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
|
||||
{
|
||||
SigMatch *sm = NULL;
|
||||
char cmd_str[16] = "", name[64] = "";
|
||||
enum DatasetTypes type = DATASET_TYPE_NOTSET;
|
||||
char load[PATH_MAX];
|
||||
uint16_t value = 0;
|
||||
|
||||
if (DetectBufferGetActiveList(de_ctx, s) == -1) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"datarep is only supported for sticky buffers");
|
||||
SCReturnInt(-1);
|
||||
}
|
||||
|
||||
int list = s->init_data->list;
|
||||
if (list == DETECT_SM_LIST_NOTSET) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"datarep is only supported for sticky buffers");
|
||||
SCReturnInt(-1);
|
||||
}
|
||||
|
||||
if (!DetectDatarepParse(rawstr, cmd_str, sizeof(cmd_str), name,
|
||||
sizeof(name), &type, load, sizeof(load), &value)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strlen(load) != 0) {
|
||||
if (SetupLoadPath(de_ctx, load, sizeof(load)) != 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
enum DetectDatarepOp op;
|
||||
if (strcmp(cmd_str,">") == 0) {
|
||||
op = DATAREP_OP_GT;
|
||||
} else if (strcmp(cmd_str,"<") == 0) {
|
||||
op = DATAREP_OP_LT;
|
||||
} else if (strcmp(cmd_str,"==") == 0) {
|
||||
op = DATAREP_OP_EQ;
|
||||
} else {
|
||||
SCLogError(SC_ERR_UNKNOWN_VALUE,
|
||||
"datarep operation \"%s\" is not supported.", cmd_str);
|
||||
return -1;
|
||||
}
|
||||
|
||||
Dataset *set = DatasetGet(name, type, /* no save */ NULL, load);
|
||||
if (set == NULL) {
|
||||
SCLogError(SC_ERR_UNKNOWN_VALUE,
|
||||
"failed to set up datarep set '%s'.", name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
DetectDatarepData *cd = SCCalloc(1, sizeof(DetectDatarepData));
|
||||
if (unlikely(cd == NULL))
|
||||
goto error;
|
||||
|
||||
cd->set = set;
|
||||
cd->op = op;
|
||||
cd->rep.value = value;
|
||||
|
||||
SCLogNotice("cmd %s, name %s",
|
||||
cmd_str, strlen(name) ? name : "(none)");
|
||||
|
||||
/* Okay so far so good, lets get this into a SigMatch
|
||||
* and put it in the Signature. */
|
||||
sm = SigMatchAlloc();
|
||||
if (sm == NULL)
|
||||
goto error;
|
||||
|
||||
sm->type = DETECT_DATAREP;
|
||||
sm->ctx = (SigMatchCtx *)cd;
|
||||
SigMatchAppendSMToList(s, sm, list);
|
||||
return 0;
|
||||
|
||||
error:
|
||||
if (cd != NULL)
|
||||
SCFree(cd);
|
||||
if (sm != NULL)
|
||||
SCFree(sm);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void DetectDatarepFree (void *ptr)
|
||||
{
|
||||
DetectDatarepData *fd = (DetectDatarepData *)ptr;
|
||||
|
||||
if (fd == NULL)
|
||||
return;
|
||||
|
||||
SCFree(fd);
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
/* Copyright (C) 2018 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DETECT_DATAREP_H__
|
||||
#define __DETECT_DATAREP_H__
|
||||
|
||||
#include "datasets.h"
|
||||
#include "datasets-reputation.h"
|
||||
|
||||
enum DetectDatarepOp {
|
||||
DATAREP_OP_GT, /* rep is greater than requested */
|
||||
DATAREP_OP_LT, /* rep is smaller than requested */
|
||||
DATAREP_OP_EQ, /* rep is smaller than requested */
|
||||
};
|
||||
|
||||
typedef struct DetectDatarepData_ {
|
||||
Dataset *set;
|
||||
uint8_t cmd;
|
||||
enum DetectDatarepOp op;
|
||||
DataRepType rep;
|
||||
} DetectDatarepData;
|
||||
|
||||
int DetectDatarepBufferMatch(DetectEngineThreadCtx *det_ctx,
|
||||
const DetectDatarepData *sd,
|
||||
const uint8_t *data, const uint32_t data_len);
|
||||
|
||||
/* prototypes */
|
||||
void DetectDatarepRegister (void);
|
||||
|
||||
#endif /* __DETECT_DATAREP_H__ */
|
||||
@ -0,0 +1,411 @@
|
||||
/* Copyright (C) 2018-2019 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*
|
||||
* Implements the dataset keyword
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "decode.h"
|
||||
#include "detect.h"
|
||||
#include "threads.h"
|
||||
#include "datasets.h"
|
||||
#include "detect-dataset.h"
|
||||
|
||||
#include "detect-parse.h"
|
||||
#include "detect-engine.h"
|
||||
#include "detect-engine-mpm.h"
|
||||
#include "detect-engine-state.h"
|
||||
|
||||
#include "util-debug.h"
|
||||
#include "util-print.h"
|
||||
|
||||
#define PARSE_REGEX "([a-z]+)(?:,\\s*([\\-_A-z0-9\\s\\.]+)){1,4}"
|
||||
static pcre *parse_regex;
|
||||
static pcre_extra *parse_regex_study;
|
||||
|
||||
int DetectDatasetMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *,
|
||||
const Signature *, const SigMatchCtx *);
|
||||
static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
|
||||
void DetectDatasetFree (void *);
|
||||
|
||||
void DetectDatasetRegister (void)
|
||||
{
|
||||
sigmatch_table[DETECT_DATASET].name = "dataset";
|
||||
sigmatch_table[DETECT_DATASET].desc = "match sticky buffer against datasets";
|
||||
sigmatch_table[DETECT_DATASET].url = DOC_URL DOC_VERSION "/rules/dataset-keywords.html#dataset";
|
||||
sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup;
|
||||
sigmatch_table[DETECT_DATASET].Free = DetectDatasetFree;
|
||||
|
||||
DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
|
||||
}
|
||||
|
||||
/*
|
||||
1 match
|
||||
0 no match
|
||||
-1 can't match
|
||||
*/
|
||||
int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
|
||||
const DetectDatasetData *sd,
|
||||
const uint8_t *data, const uint32_t data_len)
|
||||
{
|
||||
if (data == NULL || data_len == 0)
|
||||
return 0;
|
||||
|
||||
switch (sd->cmd) {
|
||||
case DETECT_DATASET_CMD_ISSET: {
|
||||
//PrintRawDataFp(stdout, data, data_len);
|
||||
int r = DatasetLookup(sd->set, data, data_len);
|
||||
SCLogDebug("r %d", r);
|
||||
if (r == 1)
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
case DETECT_DATASET_CMD_ISNOTSET: {
|
||||
//PrintRawDataFp(stdout, data, data_len);
|
||||
int r = DatasetLookup(sd->set, data, data_len);
|
||||
SCLogDebug("r %d", r);
|
||||
if (r == 0)
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
case DETECT_DATASET_CMD_SET: {
|
||||
//PrintRawDataFp(stdout, data, data_len);
|
||||
int r = DatasetAdd(sd->set, data, data_len);
|
||||
if (r == 1)
|
||||
return 1;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
abort();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int DetectDatasetParse(const char *str,
|
||||
char *cmd, int cmd_len,
|
||||
char *name, int name_len,
|
||||
enum DatasetTypes *type,
|
||||
char *load, size_t load_size,
|
||||
char *save, size_t save_size)
|
||||
{
|
||||
bool cmd_set = false;
|
||||
bool name_set = false;
|
||||
bool load_set = false;
|
||||
bool save_set = false;
|
||||
bool state_set = false;
|
||||
|
||||
char copy[strlen(str)+1];
|
||||
strlcpy(copy, str, sizeof(copy));
|
||||
char *xsaveptr = NULL;
|
||||
char *key = strtok_r(copy, ",", &xsaveptr);
|
||||
while (key != NULL) {
|
||||
while (*key != '\0' && isblank(*key)) {
|
||||
key++;
|
||||
}
|
||||
char *val = strchr(key, ' ');
|
||||
if (val != NULL) {
|
||||
*val++ = '\0';
|
||||
while (*val != '\0' && isblank(*val)) {
|
||||
val++;
|
||||
SCLogDebug("cmd %s val %s", key, val);
|
||||
}
|
||||
} else {
|
||||
SCLogDebug("cmd %s", key);
|
||||
}
|
||||
|
||||
if (strlen(key) == 0) {
|
||||
goto next;
|
||||
}
|
||||
|
||||
if (!cmd_set) {
|
||||
if (val) {
|
||||
return -1;
|
||||
}
|
||||
strlcpy(cmd, key, cmd_len);
|
||||
cmd_set = true;
|
||||
} else if (!name_set) {
|
||||
if (val) {
|
||||
return -1;
|
||||
}
|
||||
strlcpy(name, key, name_len);
|
||||
name_set = true;
|
||||
} else {
|
||||
if (val == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strcmp(key, "type") == 0) {
|
||||
SCLogDebug("type %s", val);
|
||||
|
||||
if (strcmp(val, "md5") == 0) {
|
||||
*type = DATASET_TYPE_MD5;
|
||||
} else if (strcmp(val, "sha256") == 0) {
|
||||
*type = DATASET_TYPE_SHA256;
|
||||
} else if (strcmp(val, "string") == 0) {
|
||||
*type = DATASET_TYPE_STRING;
|
||||
} else {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE, "bad type %s", val);
|
||||
return -1;
|
||||
}
|
||||
|
||||
} else if (strcmp(key, "save") == 0) {
|
||||
if (save_set) {
|
||||
SCLogWarning(SC_ERR_INVALID_SIGNATURE,
|
||||
"'save' can only appear once");
|
||||
return -1;
|
||||
}
|
||||
SCLogDebug("save %s", val);
|
||||
strlcpy(save, val, save_size);
|
||||
save_set = true;
|
||||
} else if (strcmp(key, "load") == 0) {
|
||||
if (load_set) {
|
||||
SCLogWarning(SC_ERR_INVALID_SIGNATURE,
|
||||
"'load' can only appear once");
|
||||
return -1;
|
||||
}
|
||||
SCLogDebug("load %s", val);
|
||||
strlcpy(load, val, load_size);
|
||||
load_set = true;
|
||||
} else if (strcmp(key, "state") == 0) {
|
||||
if (state_set) {
|
||||
SCLogWarning(SC_ERR_INVALID_SIGNATURE,
|
||||
"'state' can only appear once");
|
||||
return -1;
|
||||
}
|
||||
SCLogDebug("state %s", val);
|
||||
strlcpy(load, val, load_size);
|
||||
strlcpy(save, val, save_size);
|
||||
state_set = true;
|
||||
}
|
||||
}
|
||||
|
||||
SCLogDebug("key: %s, value: %s", key, val);
|
||||
|
||||
next:
|
||||
key = strtok_r(NULL, ",", &xsaveptr);
|
||||
}
|
||||
|
||||
if ((load_set || save_set) && state_set) {
|
||||
SCLogWarning(SC_ERR_INVALID_SIGNATURE,
|
||||
"'state' can not be mixed with 'load' and 'save'");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Trim trailing whitespace. */
|
||||
while (strlen(name) > 0 && isblank(name[strlen(name) - 1])) {
|
||||
name[strlen(name) - 1] = '\0';
|
||||
}
|
||||
|
||||
/* Validate name, spaces are not allowed. */
|
||||
for (size_t i = 0; i < strlen(name); i++) {
|
||||
if (isblank(name[i])) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"spaces not allowed in dataset names");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/** \brief wrapper around dirname that does leave input untouched */
|
||||
static void GetDirName(const char *in, char *out, size_t outs)
|
||||
{
|
||||
if (strlen(in) == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t size = strlen(in) + 1;
|
||||
char tmp[size];
|
||||
strlcpy(tmp, in, size);
|
||||
|
||||
char *dir = dirname(tmp);
|
||||
BUG_ON(dir == NULL);
|
||||
strlcpy(out, dir, outs);
|
||||
return;
|
||||
}
|
||||
|
||||
static int SetupLoadPath(const DetectEngineCtx *de_ctx,
|
||||
char *load, size_t load_size)
|
||||
{
|
||||
SCLogDebug("load %s", load);
|
||||
|
||||
if (PathIsAbsolute(load)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool done = false;
|
||||
#ifdef HAVE_LIBGEN_H
|
||||
BUG_ON(de_ctx->rule_file == NULL);
|
||||
|
||||
char dir[PATH_MAX] = "";
|
||||
GetDirName(de_ctx->rule_file, dir, sizeof(dir));
|
||||
|
||||
SCLogDebug("rule_file %s dir %s", de_ctx->rule_file, dir);
|
||||
char path[PATH_MAX];
|
||||
if (snprintf(path, sizeof(path), "%s/%s", dir, load) >= (int)sizeof(path)) // TODO windows path
|
||||
return -1;
|
||||
|
||||
if (SCPathExists(load)) {
|
||||
done = true;
|
||||
strlcpy(load, path, load_size);
|
||||
SCLogDebug("using path '%s' (HAVE_LIBGEN_H)", load);
|
||||
}
|
||||
#endif
|
||||
if (!done) {
|
||||
char *loadp = DetectLoadCompleteSigPath(de_ctx, load);
|
||||
if (loadp == NULL) {
|
||||
return -1;
|
||||
}
|
||||
SCLogDebug("loadp %s", loadp);
|
||||
|
||||
if (SCPathExists(loadp)) {
|
||||
strlcpy(load, loadp, load_size);
|
||||
SCLogDebug("using path '%s' (non-HAVE_LIBGEN_H)", load);
|
||||
}
|
||||
SCFree(loadp);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int SetupSavePath(const DetectEngineCtx *de_ctx,
|
||||
char *save, size_t save_size)
|
||||
{
|
||||
SCLogDebug("save %s", save);
|
||||
|
||||
if (PathIsAbsolute(save)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// data dir
|
||||
const char *dir = ConfigGetDataDirectory();
|
||||
BUG_ON(dir == NULL); // should not be able to fail
|
||||
char path[PATH_MAX];
|
||||
if (snprintf(path, sizeof(path), "%s/%s", dir, save) >= (int)sizeof(path)) // TODO windows path
|
||||
return -1;
|
||||
|
||||
/* TODO check if location exists and is writable */
|
||||
|
||||
strlcpy(save, path, save_size);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
|
||||
{
|
||||
DetectDatasetData *cd = NULL;
|
||||
SigMatch *sm = NULL;
|
||||
uint8_t cmd = 0;
|
||||
char cmd_str[16] = "", name[64] = "";
|
||||
enum DatasetTypes type = DATASET_TYPE_NOTSET;
|
||||
char load[PATH_MAX] = "";
|
||||
char save[PATH_MAX] = "";
|
||||
|
||||
if (DetectBufferGetActiveList(de_ctx, s) == -1) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"datasets are only supported for sticky buffers");
|
||||
SCReturnInt(-1);
|
||||
}
|
||||
|
||||
int list = s->init_data->list;
|
||||
if (list == DETECT_SM_LIST_NOTSET) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"datasets are only supported for sticky buffers");
|
||||
SCReturnInt(-1);
|
||||
}
|
||||
|
||||
if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name,
|
||||
sizeof(name), &type, load, sizeof(load), save, sizeof(save))) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strcmp(cmd_str,"isset") == 0) {
|
||||
cmd = DETECT_DATASET_CMD_ISSET;
|
||||
} else if (strcmp(cmd_str,"isnotset") == 0) {
|
||||
cmd = DETECT_DATASET_CMD_ISNOTSET;
|
||||
} else if (strcmp(cmd_str,"set") == 0) {
|
||||
cmd = DETECT_DATASET_CMD_SET;
|
||||
} else if (strcmp(cmd_str,"unset") == 0) {
|
||||
cmd = DETECT_DATASET_CMD_UNSET;
|
||||
} else {
|
||||
SCLogError(SC_ERR_UNKNOWN_VALUE,
|
||||
"dataset action \"%s\" is not supported.", cmd_str);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* if just 'load' is set, we load data from the same dir as the
|
||||
* rule file. If load+save is used, we use data dir */
|
||||
if (strlen(save) == 0 && strlen(load) != 0) {
|
||||
if (SetupLoadPath(de_ctx, load, sizeof(load)) != 0)
|
||||
return -1;
|
||||
/* if just 'save' is set, we use either full path or the
|
||||
* data-dir */
|
||||
} else if (strlen(save) != 0 && strlen(load) == 0) {
|
||||
if (SetupSavePath(de_ctx, save, sizeof(save)) != 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
SCLogDebug("name '%s' load '%s' save '%s'", name, load, save);
|
||||
Dataset *set = DatasetGet(name, type, save, load);
|
||||
if (set == NULL) {
|
||||
SCLogError(SC_ERR_INVALID_SIGNATURE,
|
||||
"failed to set up dataset '%s'.", name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
cd = SCCalloc(1, sizeof(DetectDatasetData));
|
||||
if (unlikely(cd == NULL))
|
||||
goto error;
|
||||
|
||||
cd->set = set;
|
||||
cd->cmd = cmd;
|
||||
|
||||
SCLogDebug("cmd %s, name %s",
|
||||
cmd_str, strlen(name) ? name : "(none)");
|
||||
|
||||
/* Okay so far so good, lets get this into a SigMatch
|
||||
* and put it in the Signature. */
|
||||
sm = SigMatchAlloc();
|
||||
if (sm == NULL)
|
||||
goto error;
|
||||
|
||||
sm->type = DETECT_DATASET;
|
||||
sm->ctx = (SigMatchCtx *)cd;
|
||||
SigMatchAppendSMToList(s, sm, list);
|
||||
return 0;
|
||||
|
||||
error:
|
||||
if (cd != NULL)
|
||||
SCFree(cd);
|
||||
if (sm != NULL)
|
||||
SCFree(sm);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void DetectDatasetFree (void *ptr)
|
||||
{
|
||||
DetectDatasetData *fd = (DetectDatasetData *)ptr;
|
||||
if (fd == NULL)
|
||||
return;
|
||||
|
||||
SCFree(fd);
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/* Copyright (C) 2018 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __DETECT_DATASET_H__
|
||||
#define __DETECT_DATASET_H__
|
||||
|
||||
#include "datasets.h"
|
||||
|
||||
#define DETECT_DATASET_CMD_SET 0
|
||||
#define DETECT_DATASET_CMD_UNSET 1
|
||||
#define DETECT_DATASET_CMD_ISNOTSET 2
|
||||
#define DETECT_DATASET_CMD_ISSET 3
|
||||
|
||||
#define DETECT_DATASET_CMD_MAX 4
|
||||
|
||||
typedef struct DetectDatasetData_ {
|
||||
Dataset *set;
|
||||
uint8_t cmd;
|
||||
} DetectDatasetData;
|
||||
|
||||
int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
|
||||
const DetectDatasetData *sd,
|
||||
const uint8_t *data, const uint32_t data_len);
|
||||
|
||||
/* prototypes */
|
||||
void DetectDatasetRegister (void);
|
||||
|
||||
#endif /* __DETECT_DATASET_H__ */
|
||||
Loading…
Reference in New Issue