aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

util: remove unused bloom filter code

Browse Source 
Ticket: 4083
pull/10832/head
Philippe Antoine 2 years ago committed by Victor Julien
parent b113bdd9e3
commit 784ce30ae1
6 changed files with 0 additions and 786 deletions
Show Stats Download Patch File Download Diff File

4
src/Makefile.am
Unescape Escape View File

@ -507,8 +507,6 @@ noinst_HEADERS = \
util-affinity.h \ util-affinity.h \
util-atomic.h \ util-atomic.h \
util-base64.h \ util-base64.h \
util-bloomfilter-counting.h \
util-bloomfilter.h \
util-bpf.h \ util-bpf.h \
util-buffer.h \ util-buffer.h \
util-byte.h \ util-byte.h \
@ -1107,8 +1105,6 @@ libsuricata_c_a_SOURCES = \
util-affinity.c \ util-affinity.c \
util-atomic.c \ util-atomic.c \
util-base64.c \ util-base64.c \
util-bloomfilter.c \
util-bloomfilter-counting.c \
util-bpf.c \ util-bpf.c \
util-buffer.c \ util-buffer.c \
util-byte.c \ util-byte.c \

4
src/runmode-unittests.c
Unescape Escape View File

@ -85,8 +85,6 @@
#include "util-spm.h" #include "util-spm.h"
#include "util-hash.h" #include "util-hash.h"
#include "util-hashlist.h" #include "util-hashlist.h"
#include "util-bloomfilter.h"
#include "util-bloomfilter-counting.h"
#include "util-pool.h" #include "util-pool.h"
#include "util-byte.h" #include "util-byte.h"
#include "util-proto-name.h" #include "util-proto-name.h"
@ -139,8 +137,6 @@ static void RegisterUnittests(void)
SigTableRegisterTests(); SigTableRegisterTests();
HashTableRegisterTests(); HashTableRegisterTests();
HashListTableRegisterTests(); HashListTableRegisterTests();
BloomFilterRegisterTests();
BloomFilterCountingRegisterTests();
PoolRegisterTests(); PoolRegisterTests();
ByteRegisterTests(); ByteRegisterTests();
MpmRegisterTests(); MpmRegisterTests();

407
src/util-bloomfilter-counting.c
Unescape Escape View File

@ -1,407 +0,0 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
* Counting Bloom Filter implementation. Can be used with 8, 16, 32 bits
* counters.
*/
#include "suricata-common.h"
#include "util-bloomfilter-counting.h"
#include "util-unittest.h"
/* type: 1, 2 or 4 for 8, 16, or 32 bit counters
*
*/
BloomFilterCounting *BloomFilterCountingInit(uint32_t size, uint8_t type, uint8_t iter, uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t)) {
BloomFilterCounting *bf = NULL;
if (iter == 0)
goto error;
if (Hash == NULL || size == 0) {
//printf("ERROR: BloomFilterCountingInit no Hash function\n");
goto error;
}
if (type != 1 && type != 2 && type != 4) {
//printf("ERROR: BloomFilterCountingInit only 1, 2 and 4 bytes are supported\n");
goto error;
}
/* setup the filter */
bf = SCCalloc(1, sizeof(BloomFilterCounting));
if (unlikely(bf == NULL))
goto error;
bf->type = type; /* size of the type: 1, 2, 4 */
bf->array_size = size;
bf->hash_iterations = iter;
bf->Hash = Hash;
/* setup the bitarray */
bf->array = SCCalloc(bf->array_size, bf->type);
if (bf->array == NULL)
goto error;
return bf;
error:
if (bf != NULL) {
if (bf->array != NULL)
SCFree(bf->array);
SCFree(bf);
}
return NULL;
}
void BloomFilterCountingFree(BloomFilterCounting *bf)
{
if (bf != NULL) {
if (bf->array != NULL)
SCFree(bf->array);
SCFree(bf);
}
}
int BloomFilterCountingAdd(BloomFilterCounting *bf, const void *data, uint16_t datalen)
{
uint8_t iter = 0;
uint32_t hash = 0;
if (bf == NULL || data == NULL || datalen == 0)
return -1;
for (iter = 0; iter < bf->hash_iterations; iter++) {
hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
if (bf->type == 1) {
uint8_t *u8 = (uint8_t *)&bf->array[hash];
if ((*u8) != 255)
(*u8)++;
} else if (bf->type == 2) {
uint16_t *u16 = (uint16_t *)&bf->array[hash];
if ((*u16) != 65535)
(*u16)++;
} else if (bf->type == 4) {
uint32_t *u32 = (uint32_t *)&bf->array[hash];
if ((*u32) != 4294967295UL)
(*u32)++;
}
}
return 0;
}
int BloomFilterCountingRemove(BloomFilterCounting *bf, const void *data, uint16_t datalen)
{
uint8_t iter = 0;
uint32_t hash = 0;
if (bf == NULL || data == NULL || datalen == 0)
return -1;
/* only remove data that was actually added */
if (BloomFilterCountingTest(bf, data, datalen) == 0) {
printf("ERROR: BloomFilterCountingRemove tried to remove data "
"that was never added to the set or was already removed.\n");
return -1;
}
/* decrease counters for every iteration */
for (iter = 0; iter < bf->hash_iterations; iter++) {
hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
if (bf->type == 1) {
uint8_t *u8 = (uint8_t *)&bf->array[hash];
if ((*u8) > 0)
(*u8)--;
else {
printf("ERROR: BloomFilterCountingRemove tried to decrease a "
"counter below zero.\n");
return -1;
}
} else if (bf->type == 2) {
uint16_t *u16 = (uint16_t *)&bf->array[hash];
if ((*u16) > 0)
(*u16)--;
else {
printf("ERROR: BloomFilterCountingRemove tried to decrease a "
"counter below zero.\n");
return -1;
}
} else if (bf->type == 4) {
uint32_t *u32 = (uint32_t *)&bf->array[hash];
if ((*u32) > 0)
(*u32)--;
else {
printf("ERROR: BloomFilterCountingRemove tried to decrease a "
"counter below zero.\n");
return -1;
}
}
}
return 0;
}
/* Test if data matches our filter and is likely to be in the set
*
* returns 0: for no match
* 1: match
*/
int BloomFilterCountingTest(BloomFilterCounting *bf, const void *data, uint16_t datalen)
{
uint8_t iter = 0;
uint32_t hash = 0;
int hit = 1;
/* check each hash iteration */
for (iter = 0; iter < bf->hash_iterations; iter++) {
hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
if (bf->type == 1) {
uint8_t *u8 = (uint8_t *)&bf->array[hash];
if ((*u8) == 0x00) {
hit = 0;
break;
}
} else if (bf->type == 2) {
uint16_t *u16 = (uint16_t *)&bf->array[hash];
if ((*u16) == 0x0000) {
hit = 0;
break;
}
} else if (bf->type == 4) {
uint32_t *u32 = (uint32_t *)&bf->array[hash];
if ((*u32) == 0x00000000) {
hit = 0;
break;
}
}
}
return hit;
}
/*
* ONLY TESTS BELOW THIS COMMENT
*/
#ifdef UNITTESTS
static uint32_t BloomHash(const void *data, uint16_t datalen, uint8_t iter, uint32_t hash_size)
{
uint8_t *d = (uint8_t *)data;
uint32_t i;
uint32_t hash = 0;
for (i = 0; i < datalen; i++) {
if (i == 0) hash += (((uint32_t)*d++));
else if (i == 1) hash += (((uint32_t)*d++) * datalen);
else hash *= (((uint32_t)*d++) * i);
}
hash *= (iter + datalen);
hash %= hash_size;
return hash;
}
static int BloomFilterCountingTestInit01 (void)
{
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
if (bf == NULL)
return 0;
BloomFilterCountingFree(bf);
return 1;
}
/* no hash function, so it should fail */
static int BloomFilterCountingTestInit02 (void)
{
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, NULL);
if (bf == NULL)
return 1;
BloomFilterCountingFree(bf);
return 0;
}
static int BloomFilterCountingTestInit03 (void)
{
int result = 0;
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
if (bf == NULL)
return 0;
if (bf->Hash == BloomHash)
result = 1;
BloomFilterCountingFree(bf);
return result;
}
static int BloomFilterCountingTestInit04 (void)
{
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 0, 4, BloomHash);
if (bf == NULL)
return 1;
BloomFilterCountingFree(bf);
return 0;
}
static int BloomFilterCountingTestInit05 (void)
{
BloomFilterCounting *bf = BloomFilterCountingInit(0, 4, 4, BloomHash);
if (bf == NULL)
return 1;
BloomFilterCountingFree(bf);
return 0;
}
static int BloomFilterCountingTestInit06 (void)
{
BloomFilterCounting *bf = BloomFilterCountingInit(32, 3, 4, BloomHash);
if (bf == NULL)
return 1;
BloomFilterCountingFree(bf);
return 0;
}
static int BloomFilterCountingTestAdd01 (void)
{
int result = 0;
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
if (bf == NULL)
return 0;
int r = BloomFilterCountingAdd(bf, "test", 0);
if (r == 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterCountingFree(bf);
return result;
}
static int BloomFilterCountingTestAdd02 (void)
{
int result = 0;
BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
if (bf == NULL)
return 0;
int r = BloomFilterCountingAdd(bf, NULL, 4);
if (r == 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterCountingFree(bf);
return result;
}
static int BloomFilterCountingTestFull01 (void)
{
int result = 0;
BloomFilterCounting *bf = BloomFilterCountingInit(32, 4, 4, BloomHash);
if (bf == NULL) {
printf("init failed: ");
goto end;
}
int r = BloomFilterCountingAdd(bf, "test", 4);
if (r != 0) {
printf("first add: ");
goto end;
}
r = BloomFilterCountingTest(bf, "test", 4);
if (r != 1) {
printf("2nd add: ");
goto end;
}
r = BloomFilterCountingRemove(bf, "test", 4);
if (r != 0) {
printf("3rd add: ");
goto end;
}
/* all is good! */
result = 1;
end:
if (bf != NULL)
BloomFilterCountingFree(bf);
return result;
}
static int BloomFilterCountingTestFull02 (void)
{
int result = 0;
BloomFilterCounting *bf = BloomFilterCountingInit(32, 4, 4, BloomHash);
if (bf == NULL)
goto end;
int r = BloomFilterCountingTest(bf, "test", 4);
if (r != 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterCountingFree(bf);
return result;
}
#endif
void BloomFilterCountingRegisterTests(void)
{
#ifdef UNITTESTS
UtRegisterTest("BloomFilterCountingTestInit01",
BloomFilterCountingTestInit01);
UtRegisterTest("BloomFilterCountingTestInit02",
BloomFilterCountingTestInit02);
UtRegisterTest("BloomFilterCountingTestInit03",
BloomFilterCountingTestInit03);
UtRegisterTest("BloomFilterCountingTestInit04",
BloomFilterCountingTestInit04);
UtRegisterTest("BloomFilterCountingTestInit05",
BloomFilterCountingTestInit05);
UtRegisterTest("BloomFilterCountingTestInit06",
BloomFilterCountingTestInit06);
UtRegisterTest("BloomFilterCountingTestAdd01",
BloomFilterCountingTestAdd01);
UtRegisterTest("BloomFilterCountingTestAdd02",
BloomFilterCountingTestAdd02);
UtRegisterTest("BloomFilterCountingTestFull01",
BloomFilterCountingTestFull01);
UtRegisterTest("BloomFilterCountingTestFull02",
BloomFilterCountingTestFull02);
#endif
}

45
src/util-bloomfilter-counting.h
Unescape Escape View File

@ -1,45 +0,0 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#ifndef SURICATA_BLOOMFILTERCOUNTING_H
#define SURICATA_BLOOMFILTERCOUNTING_H
/* Bloom filter structure */
typedef struct BloomFilterCounting_ {
uint8_t *array;
uint32_t array_size; /* size in buckets */
uint8_t type; /* 1, 2 or 4 byte counters */
uint8_t hash_iterations;
uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t);
} BloomFilterCounting;
/* prototypes */
BloomFilterCounting *BloomFilterCountingInit(uint32_t, uint8_t, uint8_t, uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t));
void BloomFilterCountingFree(BloomFilterCounting *);
int BloomFilterCountingAdd(BloomFilterCounting *, const void *, uint16_t);
int BloomFilterCountingRemove(BloomFilterCounting *, const void *, uint16_t);
int BloomFilterCountingTest(BloomFilterCounting *, const void *, uint16_t);
void BloomFilterCountingRegisterTests(void);
#endif /* SURICATA_BLOOMFILTERCOUNTING_H */

263
src/util-bloomfilter.c
Unescape Escape View File

@ -1,263 +0,0 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
* Bitwise bloom filter implementation
*/
#include "suricata-common.h"
#include "util-bloomfilter.h"
#include "util-unittest.h"
BloomFilter *BloomFilterInit(uint32_t size, uint8_t iter,
uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t)) {
BloomFilter *bf = NULL;
if (size == 0 || iter == 0)
goto error;
if (Hash == NULL) {
//printf("ERROR: BloomFilterInit no Hash function\n");
goto error;
}
/* setup the filter */
bf = SCCalloc(1, sizeof(BloomFilter));
if (unlikely(bf == NULL))
goto error;
bf->bitarray_size = size;
bf->hash_iterations = iter;
bf->Hash = Hash;
/* setup the bitarray */
bf->bitarray = SCCalloc(1, (bf->bitarray_size / 8) + 1);
if (bf->bitarray == NULL)
goto error;
return bf;
error:
if (bf != NULL) {
if (bf->bitarray != NULL)
SCFree(bf->bitarray);
SCFree(bf);
}
return NULL;
}
void BloomFilterFree(BloomFilter *bf)
{
if (bf != NULL) {
if (bf->bitarray != NULL)
SCFree(bf->bitarray);
SCFree(bf);
}
}
int BloomFilterAdd(BloomFilter *bf, const void *data, uint16_t datalen)
{
uint8_t iter = 0;
uint32_t hash = 0;
if (bf == NULL || data == NULL || datalen == 0)
return -1;
for (iter = 0; iter < bf->hash_iterations; iter++) {
hash = bf->Hash(data, datalen, iter, bf->bitarray_size);
bf->bitarray[hash/8] |= (1<<hash%8);
}
return 0;
}
/*
* ONLY TESTS BELOW THIS COMMENT
*/
#ifdef UNITTESTS
static uint32_t BloomFilterTestHash(const void *data, uint16_t datalen, uint8_t iter, uint32_t hash_size)
{
uint8_t *d = (uint8_t *)data;
uint32_t i;
uint32_t hash = 0;
for (i = 0; i < datalen; i++) {
if (i == 0) hash += (((uint32_t)*d++));
else if (i == 1) hash += (((uint32_t)*d++) * datalen);
else hash *= (((uint32_t)*d++) * i);
}
hash *= (iter + datalen);
hash %= hash_size;
return hash;
}
static int BloomFilterTestInit01 (void)
{
BloomFilter *bf = BloomFilterInit(1024, 4, BloomFilterTestHash);
if (bf == NULL)
return 0;
BloomFilterFree(bf);
return 1;
}
/* no hash function, so it should fail */
static int BloomFilterTestInit02 (void)
{
BloomFilter *bf = BloomFilterInit(1024, 4, NULL);
if (bf == NULL)
return 1;
BloomFilterFree(bf);
return 0;
}
static int BloomFilterTestInit03 (void)
{
int result = 0;
BloomFilter *bf = BloomFilterInit(1024, 4, BloomFilterTestHash);
if (bf == NULL)
return 0;
if (bf->Hash == BloomFilterTestHash)
result = 1;
BloomFilterFree(bf);
return result;
}
static int BloomFilterTestInit04 (void)
{
BloomFilter *bf = BloomFilterInit(1024, 0, BloomFilterTestHash);
if (bf == NULL)
return 1;
BloomFilterFree(bf);
return 0;
}
static int BloomFilterTestInit05 (void)
{
BloomFilter *bf = BloomFilterInit(0, 4, BloomFilterTestHash);
if (bf == NULL)
return 1;
BloomFilterFree(bf);
return 0;
}
static int BloomFilterTestAdd01 (void)
{
int result = 0;
BloomFilter *bf = BloomFilterInit(1024, 4, BloomFilterTestHash);
if (bf == NULL)
return 0;
int r = BloomFilterAdd(bf, "test", 0);
if (r == 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterFree(bf);
return result;
}
static int BloomFilterTestAdd02 (void)
{
int result = 0;
BloomFilter *bf = BloomFilterInit(1024, 4, BloomFilterTestHash);
if (bf == NULL)
return 0;
int r = BloomFilterAdd(bf, NULL, 4);
if (r == 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterFree(bf);
return result;
}
static int BloomFilterTestFull01 (void)
{
int result = 0;
BloomFilter *bf = BloomFilterInit(32, 4, BloomFilterTestHash);
if (bf == NULL)
goto end;
int r = BloomFilterAdd(bf, "test", 4);
if (r != 0)
goto end;
r = BloomFilterTest(bf, "test", 4);
if (r != 1)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterFree(bf);
return result;
}
static int BloomFilterTestFull02 (void)
{
int result = 0;
BloomFilter *bf = BloomFilterInit(32, 4, BloomFilterTestHash);
if (bf == NULL)
goto end;
int r = BloomFilterTest(bf, "test", 4);
if (r != 0)
goto end;
/* all is good! */
result = 1;
end:
if (bf != NULL) BloomFilterFree(bf);
return result;
}
#endif /* UNITTESTS */
void BloomFilterRegisterTests(void)
{
#ifdef UNITTESTS
UtRegisterTest("BloomFilterTestInit01", BloomFilterTestInit01);
UtRegisterTest("BloomFilterTestInit02", BloomFilterTestInit02);
UtRegisterTest("BloomFilterTestInit03", BloomFilterTestInit03);
UtRegisterTest("BloomFilterTestInit04", BloomFilterTestInit04);
UtRegisterTest("BloomFilterTestInit05", BloomFilterTestInit05);
UtRegisterTest("BloomFilterTestAdd01", BloomFilterTestAdd01);
UtRegisterTest("BloomFilterTestAdd02", BloomFilterTestAdd02);
UtRegisterTest("BloomFilterTestFull01", BloomFilterTestFull01);
UtRegisterTest("BloomFilterTestFull02", BloomFilterTestFull02);
#endif /* UNITTESTS */
}

63
src/util-bloomfilter.h
Unescape Escape View File

@ -1,63 +0,0 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#ifndef SURICATA_BLOOMFILTER_H
#define SURICATA_BLOOMFILTER_H
/* Bloom Filter structure */
typedef struct BloomFilter_ {
uint8_t hash_iterations;
uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t);
uint32_t bitarray_size;
uint8_t *bitarray;
} BloomFilter;
/* prototypes */
BloomFilter *BloomFilterInit(uint32_t, uint8_t, uint32_t (*Hash)(const void *, uint16_t, uint8_t, uint32_t));
void BloomFilterFree(BloomFilter *);
int BloomFilterAdd(BloomFilter *, const void *, uint16_t);
void BloomFilterRegisterTests(void);
/** ----- Inline functions ---- */
static inline int BloomFilterTest(const BloomFilter *, const void *, uint16_t);
static inline int BloomFilterTest(const BloomFilter *bf, const void *data, uint16_t datalen)
{
uint8_t iter = 0;
uint32_t hash = 0;
int hit = 1;
for (iter = 0; iter < bf->hash_iterations; iter++) {
hash = bf->Hash(data, datalen, iter, bf->bitarray_size);
if (!(bf->bitarray[hash/8] & (1<<hash%8))) {
hit = 0;
break;
}
}
return hit;
}
#endif /* SURICATA_BLOOMFILTER_H */
Write Preview
Loading…
Cancel
Save