Add hashing and bloomfilter api's

src/util-bloomfilter-counting.c

@@ -0,0 +1,350 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+/* Counting Bloom Filter implementation. Can be used with 8, 16, 32 bits
+ * counters.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <string.h>
+
+#include "util-bloomfilter-counting.h"
+
+#include "util-unittest.h"
+
+/* type: 1, 2 or 4 for 8, 16, or 32 bit counters
+ *
+ */
+BloomFilterCounting *BloomFilterCountingInit(u_int32_t size, u_int8_t type, u_int8_t iter, u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t)) {
+    BloomFilterCounting *bf = NULL;
+
+    if (iter == 0)
+        goto error;
+
+    if (Hash == NULL || size == 0) {
+        //printf("ERROR: BloomFilterCountingInit no Hash function\n");
+        goto error;
+    }
+
+    if (type != 1 && type != 2 && type != 4) {
+        //printf("ERROR: BloomFilterCountingInit only 1, 2 and 4 bytes are supported\n");
+        goto error;
+    }
+
+    /* setup the filter */
+    bf = malloc(sizeof(BloomFilterCounting));
+    if (bf == NULL)
+        goto error;
+    memset(bf,0,sizeof(BloomFilterCounting));
+    bf->type = type; /* size of the type: 1, 2, 4 */
+    bf->array_size = size;
+    bf->hash_iterations = iter;
+    bf->Hash = Hash;
+
+    /* setup the bitarray */
+    bf->array = malloc(bf->array_size * bf->type);
+    if (bf->array == NULL)
+        goto error;
+    memset(bf->array,0,bf->array_size * bf->type);
+
+    return bf;
+
+error:
+    if (bf != NULL) {
+        if (bf->array != NULL)
+            free(bf->array);
+
+        free(bf);
+    }
+    return NULL;
+}
+
+void BloomFilterCountingFree(BloomFilterCounting *bf) {
+    if (bf != NULL) {
+        if (bf->array != NULL)
+            free(bf->array);
+
+        free(bf);
+    }
+}
+
+void BloomFilterCountingPrint(BloomFilterCounting *bf) {
+    printf("\n------ Counting Bloom Filter Stats ------\n");
+    printf("Buckets:               %u\n", bf->array_size);
+    printf("Counter size:          %u\n", bf->type);
+    printf("Memory size:           %u bytes\n", bf->array_size * bf->type);
+    printf("Hash function pointer: %p\n", bf->Hash);
+    printf("Hash functions:        %u\n", bf->hash_iterations);
+    printf("-----------------------------------------\n");
+}
+
+int BloomFilterCountingAdd(BloomFilterCounting *bf, void *data, u_int16_t datalen) {
+    u_int8_t iter = 0;
+    u_int32_t hash = 0;
+
+    if (bf == NULL || data == NULL || datalen == 0)
+        return -1;
+
+    for (iter = 0; iter < bf->hash_iterations; iter++) {
+        hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
+        if (bf->type == 1) {
+            u_int8_t *u8 = (u_int8_t *)&bf->array[hash];
+            if ((*u8) != 255)
+                (*u8)++;
+        } else if (bf->type == 2) {
+            u_int16_t *u16 = (u_int16_t *)&bf->array[hash];
+            if ((*u16) != 65535)
+                (*u16)++;
+        } else if (bf->type == 4) {
+            u_int32_t *u32 = (u_int32_t *)&bf->array[hash];
+            if ((*u32) != 4294967295UL)
+                (*u32)++;
+        }
+    }
+
+    return 0;
+}
+
+int BloomFilterCountingRemove(BloomFilterCounting *bf, void *data, u_int16_t datalen) {
+    u_int8_t iter = 0;
+    u_int32_t hash = 0;
+
+    if (bf == NULL || data == NULL || datalen == 0)
+        return -1;
+
+    /* only remove data that was actually added */
+    if (BloomFilterCountingTest(bf, data, datalen) == 0) {
+        printf("ERROR: BloomFilterCountingRemove tried to remove data "
+               "that was never added to the set or was already removed.\n");
+        return -1;
+    }
+
+    /* decrease counters for every iteration */
+    for (iter = 0; iter < bf->hash_iterations; iter++) {
+        hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
+        if (bf->type == 1) {
+            u_int8_t *u8 = (u_int8_t *)&bf->array[hash];
+            if ((*u8) > 0)
+                (*u8)--;
+            else {
+                printf("ERROR: BloomFilterCountingRemove tried to decrease a "
+                       "counter below zero.\n");
+                return -1;
+            }
+        } else if (bf->type == 2) {
+            u_int16_t *u16 = (u_int16_t *)&bf->array[hash];
+            if ((*u16) > 0)
+                (*u16)--;
+            else {
+                printf("ERROR: BloomFilterCountingRemove tried to decrease a "
+                       "counter below zero.\n");
+                return -1;
+            }
+        } else if (bf->type == 4) {
+            u_int32_t *u32 = (u_int32_t *)&bf->array[hash];
+            if ((*u32) > 0)
+                (*u32)--;
+            else {
+                printf("ERROR: BloomFilterCountingRemove tried to decrease a "
+                       "counter below zero.\n");
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+/* Test if data matches our filter and is likely to be in the set
+ *
+ * returns 0: for no match
+ *         1: match
+ */
+int BloomFilterCountingTest(BloomFilterCounting *bf, void *data, u_int16_t datalen) {
+    u_int8_t iter = 0;
+    u_int32_t hash = 0;
+    int hit = 1;
+
+    /* check each hash iteration */
+    for (iter = 0; iter < bf->hash_iterations; iter++) {
+        hash = bf->Hash(data, datalen, iter, bf->array_size) * bf->type;
+        if (!(bf->array[hash])) {
+            hit = 0;
+            break;
+        }
+    }
+
+    return hit;
+}
+
+static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) {
+     u_int8_t *d = (u_int8_t *)data;
+     u_int32_t i;
+     u_int32_t hash = 0;
+
+     for (i = 0; i < datalen; i++) {
+         if (i == 0)      hash += (((u_int32_t)*d++));
+         else if (i == 1) hash += (((u_int32_t)*d++) * datalen);
+         else             hash *= (((u_int32_t)*d++) * i);
+     }
+
+     hash *= (iter + datalen);
+     hash %= hash_size;
+     return hash;
+}
+
+/*
+ * ONLY TESTS BELOW THIS COMMENT
+ */
+
+static int BloomFilterCountingTestInit01 (void) {
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    BloomFilterCountingFree(bf);
+    return 1;
+}
+
+/* no hash function, so it should fail */
+static int BloomFilterCountingTestInit02 (void) {
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, NULL);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterCountingFree(bf);
+    return 0;
+}
+
+static int BloomFilterCountingTestInit03 (void) {
+    int result = 0;
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    if (bf->Hash == BloomHash)
+        result = 1;
+
+    BloomFilterCountingFree(bf);
+    return result;
+}
+
+static int BloomFilterCountingTestInit04 (void) {
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 0, 4, BloomHash);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterCountingFree(bf);
+    return 0;
+}
+
+static int BloomFilterCountingTestInit05 (void) {
+    BloomFilterCounting *bf = BloomFilterCountingInit(0, 4, 4, BloomHash);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterCountingFree(bf);
+    return 0;
+}
+
+static int BloomFilterCountingTestInit06 (void) {
+    BloomFilterCounting *bf = BloomFilterCountingInit(32, 3, 4, BloomHash);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterCountingFree(bf);
+    return 0;
+}
+
+static int BloomFilterCountingTestAdd01 (void) {
+    int result = 0;
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    int r = BloomFilterCountingAdd(bf, "test", 0);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterCountingFree(bf);
+    return result;
+}
+
+static int BloomFilterCountingTestAdd02 (void) {
+    int result = 0;
+    BloomFilterCounting *bf = BloomFilterCountingInit(1024, 4, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    int r = BloomFilterCountingAdd(bf, NULL, 4);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterCountingFree(bf);
+    return result;
+}
+
+static int BloomFilterCountingTestFull01 (void) {
+    int result = 0;
+    BloomFilterCounting *bf = BloomFilterCountingInit(32, 4, 4, BloomHash);
+    if (bf == NULL)
+        goto end;
+
+    int r = BloomFilterCountingAdd(bf, "test", 4);
+    if (r != 0)
+        goto end;
+
+    r = BloomFilterCountingTest(bf, "test", 4);
+    if (r != 1)
+        goto end;
+
+    r = BloomFilterCountingRemove(bf, "test", 4);
+    if (r != 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterCountingFree(bf);
+    return result;
+}
+
+static int BloomFilterCountingTestFull02 (void) {
+    int result = 0;
+    BloomFilterCounting *bf = BloomFilterCountingInit(32, 4, 4, BloomHash);
+    if (bf == NULL)
+        goto end;
+
+    int r = BloomFilterCountingTest(bf, "test", 4);
+    if (r != 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterCountingFree(bf);
+    return result;
+}
+
+void BloomFilterCountingRegisterTests(void) {
+    UtRegisterTest("BloomFilterCountingTestInit01", BloomFilterCountingTestInit01, 1);
+    UtRegisterTest("BloomFilterCountingTestInit02", BloomFilterCountingTestInit02, 1);
+    UtRegisterTest("BloomFilterCountingTestInit03", BloomFilterCountingTestInit03, 1);
+    UtRegisterTest("BloomFilterCountingTestInit04", BloomFilterCountingTestInit04, 1);
+    UtRegisterTest("BloomFilterCountingTestInit05", BloomFilterCountingTestInit05, 1);
+    UtRegisterTest("BloomFilterCountingTestInit06", BloomFilterCountingTestInit06, 1);
+
+    UtRegisterTest("BloomFilterCountingTestAdd01", BloomFilterCountingTestAdd01, 1);
+    UtRegisterTest("BloomFilterCountingTestAdd02", BloomFilterCountingTestAdd02, 1);
+
+    UtRegisterTest("BloomFilterCountingTestFull01", BloomFilterCountingTestFull01, 1);
+    UtRegisterTest("BloomFilterCountingTestFull02", BloomFilterCountingTestFull02, 1);
+}
+

src/util-bloomfilter-counting.h

@@ -0,0 +1,24 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+#ifndef __BLOOMFILTERCOUNTING_H__
+#define __BLOOMFILTERCOUNTING_H__
+
+/* Bloom filter structure */
+typedef struct _BloomFilterCounting {
+    u_int8_t *array;
+    u_int32_t array_size; /* size in buckets */
+    u_int8_t type; /* 1, 2 or 4 byte counters */
+    u_int8_t hash_iterations;
+    u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t);
+} BloomFilterCounting;
+
+/* prototypes */
+BloomFilterCounting *BloomFilterCountingInit(u_int32_t, u_int8_t, u_int8_t, u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t));
+void BloomFilterCountingFree(BloomFilterCounting *);
+void BloomFilterCountingPrint(BloomFilterCounting *);
+int BloomFilterCountingAdd(BloomFilterCounting *, void *, u_int16_t);
+int BloomFilterCountingRemove(BloomFilterCounting *, void *, u_int16_t);
+int BloomFilterCountingTest(BloomFilterCounting *, void *, u_int16_t);
+
+#endif /* __BLOOMFILTERCOUNTING_H__ */
+

src/util-bloomfilter.c

@@ -0,0 +1,256 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+/* Bitwise bloom filter implementation. */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <string.h>
+
+#include "util-bloomfilter.h"
+
+#include "util-unittest.h"
+
+BloomFilter *BloomFilterInit(u_int32_t size, u_int8_t iter, u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t)) {
+    BloomFilter *bf = NULL;
+
+    if (size == 0 || iter == 0)
+        goto error;
+
+    if (Hash == NULL) {
+        //printf("ERROR: BloomFilterInit no Hash function\n");
+        goto error;
+    }
+
+    /* setup the filter */
+    bf = malloc(sizeof(BloomFilter));
+    if (bf == NULL)
+        goto error;
+    memset(bf,0,sizeof(BloomFilter));
+    bf->bitarray_size = size;
+    bf->hash_iterations = iter;
+    bf->Hash = Hash;
+
+    /* setup the bitarray */
+    bf->bitarray = malloc(bf->bitarray_size/8);
+    if (bf->bitarray == NULL)
+        goto error;
+    memset(bf->bitarray,0,bf->bitarray_size/8);
+
+    return bf;
+
+error:
+    if (bf != NULL) {
+        if (bf->bitarray != NULL)
+            free(bf->bitarray);
+
+        free(bf);
+    }
+    return NULL;
+}
+
+void BloomFilterFree(BloomFilter *bf) {
+    if (bf != NULL) {
+        if (bf->bitarray != NULL)
+            free(bf->bitarray);
+
+        free(bf);
+    }
+}
+
+void BloomFilterPrint(BloomFilter *bf) {
+    printf("\n---------- Bloom Filter Stats -----------\n");
+    printf("Buckets:               %u\n", bf->bitarray_size);
+    printf("Memory size:           %u bytes\n", bf->bitarray_size/8);
+    printf("Hash function pointer: %p\n", bf->Hash);
+    printf("Hash functions:        %u\n", bf->hash_iterations);
+    printf("-----------------------------------------\n");
+}
+
+int BloomFilterAdd(BloomFilter *bf, void *data, u_int16_t datalen) {
+    u_int8_t iter = 0;
+    u_int32_t hash = 0;
+
+    if (bf == NULL || data == NULL || datalen == 0)
+        return -1;
+
+    for (iter = 0; iter < bf->hash_iterations; iter++) {
+        hash = bf->Hash(data, datalen, iter, bf->bitarray_size);
+        bf->bitarray[hash/8] |= (1<<hash%8);
+    }
+
+    return 0;
+}
+
+int BloomFilterTest(BloomFilter *bf, void *data, u_int16_t datalen) {
+    u_int8_t iter = 0;
+    u_int32_t hash = 0;
+    int hit = 1;
+
+    for (iter = 0; iter < bf->hash_iterations; iter++) {
+        hash = bf->Hash(data, datalen, iter, bf->bitarray_size);
+        if (!(bf->bitarray[hash/8] & (1<<hash%8))) {
+            hit = 0;
+            break;
+        }
+    }
+
+    return hit;
+}
+
+static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) {
+     u_int8_t *d = (u_int8_t *)data;
+     u_int32_t i;
+     u_int32_t hash = 0;
+
+     for (i = 0; i < datalen; i++) {
+         if (i == 0)      hash += (((u_int32_t)*d++));
+         else if (i == 1) hash += (((u_int32_t)*d++) * datalen);
+         else             hash *= (((u_int32_t)*d++) * i);
+     }
+
+     hash *= (iter + datalen);
+     hash %= hash_size;
+     return hash;
+}
+
+/*
+ * ONLY TESTS BELOW THIS COMMENT
+ */
+
+static int BloomFilterTestInit01 (void) {
+    BloomFilter *bf = BloomFilterInit(1024, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    BloomFilterFree(bf);
+    return 1;
+}
+
+/* no hash function, so it should fail */
+static int BloomFilterTestInit02 (void) {
+    BloomFilter *bf = BloomFilterInit(1024, 4, NULL);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterFree(bf);
+    return 0;
+}
+
+static int BloomFilterTestInit03 (void) {
+    int result = 0;
+    BloomFilter *bf = BloomFilterInit(1024, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    if (bf->Hash == BloomHash)
+        result = 1;
+
+    BloomFilterFree(bf);
+    return result;
+}
+
+static int BloomFilterTestInit04 (void) {
+    BloomFilter *bf = BloomFilterInit(1024, 0, BloomHash);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterFree(bf);
+    return 0;
+}
+
+static int BloomFilterTestInit05 (void) {
+    BloomFilter *bf = BloomFilterInit(0, 4, BloomHash);
+    if (bf == NULL)
+        return 1;
+
+    BloomFilterFree(bf);
+    return 0;
+}
+
+static int BloomFilterTestAdd01 (void) {
+    int result = 0;
+    BloomFilter *bf = BloomFilterInit(1024, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    int r = BloomFilterAdd(bf, "test", 0);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterFree(bf);
+    return result;
+}
+
+static int BloomFilterTestAdd02 (void) {
+    int result = 0;
+    BloomFilter *bf = BloomFilterInit(1024, 4, BloomHash);
+    if (bf == NULL)
+        return 0;
+
+    int r = BloomFilterAdd(bf, NULL, 4);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterFree(bf);
+    return result;
+}
+
+static int BloomFilterTestFull01 (void) {
+    int result = 0;
+    BloomFilter *bf = BloomFilterInit(32, 4, BloomHash);
+    if (bf == NULL)
+        goto end;
+
+    int r = BloomFilterAdd(bf, "test", 4);
+    if (r != 0)
+        goto end;
+
+    r = BloomFilterTest(bf, "test", 4);
+    if (r != 1)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterFree(bf);
+    return result;
+}
+
+static int BloomFilterTestFull02 (void) {
+    int result = 0;
+    BloomFilter *bf = BloomFilterInit(32, 4, BloomHash);
+    if (bf == NULL)
+        goto end;
+
+    int r = BloomFilterTest(bf, "test", 4);
+    if (r != 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (bf != NULL) BloomFilterFree(bf);
+    return result;
+}
+
+void BloomFilterRegisterTests(void) {
+    UtRegisterTest("BloomFilterTestInit01", BloomFilterTestInit01, 1);
+    UtRegisterTest("BloomFilterTestInit02", BloomFilterTestInit02, 1);
+    UtRegisterTest("BloomFilterTestInit03", BloomFilterTestInit03, 1);
+    UtRegisterTest("BloomFilterTestInit04", BloomFilterTestInit04, 1);
+    UtRegisterTest("BloomFilterTestInit05", BloomFilterTestInit05, 1);
+
+    UtRegisterTest("BloomFilterTestAdd01", BloomFilterTestAdd01, 1);
+    UtRegisterTest("BloomFilterTestAdd02", BloomFilterTestAdd02, 1);
+
+    UtRegisterTest("BloomFilterTestFull01", BloomFilterTestFull01, 1);
+    UtRegisterTest("BloomFilterTestFull02", BloomFilterTestFull02, 1);
+}
+

src/util-bloomfilter.h

@@ -0,0 +1,22 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+#ifndef __BLOOMFILTER_H__
+#define __BLOOMFILTER_H__
+
+/* Bloom Filter structure */
+typedef struct _BloomFilter {
+    u_int8_t *bitarray;
+    u_int32_t bitarray_size;
+    u_int8_t hash_iterations;
+    u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t);
+} BloomFilter;
+
+/* prototypes */
+BloomFilter *BloomFilterInit(u_int32_t, u_int8_t, u_int32_t (*Hash)(void *, u_int16_t, u_int8_t, u_int32_t));
+void BloomFilterFree(BloomFilter *);
+void BloomFilterPrint(BloomFilter *);
+int BloomFilterAdd(BloomFilter *, void *, u_int16_t);
+int BloomFilterTest(BloomFilter *, void *, u_int16_t);
+
+#endif /* __BLOOMFILTER_H__ */
+

src/util-hash.c

@@ -0,0 +1,342 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+/* Chained hash table implementation
+ *
+ * The 'Free' pointer can be used to have the API free your
+ * hashed data. If it's NULL it's the callers responsebility */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <string.h>
+
+#include "util-hash.h"
+
+#include "util-unittest.h"
+
+HashTable* HashTableInit(u_int32_t size, u_int32_t (*Hash)(struct _HashTable *, void *, u_int16_t), void (*Free)(void *)) {
+
+    HashTable *ht = NULL;
+
+    if (size == 0) {
+        goto error;
+    }
+
+    if (Hash == NULL) {
+        //printf("ERROR: HashTableInit no Hash function\n");
+        goto error;
+    }
+
+    /* setup the filter */
+    ht = malloc(sizeof(HashTable));
+    if (ht == NULL)
+        goto error;
+    memset(ht,0,sizeof(HashTable));
+    ht->array_size = size;
+    ht->Hash = Hash;
+    ht->Free = Free;
+
+    /* setup the bitarray */
+    ht->array = malloc(ht->array_size * sizeof(HashTableBucket *));
+    if (ht->array == NULL)
+        goto error;
+    memset(ht->array,0,ht->array_size * sizeof(HashTableBucket *));
+
+    return ht;
+
+error:
+    if (ht != NULL) {
+        if (ht->array != NULL)
+            free(ht->array);
+
+        free(ht);
+    }
+    return NULL;
+}
+
+void HashTableFree(HashTable *ht) {
+    u_int32_t i = 0;
+
+    if (ht == NULL)
+        return;
+
+    /* free the buckets */
+    for (i = 0; i < ht->array_size; i++) {
+        HashTableBucket *hashbucket = ht->array[i];
+        while (hashbucket != NULL) {
+            HashTableBucket *next_hashbucket = hashbucket->next;
+            if (ht->Free != NULL)
+                ht->Free(hashbucket->data);
+            free(hashbucket);
+            hashbucket = next_hashbucket;
+        }
+    }
+
+    /* free the arrray */
+    if (ht->array != NULL)
+        free(ht->array);
+
+    free(ht);
+}
+
+void HashTablePrint(HashTable *ht) {
+    printf("\n----------- Hash Table Stats ------------\n");
+    printf("Buckets:               %u\n", ht->array_size);
+    printf("Hash function pointer: %p\n", ht->Hash);
+    printf("-----------------------------------------\n");
+}
+
+int HashTableAdd(HashTable *ht, void *data, u_int16_t datalen) {
+    if (ht == NULL || data == NULL || datalen == 0)
+        return -1;
+
+    u_int32_t hash = ht->Hash(ht, data, datalen);
+
+    HashTableBucket *hb = malloc(sizeof(HashTableBucket));
+    if (hb == NULL) {
+        goto error;
+    }
+    memset(hb, 0, sizeof(HashTableBucket));
+    hb->data = data;
+    hb->size = datalen;
+    hb->next = NULL;
+
+    if (ht->array[hash] == NULL) {
+        ht->array[hash] = hb;
+    } else {
+        hb->next = ht->array[hash];
+        ht->array[hash] = hb;
+    }
+
+    return 0;
+
+error:
+    return -1;
+}
+
+int HashTableRemove(HashTable *ht, void *data, u_int16_t datalen) {
+    u_int32_t hash = ht->Hash(ht, data, datalen);
+
+    if (ht->array[hash] == NULL) {
+        return -1;
+    }
+
+    if (ht->array[hash]->next == NULL) {
+        if (ht->Free != NULL)
+            ht->Free(ht->array[hash]->data);
+        free(ht->array[hash]);
+        ht->array[hash] = NULL;
+        return 0;
+    }
+
+    HashTableBucket *hashbucket = ht->array[hash], *prev_hashbucket = NULL;
+    do {
+        if (hashbucket->size != datalen) {
+            prev_hashbucket = hashbucket;
+            hashbucket = hashbucket->next;
+            continue;
+        }
+
+        if (memcmp(hashbucket->data,data,datalen) == 0) {
+            if (prev_hashbucket == NULL) {
+                /* root bucket */
+                ht->array[hash] = hashbucket->next;
+            } else {
+                /* child bucket */
+                prev_hashbucket->next = hashbucket->next;
+            }
+
+            /* remove this */
+            if (ht->Free != NULL)
+                ht->Free(hashbucket->data);
+            free(hashbucket);
+            return 0;
+        }
+
+        prev_hashbucket = hashbucket;
+        hashbucket = hashbucket->next;
+    } while (hashbucket != NULL);
+
+    return -1;
+}
+
+void *HashTableLookup(HashTable *ht, void *data, u_int16_t datalen) {
+    u_int32_t hash = ht->Hash(ht, data, datalen);
+
+    if (ht->array[hash] == NULL)
+        return NULL;
+
+    HashTableBucket *hashbucket = ht->array[hash];
+    do {
+        if (hashbucket->size != datalen) {
+            hashbucket = hashbucket->next;
+            continue;
+        }
+
+        if (memcmp(hashbucket->data,data,datalen) == 0)
+            return hashbucket->data;
+
+        hashbucket = hashbucket->next;
+    } while (hashbucket != NULL);
+
+    return NULL;
+}
+
+u_int32_t HashTableGenericHash(HashTable *ht, void *data, u_int16_t datalen) {
+     u_int8_t *d = (u_int8_t *)data;
+     u_int32_t i;
+     u_int32_t hash = 0;
+
+     for (i = 0; i < datalen; i++) {
+         if (i == 0)      hash += (((u_int32_t)*d++));
+         else if (i == 1) hash += (((u_int32_t)*d++) * datalen);
+         else             hash *= (((u_int32_t)*d++) * i) + datalen + i;
+     }
+
+     hash *= datalen;
+     hash %= ht->array_size;
+     return hash;
+}
+
+/*
+ * ONLY TESTS BELOW THIS COMMENT
+ */
+
+static int HashTableTestInit01 (void) {
+    HashTable *ht = HashTableInit(1024, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        return 0;
+
+    HashTableFree(ht);
+    return 1;
+}
+
+/* no hash function, so it should fail */
+static int HashTableTestInit02 (void) {
+    HashTable *ht = HashTableInit(1024, NULL, NULL);
+    if (ht == NULL)
+        return 1;
+
+    HashTableFree(ht);
+    return 0;
+}
+
+static int HashTableTestInit03 (void) {
+    int result = 0;
+    HashTable *ht = HashTableInit(1024, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        return 0;
+
+    if (ht->Hash == HashTableGenericHash)
+        result = 1;
+
+    HashTableFree(ht);
+    return result;
+}
+
+static int HashTableTestInit04 (void) {
+    HashTable *ht = HashTableInit(0, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        return 1;
+
+    HashTableFree(ht);
+    return 0;
+}
+
+static int HashTableTestAdd01 (void) {
+    int result = 0;
+    HashTable *ht = HashTableInit(32, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        goto end;
+
+    int r = HashTableAdd(ht, "test", 0);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (ht != NULL) HashTableFree(ht);
+    return result;
+}
+
+static int HashTableTestAdd02 (void) {
+    int result = 0;
+    HashTable *ht = HashTableInit(32, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        goto end;
+
+    int r = HashTableAdd(ht, NULL, 4);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (ht != NULL) HashTableFree(ht);
+    return result;
+}
+
+static int HashTableTestFull01 (void) {
+    int result = 0;
+    HashTable *ht = HashTableInit(32, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        goto end;
+
+    int r = HashTableAdd(ht, "test", 4);
+    if (r != 0)
+        goto end;
+
+    char *rp = HashTableLookup(ht, "test", 4);
+    if (rp == NULL)
+        goto end;
+
+    r = HashTableRemove(ht, "test", 4);
+    if (r != 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (ht != NULL) HashTableFree(ht);
+    return result;
+}
+
+static int HashTableTestFull02 (void) {
+    int result = 0;
+    HashTable *ht = HashTableInit(32, HashTableGenericHash, NULL);
+    if (ht == NULL)
+        goto end;
+
+    int r = HashTableAdd(ht, "test", 4);
+    if (r != 0)
+        goto end;
+
+    char *rp = HashTableLookup(ht, "test", 4);
+    if (rp == NULL)
+        goto end;
+
+    r = HashTableRemove(ht, "test2", 5);
+    if (r == 0)
+        goto end;
+
+    /* all is good! */
+    result = 1;
+end:
+    if (ht != NULL) HashTableFree(ht);
+    return result;
+}
+
+void HashTableRegisterTests(void) {
+    UtRegisterTest("HashTableTestInit01", HashTableTestInit01, 1);
+    UtRegisterTest("HashTableTestInit02", HashTableTestInit02, 1);
+    UtRegisterTest("HashTableTestInit03", HashTableTestInit03, 1);
+    UtRegisterTest("HashTableTestInit04", HashTableTestInit04, 1);
+
+    UtRegisterTest("HashTableTestAdd01", HashTableTestAdd01, 1);
+    UtRegisterTest("HashTableTestAdd02", HashTableTestAdd02, 1);
+
+    UtRegisterTest("HashTableTestFull01", HashTableTestFull01, 1);
+    UtRegisterTest("HashTableTestFull02", HashTableTestFull02, 1);
+}
+

src/util-hash.h

@@ -0,0 +1,32 @@
+/* Copyright (c) 2008 by Victor Julien <victor@inliniac.net> */
+
+#ifndef __HASH_H__
+#define __HASH_H__
+
+/* hash bucket structure */
+typedef struct _HashTableBucket {
+    void *data;
+    u_int16_t size;
+    struct _HashTableBucket *next;
+} HashTableBucket;
+
+/* hash table structure */
+typedef struct _HashTable {
+    HashTableBucket **array;
+    u_int32_t array_size;
+    u_int32_t (*Hash)(struct _HashTable *, void *, u_int16_t);
+    void (*Free)(void *);
+} HashTable;
+
+/* prototypes */
+HashTable* HashTableInit(u_int32_t, u_int32_t (*Hash)(struct _HashTable *, void *, u_int16_t), void (*Free)(void *));
+void HashTableFree(HashTable *);
+void HashTablePrint(HashTable *);
+int HashTableAdd(HashTable *, void *, u_int16_t);
+int HashTableRemove(HashTable *, void *, u_int16_t);
+void *HashTableLookup(HashTable *, void *, u_int16_t);
+u_int32_t HashTableGenericHash(HashTable *, void *, u_int16_t);
+
+
+#endif /* __HASH_H__ */
+

src/vips.c

@@ -20,7 +20,11 @@
 #include "packet-queue.h"
 #include "threads.h"
 #include "threadvars.h"
+
 #include "util-binsearch.h"
+#include "util-hash.h"
+#include "util-bloomfilter.h"
+#include "util-bloomfilter-counting.h"
 
 #include "detect-parse.h"
 #include "detect-engine-mpm.h"
@@ -193,9 +197,12 @@ int main(int argc, char **argv)
     MpmRegisterTests();
     SigTableRegisterTests();
     SigRegisterTests();
-    //UtRunTests();
+    HashTableRegisterTests();
+    BloomFilterRegisterTests();
+    BloomFilterCountingRegisterTests();
+    UtRunTests();
     UtCleanup();
-    //exit(1);
+    exit(1);
 
     //LoadConfig();
     //exit(1);