datasets: introduce ipv4 type

This patch introduce the IPv4 type for dataset so Suricata commandmatch
on a set of IPv4 addresses. This is meant to complement iprep feature
for people that needs more flexibility such as settings the IP on
the packet path.

Feature: #5383

src/Makefile.am

@@ -61,6 +61,7 @@ noinst_HEADERS = \
 	conf-yaml-loader.h \
 	counters.h \
 	datasets.h \
+	datasets-ipv4.h \
 	datasets-md5.h \
 	datasets-reputation.h \
 	datasets-sha256.h \
@@ -667,6 +668,7 @@ libsuricata_c_a_SOURCES = \
 	conf-yaml-loader.c \
 	counters.c \
 	datasets.c \
+	datasets-ipv4.c \
 	datasets-md5.c \
 	datasets-sha256.c \
 	datasets-string.c \

src/datasets-ipv4.c

@@ -0,0 +1,62 @@
+/* Copyright (C) 2022 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Eric Leblond <el@stamus-networks.com>
+ */
+
+#include "suricata-common.h"
+#include "conf.h"
+#include "datasets.h"
+#include "datasets-ipv4.h"
+#include "util-thash.h"
+#include "util-print.h"
+
+int IPv4Set(void *dst, void *src)
+{
+    IPv4Type *src_s = src;
+    IPv4Type *dst_s = dst;
+    memcpy(dst_s->ipv4, src_s->ipv4, sizeof(dst_s->ipv4));
+    dst_s->rep = src_s->rep;
+    return 0;
+}
+
+bool IPv4Compare(void *a, void *b)
+{
+    const IPv4Type *as = a;
+    const IPv4Type *bs = b;
+
+    return (memcmp(as->ipv4, bs->ipv4, sizeof(as->ipv4)) == 0);
+}
+
+uint32_t IPv4Hash(void *s)
+{
+    const IPv4Type *str = s;
+    uint32_t hash = 5381;
+
+    for (int i = 0; i < (int)sizeof(str->ipv4); i++) {
+        hash = ((hash << 5) + hash) + str->ipv4[i]; /* hash * 33 + c */
+    }
+    return hash;
+}
+
+// data stays in hash
+void IPv4Free(void *s)
+{
+}

src/datasets-ipv4.h

@@ -0,0 +1,39 @@
+/* Copyright (C) 2022 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Eric Leblond <el@stamus-networks.com>
+ */
+
+#ifndef __DATASETS_IPV4_H__
+#define __DATASETS_IPV4_H__
+
+#include "datasets-reputation.h"
+
+typedef struct IPv4Type {
+    uint8_t ipv4[4];
+    DataRepType rep;
+} IPv4Type;
+
+int IPv4Set(void *dst, void *src);
+bool IPv4Compare(void *a, void *b);
+uint32_t IPv4Hash(void *s);
+void IPv4Free(void *s);
+
+#endif /* __DATASETS_IPV4_H__ */

src/datasets.c

@@ -25,6 +25,7 @@
 #include "conf.h"
 #include "datasets.h"
 #include "datasets-string.h"
+#include "datasets-ipv4.h"
 #include "datasets-md5.h"
 #include "datasets-sha256.h"
 #include "datasets-reputation.h"
@@ -60,6 +61,8 @@ enum DatasetTypes DatasetGetTypeFromString(const char *s)
         return DATASET_TYPE_SHA256;
     if (strcasecmp("string", s) == 0)
         return DATASET_TYPE_STRING;
+    if (strcasecmp("ipv4", s) == 0)
+        return DATASET_TYPE_IPV4;
     return DATASET_TYPE_NOTSET;
 }
 
@@ -159,6 +162,72 @@ static int ParseRepLine(const char *in, size_t ins, DataRepType *rep_out)
     return 0;
 }
 
+static int DatasetLoadIPv4(Dataset *set)
+{
+    if (strlen(set->load) == 0)
+        return 0;
+
+    SCLogConfig("dataset: %s loading from '%s'", set->name, set->load);
+    const char *fopen_mode = "r";
+    if (strlen(set->save) > 0 && strcmp(set->save, set->load) == 0) {
+        fopen_mode = "a+";
+    }
+
+    FILE *fp = fopen(set->load, fopen_mode);
+    if (fp == NULL) {
+        SCLogError(SC_ERR_DATASET, "fopen '%s' failed: %s", set->load, strerror(errno));
+        return -1;
+    }
+
+    uint32_t cnt = 0;
+    char line[1024];
+    while (fgets(line, (int)sizeof(line), fp) != NULL) {
+        char *r = strchr(line, ',');
+        if (r == NULL) {
+            line[strlen(line) - 1] = '\0';
+            SCLogDebug("line: '%s'", line);
+
+            struct in_addr in;
+            if (inet_pton(AF_INET, line, &in) != 1)
+                FatalError(SC_ERR_FATAL, "dataset data parse failed %s/%s: %s", set->name,
+                        set->load, line);
+
+            if (DatasetAdd(set, (const uint8_t *)&in.s_addr, 4) < 0)
+                FatalError(SC_ERR_FATAL, "dataset data add failed %s/%s", set->name, set->load);
+            cnt++;
+
+            /* list with rep data */
+        } else {
+            line[strlen(line) - 1] = '\0';
+            SCLogDebug("IPv4 with REP line: '%s'", line);
+
+            *r = '\0';
+
+            struct in_addr in;
+            if (inet_pton(AF_INET, line, &in) != 1)
+                FatalError(SC_ERR_FATAL, "dataset data parse failed %s/%s: %s", set->name,
+                        set->load, line);
+
+            r++;
+
+            DataRepType rep = { .value = 0 };
+            if (ParseRepLine(r, strlen(r), &rep) < 0)
+                FatalError(SC_ERR_FATAL, "bad rep for dataset %s/%s", set->name, set->load);
+
+            SCLogDebug("rep v:%u", rep.value);
+            if (DatasetAddwRep(set, (const uint8_t *)&in.s_addr, 4, &rep) < 0)
+                FatalError(SC_ERR_FATAL, "dataset data add failed %s/%s", set->name, set->load);
+
+            cnt++;
+        }
+    }
+    THashConsolidateMemcap(set->hash);
+
+    fclose(fp);
+    SCLogConfig("dataset: %s loaded %u records", set->name, cnt);
+    return 0;
+}
+
 static int DatasetLoadMd5(Dataset *set)
 {
     if (strlen(set->load) == 0)
@@ -528,6 +597,15 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             if (DatasetLoadSha256(set) < 0)
                 goto out_err;
             break;
+        case DATASET_TYPE_IPV4:
+            set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash,
+                    IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    hashsize > 0 ? hashsize : default_hashsize);
+            if (set->hash == NULL)
+                goto out_err;
+            if (DatasetLoadIPv4(set) < 0)
+                goto out_err;
+            break;
     }
 
     SCLogDebug("set %p/%s type %u save %s load %s",
@@ -790,6 +868,16 @@ static int Sha256AsAscii(const void *s, char *out, size_t out_size)
     return strlen(out);
 }
 
+static int IPv4AsAscii(const void *s, char *out, size_t out_size)
+{
+    const IPv4Type *ip4 = s;
+    char str[256];
+    PrintInet(AF_INET, ip4->ipv4, str, sizeof(str));
+    strlcat(out, str, out_size);
+    strlcat(out, "\n", out_size);
+    return strlen(out);
+}
+
 void DatasetsSave(void)
 {
     SCLogDebug("saving datasets: %p", sets);
@@ -815,6 +903,9 @@ void DatasetsSave(void)
             case DATASET_TYPE_SHA256:
                 THashWalk(set->hash, Sha256AsAscii, SaveCallback, fp);
                 break;
+            case DATASET_TYPE_IPV4:
+                THashWalk(set->hash, IPv4AsAscii, SaveCallback, fp);
+                break;
         }
 
         fclose(fp);
@@ -859,6 +950,48 @@ static DataRepResultType DatasetLookupStringwRep(Dataset *set,
     return rrep;
 }
 
+static int DatasetLookupIPv4(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 4)
+        return -1;
+
+    IPv4Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv4, data, 4);
+    THashData *rdata = THashLookupFromHash(set->hash, &lookup);
+    if (rdata) {
+        DatasetUnlockData(rdata);
+        return 1;
+    }
+    return 0;
+}
+
+static DataRepResultType DatasetLookupIPv4wRep(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
+{
+    DataRepResultType rrep = { .found = false, .rep = { .value = 0 } };
+
+    if (set == NULL)
+        return rrep;
+
+    if (data_len != 4)
+        return rrep;
+
+    IPv4Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv4, data, data_len);
+    THashData *rdata = THashLookupFromHash(set->hash, &lookup);
+    if (rdata) {
+        IPv4Type *found = rdata->data;
+        rrep.found = true;
+        rrep.rep = found->rep;
+        DatasetUnlockData(rdata);
+        return rrep;
+    }
+    return rrep;
+}
+
 static int DatasetLookupMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -964,6 +1097,8 @@ int DatasetLookup(Dataset *set, const uint8_t *data, const uint32_t data_len)
             return DatasetLookupMd5(set, data, data_len);
         case DATASET_TYPE_SHA256:
             return DatasetLookupSha256(set, data, data_len);
+        case DATASET_TYPE_IPV4:
+            return DatasetLookupIPv4(set, data, data_len);
     }
     return -1;
 }
@@ -982,6 +1117,8 @@ DataRepResultType DatasetLookupwRep(Dataset *set, const uint8_t *data, const uin
             return DatasetLookupMd5wRep(set, data, data_len, rep);
         case DATASET_TYPE_SHA256:
             return DatasetLookupSha256wRep(set, data, data_len, rep);
+        case DATASET_TYPE_IPV4:
+            return DatasetLookupIPv4wRep(set, data, data_len, rep);
     }
     return rrep;
 }
@@ -1027,6 +1164,45 @@ static int DatasetAddStringwRep(
     return -1;
 }
 
+static int DatasetAddIPv4(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL) {
+        return -1;
+    }
+
+    if (data_len < 4) {
+        return -2;
+    }
+
+    IPv4Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv4, data, 4);
+    struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
+    if (res.data) {
+        DatasetUnlockData(res.data);
+        return res.is_new ? 1 : 0;
+    }
+    return -1;
+}
+
+static int DatasetAddIPv4wRep(
+        Dataset *set, const uint8_t *data, const uint32_t data_len, const DataRepType *rep)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len < 4)
+        return -2;
+
+    IPv4Type lookup = { .rep = *rep };
+    memcpy(lookup.ipv4, data, 4);
+    struct THashDataGetResult res = THashGetFromHash(set->hash, &lookup);
+    if (res.data) {
+        DatasetUnlockData(res.data);
+        return res.is_new ? 1 : 0;
+    }
+    return -1;
+}
+
 static int DatasetAddMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -1113,6 +1289,8 @@ int DatasetAdd(Dataset *set, const uint8_t *data, const uint32_t data_len)
             return DatasetAddMd5(set, data, data_len);
         case DATASET_TYPE_SHA256:
             return DatasetAddSha256(set, data, data_len);
+        case DATASET_TYPE_IPV4:
+            return DatasetAddIPv4(set, data, data_len);
     }
     return -1;
 }
@@ -1130,6 +1308,8 @@ static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data
             return DatasetAddMd5wRep(set, data, data_len, rep);
         case DATASET_TYPE_SHA256:
             return DatasetAddSha256wRep(set, data, data_len, rep);
+        case DATASET_TYPE_IPV4:
+            return DatasetAddIPv4wRep(set, data, data_len, rep);
     }
     return -1;
 }
@@ -1137,7 +1317,7 @@ static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data
 typedef int (*DatasetOpFunc)(Dataset *set, const uint8_t *data, const uint32_t data_len);
 
 static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc DatasetOpString,
-        DatasetOpFunc DatasetOpMd5, DatasetOpFunc DatasetOpSha256)
+        DatasetOpFunc DatasetOpMd5, DatasetOpFunc DatasetOpSha256, DatasetOpFunc DatasetOpIPv4)
 {
     if (set == NULL)
         return -1;
@@ -1171,6 +1351,12 @@ static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc D
                 return -2;
             return DatasetOpSha256(set, hash, 32);
         }
+        case DATASET_TYPE_IPV4: {
+            struct in_addr in;
+            if (inet_pton(AF_INET, string, &in) != 1)
+                return -2;
+            return DatasetOpIPv4(set, (uint8_t *)&in.s_addr, 4);
+        }
     }
     return -1;
 }
@@ -1183,7 +1369,8 @@ static int DatasetOpSerialized(Dataset *set, const char *string, DatasetOpFunc D
  */
 int DatasetAddSerialized(Dataset *set, const char *string)
 {
-    return DatasetOpSerialized(set, string, DatasetAddString, DatasetAddMd5, DatasetAddSha256);
+    return DatasetOpSerialized(
+            set, string, DatasetAddString, DatasetAddMd5, DatasetAddSha256, DatasetAddIPv4);
 }
 
 /** \brief add serialized data to set
@@ -1194,8 +1381,8 @@ int DatasetAddSerialized(Dataset *set, const char *string)
  */
 int DatasetLookupSerialized(Dataset *set, const char *string)
 {
-    return DatasetOpSerialized(
-            set, string, DatasetLookupString, DatasetLookupMd5, DatasetLookupSha256);
+    return DatasetOpSerialized(set, string, DatasetLookupString, DatasetLookupMd5,
+            DatasetLookupSha256, DatasetLookupIPv4);
 }
 
 /**
@@ -1213,6 +1400,19 @@ static int DatasetRemoveString(Dataset *set, const uint8_t *data, const uint32_t
     return THashRemoveFromHash(set->hash, &lookup);
 }
 
+static int DatasetRemoveIPv4(Dataset *set, const uint8_t *data, const uint32_t data_len)
+{
+    if (set == NULL)
+        return -1;
+
+    if (data_len != 4)
+        return -2;
+
+    IPv4Type lookup = { .rep.value = 0 };
+    memcpy(lookup.ipv4, data, 4);
+    return THashRemoveFromHash(set->hash, &lookup);
+}
+
 static int DatasetRemoveMd5(Dataset *set, const uint8_t *data, const uint32_t data_len)
 {
     if (set == NULL)
@@ -1246,6 +1446,6 @@ static int DatasetRemoveSha256(Dataset *set, const uint8_t *data, const uint32_t
  *  \retval int -2 DATA error */
 int DatasetRemoveSerialized(Dataset *set, const char *string)
 {
-    return DatasetOpSerialized(
-            set, string, DatasetRemoveString, DatasetRemoveMd5, DatasetRemoveSha256);
+    return DatasetOpSerialized(set, string, DatasetRemoveString, DatasetRemoveMd5,
+            DatasetRemoveSha256, DatasetRemoveIPv4);
 }

src/datasets.h

@@ -32,6 +32,7 @@ enum DatasetTypes {
     DATASET_TYPE_STRING = 1,
     DATASET_TYPE_MD5,
     DATASET_TYPE_SHA256,
+    DATASET_TYPE_IPV4,
 };
 
 #define DATASET_NAME_MAX_LEN 63

src/detect-datarep.c

@@ -159,6 +159,8 @@ static int DetectDatarepParse(const char *str, char *cmd, int cmd_len, char *nam
                     *type = DATASET_TYPE_SHA256;
                 } else if (strcmp(val, "string") == 0) {
                     *type = DATASET_TYPE_STRING;
+                } else if (strcmp(val, "ipv4") == 0) {
+                    *type = DATASET_TYPE_IPV4;
                 } else {
                     SCLogDebug("bad type %s", val);
                     return -1;

src/detect-dataset.c

@@ -156,6 +156,8 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
                     *type = DATASET_TYPE_SHA256;
                 } else if (strcmp(val, "string") == 0) {
                     *type = DATASET_TYPE_STRING;
+                } else if (strcmp(val, "ipv4") == 0) {
+                    *type = DATASET_TYPE_IPV4;
                 } else {
                     SCLogError(SC_ERR_INVALID_SIGNATURE, "bad type %s", val);
                     return -1;