diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index a9ad630fcf..474752586d 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -726,10 +726,15 @@ jobs: - run: suricata-update -V centos-stream9: - name: CentOS Stream 9 + name: CentOS Stream 9 (nDPI ${{ matrix.ndpi }}) runs-on: ubuntu-latest container: quay.io/centos/centos:stream9 needs: [prepare-deps, ubuntu-22-04-dist] + strategy: + fail-fast: false + matrix: + # nDPI releases supported by the plugin + ndpi: [ "4.14", "5.0", "6.0" ] steps: # Cache Rust stuff. - name: Cache cargo registry @@ -786,18 +791,18 @@ jobs: with: name: dist - - name: Build and install nDPI + - name: Build nDPI ${{ matrix.ndpi }} run: | - curl -OL https://github.com/ntop/nDPI/archive/refs/tags/4.14.tar.gz - tar xvf 4.14.tar.gz - cd nDPI-4.14 + curl -OL https://github.com/ntop/nDPI/archive/refs/tags/${{ matrix.ndpi }}.tar.gz + tar xvf ${{ matrix.ndpi }}.tar.gz + cd nDPI-${{ matrix.ndpi }} ./autogen.sh ./configure make -j ${{ env.CPUS }} - run: tar zxvf suricata-*.tar.gz --strip-components=1 - name: ./configure - run: CFLAGS="${DEFAULT_CFLAGS}" ./configure --enable-ndpi --with-ndpi=$(pwd)/nDPI-4.14 --enable-debug --enable-qa-simulation + run: CFLAGS="${DEFAULT_CFLAGS}" ./configure --enable-ndpi --with-ndpi=$(pwd)/nDPI-${{ matrix.ndpi }} --enable-debug --enable-qa-simulation - run: make -j ${{ env.CPUS }} - run: make install - run: make install-conf diff --git a/configure.ac b/configure.ac index 3e6d03c809..1819f9e23c 100644 --- a/configure.ac +++ b/configure.ac @@ -2271,7 +2271,7 @@ fi exit 1 ]) - AS_IF([test "x$enable_dpi" = "xyes"], [ + AS_IF([test "x$enable_ndpi" = "xyes"], [ if test "x$enable_shared" = "xno"; then echo echo " ERROR! ndpi cannot be enabled with --disable-shared" @@ -2292,6 +2292,18 @@ fi CPPFLAGS="${CPPFLAGS} -I$NDPI_HOME/src/include" NDPI_LIB="$NDPI_HOME/src/lib/libndpi.a" AC_SUBST([NDPI_LIB]) + + # see plugins/ndpi/ndpi-compat.h for the supported releases + AC_MSG_CHECKING([for nDPI 4.12 or later]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include "ndpi_define.h" + #if NDPI_MAJOR < 4 || (NDPI_MAJOR == 4 && NDPI_MINOR < 12) + #error "nDPI too old" + #endif + ]], [[]])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([the nDPI plugin requires nDPI 4.12 or later; make sure the nDPI source tree at $NDPI_HOME has been configured])]) else AC_MSG_RESULT(not found) enable_ndpi="no" diff --git a/doc/userguide/plugins/ndpi.rst b/doc/userguide/plugins/ndpi.rst index a07f2f5707..7d7094013d 100644 --- a/doc/userguide/plugins/ndpi.rst +++ b/doc/userguide/plugins/ndpi.rst @@ -6,6 +6,9 @@ nDPI Installation ************ +The plugin supports nDPI 4.12, 5.0 and 6.0. The nDPI source tree has to +be configured and built before building Suricata against it. + Before using nDPI, Suricata must be built with nDPI support, for example: @@ -26,6 +29,40 @@ building Suricata with nDPI support. For more information on nDPI, see https://www.ntop.org/products/deep-packet-inspection/ndpi/. +Configuration +************* + +Starting with nDPI 6.0, the library has to be told under which license it +is used, as some of its dissectors are dual-licensed by ntop. The plugin +exposes this through the optional ``ndpi`` section of ``suricata.yaml``: + +.. code-block:: yaml + + ndpi: + license: not-for-profit + +Accepted values: + +``not-for-profit`` + Not-for-profit use. Every dissector is enabled. This is the default when + the option is absent. + +``for-profit`` + For-profit use without a license agreement with ntop. nDPI does not load + its dual-licensed dissectors, DHCP, DNS, QUIC and TLS as of nDPI 6.0, so + rules matching on those protocols never fire. The plugin logs a warning + at startup when this value is selected. + +``for-profit-dual`` + For-profit use with a license agreement signed with ntop covering the + dual-licensed components. Every dissector is enabled. + +An invalid value is a fatal error. With nDPI releases older than 6.0 the +option has no effect and a warning is logged when it is set. + +See https://www.ntop.org/announcing-ndpi-dual-license-change/ for the +licensing terms themselves. + Keywords ******** diff --git a/plugins/ndpi/Makefile.am b/plugins/ndpi/Makefile.am index 4d5dcbe4ed..b28c48323c 100644 --- a/plugins/ndpi/Makefile.am +++ b/plugins/ndpi/Makefile.am @@ -7,7 +7,7 @@ ndpi_la_LIBADD = @NDPI_LIB@ # source directory. ndpi_la_CFLAGS = -I../../rust/gen -I../../rust/dist -ndpi_la_SOURCES = ndpi.c +ndpi_la_SOURCES = ndpi.c ndpi-compat.h install-exec-hook: cd $(DESTDIR)$(pkglibdir) && $(RM) $(pkglib_LTLIBRARIES) diff --git a/plugins/ndpi/ndpi-compat.h b/plugins/ndpi/ndpi-compat.h new file mode 100644 index 0000000000..37eb5a6801 --- /dev/null +++ b/plugins/ndpi/ndpi-compat.h @@ -0,0 +1,125 @@ +/* Copyright (C) 2025 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * Compatibility layer over the parts of the nDPI API that differ between + * nDPI 4.12, 5.0 and 6.0, selected at compile time from NDPI_MAJOR so + * that ndpi.c stays version agnostic. + */ + +#ifndef SURICATA_NDPI_COMPAT_H +#define SURICATA_NDPI_COMPAT_H + +#include +#include + +#include "ndpi_api.h" + +#if NDPI_MAJOR < 4 || (NDPI_MAJOR == 4 && NDPI_MINOR < 12) +#error "the nDPI plugin requires nDPI 4.12 or later" +#endif + +/* nDPI 6.0 requires the embedding application to declare under which + * license it uses the library; earlier releases have no notion of it. */ +#if NDPI_MAJOR >= 6 +#define NDPI_COMPAT_HAS_LICENSE 1 +typedef enum ndpi_license_type NdpiCompatLicense; +#define NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT NDPI_LICENSE_NOT_FOR_PROFIT_LGPL +#define NDPI_COMPAT_LICENSE_FOR_PROFIT NDPI_LICENSE_FOR_PROFIT_LGPL +#define NDPI_COMPAT_LICENSE_FOR_PROFIT_DUAL NDPI_LICENSE_FOR_PROFIT_DUAL_LICENSE +#else +#define NDPI_COMPAT_HAS_LICENSE 0 +/* placeholder so the license plumbing builds, the value is ignored */ +typedef int NdpiCompatLicense; +#define NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT 0 +#endif + +/* Allocate a detection module. The license is ignored on nDPI releases + * predating the declaration. */ +static inline struct ndpi_detection_module_struct *NdpiCompatInitModule( + struct ndpi_global_context *g_ctx, NdpiCompatLicense license) +{ +#if NDPI_COMPAT_HAS_LICENSE + return ndpi_init_detection_module(g_ctx, license); +#else + (void)license; + return ndpi_init_detection_module(g_ctx); +#endif +} + +/* nDPI 4.x needs every protocol enabled explicitly before finalization, + * later releases enable them by default and removed the bitmask. */ +static inline void NdpiCompatEnableAllProtocols(struct ndpi_detection_module_struct *ndpi) +{ +#if NDPI_MAJOR < 5 + NDPI_PROTOCOL_BITMASK all; + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi, &all); +#else + (void)ndpi; +#endif +} + +/* Whether nDPI reached a final classification for the flow. Once it has, + * the plugin stops feeding it packets and the keywords start matching. + * + * On nDPI 5.0 and later NDPI_STATE_CLASSIFIED with no extra dissection + * pending is final, as in nDPI's ndpiReader. NDPI_STATE_MONITORING is + * treated as final too: the classification will not change and nDPI + * would only extract more metadata, so stopping there bounds the per + * packet cost at the expense of that metadata. */ +static inline bool NdpiCompatClassificationFinal(struct ndpi_detection_module_struct *ndpi, + struct ndpi_flow_struct *flow, const ndpi_protocol *proto) +{ +#if NDPI_MAJOR >= 5 + (void)ndpi; + if (proto->state == NDPI_STATE_MONITORING) + return true; + return proto->state == NDPI_STATE_CLASSIFIED && flow->extra_packets_func == NULL; +#else + return ndpi_is_protocol_detected(*proto) != 0 && !ndpi_is_proto_unknown(proto->proto) && + !ndpi_extra_dissection_possible(ndpi, flow); +#endif +} + +/* Whether nDPI has no classification for the flow at all, in which case + * giving up on it goes through ndpi_detection_giveup() for a guess. */ +static inline bool NdpiCompatIsUnclassified(const ndpi_protocol *proto) +{ +#if NDPI_MAJOR >= 5 + return proto->state != NDPI_STATE_CLASSIFIED && proto->state != NDPI_STATE_MONITORING; +#else + return ndpi_is_protocol_detected(*proto) == 0; +#endif +} + +/* nDPI 5.0 dropped the output argument telling whether the protocol was + * guessed, the flow now carries it in protocol_was_guessed. */ +static inline ndpi_protocol NdpiCompatGiveup( + struct ndpi_detection_module_struct *ndpi, struct ndpi_flow_struct *flow) +{ +#if NDPI_MAJOR >= 5 + return ndpi_detection_giveup(ndpi, flow); +#else + uint8_t protocol_was_guessed; + return ndpi_detection_giveup(ndpi, flow, &protocol_was_guessed); +#endif +} + +#endif /* SURICATA_NDPI_COMPAT_H */ diff --git a/plugins/ndpi/ndpi.c b/plugins/ndpi/ndpi.c index c8e5caed1c..6a0f65775d 100644 --- a/plugins/ndpi/ndpi.c +++ b/plugins/ndpi/ndpi.c @@ -22,6 +22,7 @@ #include "suricata-common.h" #include "suricata-plugin.h" +#include "conf.h" #include "detect-engine-helper.h" #include "detect-parse.h" #include "flow.h" @@ -32,12 +33,22 @@ #include "thread-storage.h" #include "util-debug.h" -#include "ndpi_api.h" +#include "ndpi-compat.h" static SCThreadStorageId thread_storage_id = { .id = -1 }; static SCFlowStorageId flow_storage_id = { .id = -1 }; static int ndpi_protocol_keyword_id = -1; static int ndpi_risk_keyword_id = -1; +static struct ndpi_global_context *ndpi_g_ctx = NULL; +static NdpiCompatLicense ndpi_license = NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT; +/* EVE objects dropped because nDPI produced malformed JSON */ +static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(unsigned int, ndpi_json_dropped, 0); + +/* Initial size of the buffer ndpi_dpi2json() writes to. nDPI grows it by + * the unescaped length of a string only, while escaping can expand a byte + * to six; starting with a buffer large enough for the common case keeps + * its escaper from running out of room. */ +#define NDPI_EVE_SERIALIZER_SIZE 4096 struct NdpiThreadContext { struct ndpi_detection_module_struct *ndpi; @@ -81,6 +92,93 @@ static inline struct NdpiFlowContext *NdpiGetFlowContext(const Flow *f) return SCFlowGetStorageById(f, flow_storage_id); } +/** + * Resolve "ndpi.license". nDPI 6.0 does not load its dual-licensed + * dissectors (DHCP, DNS, QUIC and TLS as of 6.0) for for-profit use + * without an agreement with ntop. Not-for-profit is the default so the + * plugin works out of the box; an invalid value is fatal. + */ +static NdpiCompatLicense NdpiResolveLicense(void) +{ + const char *license = NULL; + + if (SCConfGet("ndpi.license", &license) != 1 || license == NULL) + return NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT; + +#if NDPI_COMPAT_HAS_LICENSE + if (strcmp(license, "not-for-profit") == 0) + return NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT; + + if (strcmp(license, "for-profit") == 0) { + SCLogWarning("ndpi.license is \"for-profit\": nDPI will not load its dual-licensed " + "dissectors (DHCP, DNS, QUIC and TLS as of nDPI 6.0)"); + return NDPI_COMPAT_LICENSE_FOR_PROFIT; + } + + if (strcmp(license, "for-profit-dual") == 0) + return NDPI_COMPAT_LICENSE_FOR_PROFIT_DUAL; + + FatalError("invalid ndpi.license value \"%s\": expected \"not-for-profit\", " + "\"for-profit\" or \"for-profit-dual\"", + license); +#else + SCLogWarning("ndpi.license has no effect with nDPI %d.%d, the license declaration was " + "introduced in nDPI 6.0", + NDPI_MAJOR, NDPI_MINOR); +#endif + return NDPI_COMPAT_LICENSE_NOT_FOR_PROFIT; +} + +/* nDPI correlates flows through LRU caches (DNS to TLS, STUN to RTP, ...) + * that are private to a detection module unless made global through a + * shared context. Suricata spreads a host's flows over all its workers, + * so without sharing a correlation never leaves the worker that saw the + * first flow. */ +static const char *ndpi_shared_lru_caches[] = { + "lru.ookla.scope", + "lru.bittorrent.scope", + "lru.stun.scope", + "lru.tls_cert.scope", + "lru.mining.scope", + "lru.msteams.scope", + "lru.fpc_dns.scope", + "lru.signal.scope", +}; + +/** + * Allocate and finalize a detection module with every protocol enabled. + * Worker modules share their LRU caches through the global context, the + * throwaway modules used while parsing rules don't need to. + * + * nDPI creates a shared cache on the first finalization without locking, + * which is fine as thread init callbacks run sequentially from + * TmThreadCreate(). + */ +static struct ndpi_detection_module_struct *NdpiModuleNew(bool shared_caches) +{ + struct ndpi_global_context *g_ctx = shared_caches ? ndpi_g_ctx : NULL; + struct ndpi_detection_module_struct *ndpi = NdpiCompatInitModule(g_ctx, ndpi_license); + if (ndpi == NULL) + return NULL; + + NdpiCompatEnableAllProtocols(ndpi); + + if (g_ctx != NULL) { + for (size_t i = 0; i < ARRAY_SIZE(ndpi_shared_lru_caches); i++) { + if (ndpi_set_config(ndpi, NULL, ndpi_shared_lru_caches[i], "1") != NDPI_CFG_OK) { + SCLogWarning("Failed to share the nDPI \"%s\" cache between threads", + ndpi_shared_lru_caches[i]); + } + } + } + + if (ndpi_finalize_initialization(ndpi) != 0) { + ndpi_exit_detection_module(ndpi); + return NULL; + } + return ndpi; +} + static void ThreadStorageFree(void *ptr) { SCLogDebug("Free'ing nDPI thread storage"); @@ -160,26 +258,35 @@ static void OnFlowUpdate(ThreadVars *tv, Flow *f, Packet *p, void *_data) if (!flowctx->detection_completed && ip_ptr != NULL && ip_len > 0) { uint64_t time_ms = ((uint64_t)p->ts.secs) * 1000 + p->ts.usecs / 1000; + struct ndpi_flow_input_info input_info; SCLogDebug("Performing nDPI detection..."); + /* Suricata knows the packet direction, spare nDPI from guessing it. + * Whether the flow beginning was seen is left unknown as the flow + * API exposes no reliable flag for it. */ + memset(&input_info, 0, sizeof(input_info)); + input_info.seen_flow_beginning = NDPI_FLOW_BEGINNING_UNKNOWN; + input_info.in_pkt_dir = + PKT_IS_TOSERVER(p) ? NDPI_IN_PKT_DIR_C_TO_S : NDPI_IN_PKT_DIR_S_TO_C; + flowctx->detected_l7_protocol = ndpi_detection_process_packet( - threadctx->ndpi, flowctx->ndpi_flow, ip_ptr, ip_len, time_ms, NULL); + threadctx->ndpi, flowctx->ndpi_flow, ip_ptr, ip_len, time_ms, &input_info); - if (ndpi_is_protocol_detected(flowctx->detected_l7_protocol) != 0) { - if (!ndpi_is_proto_unknown(flowctx->detected_l7_protocol.proto)) { - if (!ndpi_extra_dissection_possible(threadctx->ndpi, flowctx->ndpi_flow)) - flowctx->detection_completed = true; - } + if (NdpiCompatClassificationFinal( + threadctx->ndpi, flowctx->ndpi_flow, &flowctx->detected_l7_protocol)) { + flowctx->detection_completed = true; } else { - uint16_t max_num_pkts = (flow_proto == IPPROTO_UDP) ? 8 : 24; + /* stop feeding nDPI after a few packets, taking its best guess + * for flows it could not classify */ + const uint16_t max_num_pkts = (flow_proto == IPPROTO_UDP) ? 8 : 24; if ((SCFlowGetToServerPacketCount(f) + SCFlowGetToClientPacketCount(f)) > max_num_pkts) { - uint8_t proto_guessed; - - flowctx->detected_l7_protocol = - ndpi_detection_giveup(threadctx->ndpi, flowctx->ndpi_flow, &proto_guessed); + if (NdpiCompatIsUnclassified(&flowctx->detected_l7_protocol)) { + flowctx->detected_l7_protocol = + NdpiCompatGiveup(threadctx->ndpi, flowctx->ndpi_flow); + } flowctx->detection_completed = true; } } @@ -209,14 +316,10 @@ static void OnThreadInit(ThreadVars *tv, void *_data) if (context == NULL) { FatalError("Failed to allocate nDPI thread context"); } - context->ndpi = ndpi_init_detection_module(NULL); + context->ndpi = NdpiModuleNew(true); if (context->ndpi == NULL) { FatalError("Failed to initialize nDPI detection module"); } - NDPI_PROTOCOL_BITMASK protos; - NDPI_BITMASK_SET_ALL(protos); - ndpi_set_protocol_detection_bitmask2(context->ndpi, &protos); - ndpi_finalize_initialization(context->ndpi); SCThreadSetStorageById(tv, thread_storage_id, context); } @@ -272,17 +375,12 @@ static DetectnDPIProtocolData *DetectnDPIProtocolParse(const char *arg, bool neg struct ndpi_detection_module_struct *ndpi_struct; ndpi_master_app_protocol l7_protocol; char *l7_protocol_name = (char *)arg; - NDPI_PROTOCOL_BITMASK all; /* convert protocol name (string) to ID */ - ndpi_struct = ndpi_init_detection_module(NULL); + ndpi_struct = NdpiModuleNew(false); if (unlikely(ndpi_struct == NULL)) return NULL; - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); - ndpi_finalize_initialization(ndpi_struct); - l7_protocol = ndpi_get_protocol_by_name(ndpi_struct, l7_protocol_name); ndpi_exit_detection_module(ndpi_struct); @@ -399,20 +497,9 @@ static int DetectnDPIRiskPacketMatch( static DetectnDPIRiskData *DetectnDPIRiskParse(const char *arg, bool negate) { DetectnDPIRiskData *data; - struct ndpi_detection_module_struct *ndpi_struct; ndpi_risk risk_mask; - NDPI_PROTOCOL_BITMASK all; /* convert list of risk names (string) to mask */ - ndpi_struct = ndpi_init_detection_module(NULL); - if (unlikely(ndpi_struct == NULL)) - return NULL; - - NDPI_BITMASK_SET_ALL(all); - ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); - ndpi_finalize_initialization(ndpi_struct); - ndpi_exit_detection_module(ndpi_struct); - if (isdigit(arg[0])) risk_mask = atoll(arg); else { @@ -495,6 +582,159 @@ static void DetectnDPIRiskFree(DetectEngineCtx *de_ctx, void *ptr) SCFree(ptr); } +/* Syntax check of the JSON nDPI produces for a flow. It is inserted into + * the EVE record as is, so a malformed fragment would take the whole + * record down with it, and nDPI has been seen producing one (6.0 corrupts + * its output when escaping strings with non printable characters). */ + +typedef struct NdpiJsonCursor_ { + const char *p; + const char *end; +} NdpiJsonCursor; + +static bool NdpiJsonValue(NdpiJsonCursor *c, int depth); + +static void NdpiJsonSkipSpace(NdpiJsonCursor *c) +{ + while (c->p < c->end && (*c->p == ' ' || *c->p == '\t' || *c->p == '\n' || *c->p == '\r')) + c->p++; +} + +static bool NdpiJsonString(NdpiJsonCursor *c) +{ + if (c->p >= c->end || *c->p != '"') + return false; + c->p++; + + while (c->p < c->end) { + const unsigned char ch = (unsigned char)*c->p++; + if (ch == '"') + return true; + if (ch < 0x20) + return false; + if (ch != '\\') + continue; + + if (c->p >= c->end) + return false; + switch (*c->p++) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + break; + case 'u': + for (int i = 0; i < 4; i++, c->p++) { + if (c->p >= c->end || !isxdigit((unsigned char)*c->p)) + return false; + } + break; + default: + return false; + } + } + return false; +} + +static bool NdpiJsonLiteral(NdpiJsonCursor *c, const char *lit) +{ + const size_t len = strlen(lit); + if ((size_t)(c->end - c->p) < len || memcmp(c->p, lit, len) != 0) + return false; + c->p += len; + return true; +} + +static bool NdpiJsonNumber(NdpiJsonCursor *c) +{ + bool digits = false; + while (c->p < c->end && (isdigit((unsigned char)*c->p) || *c->p == '-' || *c->p == '+' || + *c->p == '.' || *c->p == 'e' || *c->p == 'E')) { + digits |= isdigit((unsigned char)*c->p) != 0; + c->p++; + } + return digits; +} + +/** + * Members of an object or elements of an array, up to the closing + * bracket, or up to the end of the input for a bare object body. + */ +static bool NdpiJsonMembers(NdpiJsonCursor *c, int depth, char closing) +{ + NdpiJsonSkipSpace(c); + if (closing != '\0' && c->p < c->end && *c->p == closing) { + c->p++; + return true; + } + + for (;;) { + if (closing != ']') { + if (!NdpiJsonString(c)) + return false; + NdpiJsonSkipSpace(c); + if (c->p >= c->end || *c->p++ != ':') + return false; + } + if (!NdpiJsonValue(c, depth)) + return false; + + NdpiJsonSkipSpace(c); + if (c->p >= c->end) + return closing == '\0'; + if (closing != '\0' && *c->p == closing) { + c->p++; + return true; + } + if (*c->p++ != ',') + return false; + NdpiJsonSkipSpace(c); + } +} + +static bool NdpiJsonValue(NdpiJsonCursor *c, int depth) +{ + if (depth > 32) + return false; + + NdpiJsonSkipSpace(c); + if (c->p >= c->end) + return false; + + switch (*c->p) { + case '{': + c->p++; + return NdpiJsonMembers(c, depth + 1, '}'); + case '[': + c->p++; + return NdpiJsonMembers(c, depth + 1, ']'); + case '"': + return NdpiJsonString(c); + case 't': + return NdpiJsonLiteral(c, "true"); + case 'f': + return NdpiJsonLiteral(c, "false"); + case 'n': + return NdpiJsonLiteral(c, "null"); + default: + return NdpiJsonNumber(c); + } +} + +/** + * \brief Whether buf holds well formed object members, i.e. the body of + * a JSON object without its braces, as SCJbSetFormatted() expects. + */ +static bool NdpiJsonFragmentValid(const char *buf, uint32_t len) +{ + NdpiJsonCursor c = { .p = buf, .end = buf + len }; + return NdpiJsonMembers(&c, 1, '\0'); +} + static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, SCJsonBuilder *jb, void *data) { /* Adding ndpi info to EVE requires a flow. */ @@ -518,7 +758,8 @@ static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, SCJsonBuilder SCLogDebug("EveCallback: tv=%p, p=%p, f=%p", tv, p, f); - if (ndpi_init_serializer(&serializer, ndpi_serialization_format_inner_json) != 0) { + if (ndpi_init_serializer_ll(&serializer, ndpi_serialization_format_inner_json, + NDPI_EVE_SERIALIZER_SIZE) != 0) { SCLogDebug("Failed to initialize nDPI serializer"); return; } @@ -529,8 +770,25 @@ static void EveCallback(ThreadVars *tv, const Packet *p, Flow *f, SCJsonBuilder buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len); if (buffer != NULL && buffer_len > 0) { - /* Inject the nDPI JSON to the JsonBuilder */ - SCJbSetFormatted(jb, buffer); + bool valid = NdpiJsonFragmentValid(buffer, buffer_len); + if (valid) { + /* SCJbSetFormatted() takes a C string and nDPI only terminates + * its buffer when there is room left for it */ + char stack_json[2048]; + char *json = buffer_len < sizeof(stack_json) ? stack_json : SCMalloc(buffer_len + 1); + if (json != NULL) { + memcpy(json, buffer, buffer_len); + json[buffer_len] = '\0'; + /* also rejects invalid UTF-8, without inserting anything */ + valid = SCJbSetFormatted(jb, json); + if (json != stack_json) + SCFree(json); + } + } + if (!valid && SC_ATOMIC_ADD(ndpi_json_dropped, 1) == 0) { + SCLogWarning("nDPI produced malformed JSON for a flow, dropping its EVE object " + "(only reported once)"); + } } ndpi_term_serializer(&serializer); @@ -572,6 +830,17 @@ static void NdpiInit(void) { SCLogDebug("Initializing nDPI plugin"); + ndpi_license = NdpiResolveLicense(); + + /* Global context shared by the worker modules, see NdpiModuleNew(). + * NULL if nDPI was built without global context support. There is + * no plugin deinit hook, so it is never released. */ + ndpi_g_ctx = ndpi_global_init(); + if (ndpi_g_ctx == NULL) { + SCLogWarning("Failed to initialize the nDPI global context: " + "per-thread nDPI caches will not be shared"); + } + /* Register thread storage. */ thread_storage_id = SCThreadStorageRegister("ndpi", ThreadStorageFree); if (thread_storage_id.id < 0) { diff --git a/suricata.yaml.in b/suricata.yaml.in index 49c1df696c..5655d528f9 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -85,6 +85,23 @@ plugins: @ndpi_comment@- @prefix@/lib/@PACKAGE_NAME@/ndpi.so # - /path/to/plugin.so +# nDPI plugin settings, only used when the nDPI plugin is loaded above. +ndpi: + # nDPI 6.0 and later require the application to declare under which + # license the library is used. Has no effect with older nDPI releases. + # + # not-for-profit : not-for-profit use, every dissector is enabled. + # This is the default. + # for-profit : for-profit use without a license agreement with + # ntop. nDPI does not load its dual-licensed + # dissectors (DHCP, DNS, QUIC and TLS as of nDPI 6.0), + # so rules matching on those protocols never fire. + # for-profit-dual : for-profit use with a license agreement signed + # with ntop for the dual-licensed components. + # + # See https://www.ntop.org/announcing-ndpi-dual-license-change/ + #license: not-for-profit + # Configure the type of alert (and other) logging you would like. outputs: # a line based alerts log similar to Snort's fast.log