QA: direct access from commandline to AppLayer API

This patch introduces a new set of commandline options meant for
assisting in fuzz testing the app layer implementations.

Per protocol, 2 commandline options are added:

--afl-http-request=<filename>
--afl-http=<filename>

In the former case, the contents of the file are passed directly to
the HTTP parser as request data.

In the latter case, the data is devided between request and responses.
First 64 bytes are request, then next 64 are response, next 64 are
request, etc, etc.

configure.ac

@@ -267,6 +267,7 @@
             AC_DEFINE([AFLFUZZ_DISABLE_MGTTHREADS], [1], [Disable all management threads])
             AC_DEFINE([AFLFUZZ_PCAP_RUNMODE], [1], [Enable special AFL 'single' runmode])
             AC_DEFINE([AFLFUZZ_CONF_TEST], [1], [Enable special --afl-parse-rules commandline option])
+            AC_DEFINE([AFLFUZZ_APPLAYER], [1], [Enable --afl-$proto-request commandline option])
     ])
 
   # disable TLS on user request

src/app-layer-htp.c

@@ -2162,7 +2162,11 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
     cfg_prec->request_inspect_window = HTP_CONFIG_DEFAULT_REQUEST_INSPECT_WINDOW;
     cfg_prec->response_inspect_min_size = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_MIN_SIZE;
     cfg_prec->response_inspect_window = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_WINDOW;
+#ifndef AFLFUZZ_NO_RANDOM
     cfg_prec->randomize = HTP_CONFIG_DEFAULT_RANDOMIZE;
+#else
+    cfg_prec->randomize = 0;
+#endif
     cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE;
 
     htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
@@ -2441,7 +2445,9 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                     (size_t)HTP_CONFIG_DEFAULT_FIELD_LIMIT_SOFT,
                     (size_t)limit);
         } else if (strcasecmp("randomize-inspection-sizes", p->name) == 0) {
+#ifndef AFLFUZZ_NO_RANDOM
             cfg_prec->randomize = ConfValIsTrue(p->val);
+#endif
         } else if (strcasecmp("randomize-inspection-range", p->name) == 0) {
             uint32_t range = atoi(p->val);
             if (range > 100) {

src/app-layer-modbus.c

@@ -1433,7 +1433,9 @@ void RegisterModbusParsers(void)
                                                 proto_name, ALPROTO_MODBUS,
                                                 0, sizeof(ModbusHeader),
                                                 ModbusProbingParser)) {
+#ifndef AFLFUZZ_APPLAYER
                 return;
+#endif
             }
         }
 
@@ -1448,11 +1450,16 @@ void RegisterModbusParsers(void)
         }
         SCLogInfo("Modbus request flood protection level: %u", request_flood);
     } else {
+#ifndef AFLFUZZ_APPLAYER
         SCLogInfo("Protocol detection and parser disabled for %s protocol.", proto_name);
         return;
+#endif
     }
-
+#ifndef AFLFUZZ_APPLAYER
     if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
+#else
+    if (1) {
+#endif
         AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_MODBUS, STREAM_TOSERVER, ModbusParseRequest);
         AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_MODBUS, STREAM_TOCLIENT, ModbusParseResponse);
         AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_MODBUS, ModbusStateAlloc, ModbusStateFree);

src/app-layer-parser.c

@@ -1189,6 +1189,141 @@ void AppLayerParserStatePrintDetails(AppLayerParserState *pstate)
 }
 #endif
 
+#ifdef AFLFUZZ_APPLAYER
+int AppLayerParserRequestFromFile(AppProto alproto, char *filename)
+{
+    int result = 1;
+    Flow *f = NULL;
+    TcpSession ssn;
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+
+    memset(&ssn, 0, sizeof(ssn));
+
+    f = SCCalloc(1, sizeof(Flow));
+    if (f == NULL)
+        goto end;
+    FLOW_INITIALIZE(f);
+
+    f->flags |= FLOW_IPV4;
+    f->src.addr_data32[0] = 0x01020304;
+    f->dst.addr_data32[0] = 0x05060708;
+    f->sp = 10000;
+    f->dp = 80;
+    f->protoctx = &ssn;
+    f->proto = IPPROTO_TCP;
+    f->alproto = alproto;
+
+    FILE *fp = fopen(filename, "r");
+    BUG_ON(fp == NULL);
+
+    uint8_t buffer[256];
+
+    int start = 1;
+    while (1) {
+        int done = 0;
+        size_t result = fread(&buffer, 1, sizeof(buffer), fp);
+        if (result < sizeof(buffer))
+            done = 1;
+
+        //SCLogInfo("result %u done %d start %d", (uint)result, done, start);
+
+        uint8_t flags = STREAM_TOSERVER;
+
+        if (start--) {
+            flags |= STREAM_START;
+        }
+        if (done) {
+            flags |= STREAM_EOF;
+        }
+        //PrintRawDataFp(stdout, buffer, result);
+
+        (void)AppLayerParserParse(alp_tctx, f, alproto, flags, buffer, result);
+        if (done)
+            break;
+    }
+
+    result = 0;
+    fclose(fp);
+end:
+    if (alp_tctx != NULL)
+        AppLayerParserThreadCtxFree(alp_tctx);
+    if (f != NULL) {
+        FlowFree(f);
+    }
+    return result;
+}
+
+int AppLayerParserFromFile(AppProto alproto, char *filename)
+{
+    int result = 1;
+    Flow *f = NULL;
+    TcpSession ssn;
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+
+    memset(&ssn, 0, sizeof(ssn));
+
+    f = SCCalloc(1, sizeof(Flow));
+    if (f == NULL)
+        goto end;
+    FLOW_INITIALIZE(f);
+
+    f->flags |= FLOW_IPV4;
+    f->src.addr_data32[0] = 0x01020304;
+    f->dst.addr_data32[0] = 0x05060708;
+    f->sp = 10000;
+    f->dp = 80;
+    f->protoctx = &ssn;
+    f->proto = IPPROTO_TCP;
+    f->alproto = alproto;
+
+    FILE *fp = fopen(filename, "r");
+    BUG_ON(fp == NULL);
+
+    uint8_t buffer[64];
+
+    int start = 1;
+    int flip = 0;
+    while (1) {
+        int done = 0;
+        size_t result = fread(&buffer, 1, sizeof(buffer), fp);
+        if (result < sizeof(buffer))
+            done = 1;
+
+        //SCLogInfo("result %u done %d start %d", (uint)result, done, start);
+
+        uint8_t flags = 0;
+        if (flip) {
+            flags = STREAM_TOCLIENT;
+            flip = 0;
+        } else {
+            flags = STREAM_TOSERVER;
+            flip = 1;
+        }
+
+        if (start--) {
+            flags |= STREAM_START;
+        }
+        if (done) {
+            flags |= STREAM_EOF;
+        }
+        //PrintRawDataFp(stdout, buffer, result);
+
+        (void)AppLayerParserParse(alp_tctx, f, alproto, flags, buffer, result);
+        if (done)
+            break;
+    }
+
+    result = 0;
+    fclose(fp);
+end:
+    if (alp_tctx != NULL)
+        AppLayerParserThreadCtxFree(alp_tctx);
+    if (f != NULL) {
+        FlowFree(f);
+    }
+    return result;
+}
+#endif
 
 /***** Unittests *****/
 

src/app-layer-parser.h

@@ -222,6 +222,11 @@ void AppLayerParserStateFree(AppLayerParserState *pstate);
 void AppLayerParserStatePrintDetails(AppLayerParserState *pstate);
 #endif
 
+#ifdef AFLFUZZ_APPLAYER
+int AppLayerParserRequestFromFile(AppProto alproto, char *filename);
+int AppLayerParserFromFile(AppProto alproto, char *filename);
+#endif
+
 /***** Unittests *****/
 
 #ifdef UNITTESTS

src/suricata.c

@@ -145,6 +145,13 @@
 #include "app-layer.h"
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
+#include "app-layer-ssl.h"
+#include "app-layer-dns-tcp.h"
+#include "app-layer-ssh.h"
+#include "app-layer-ftp.h"
+#include "app-layer-smtp.h"
+#include "app-layer-smb.h"
+#include "app-layer-modbus.h"
 
 #include "util-radix-tree.h"
 #include "util-host-os-info.h"
@@ -1136,6 +1143,22 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri)
         {"netmap", optional_argument, 0, 0},
         {"pcap", optional_argument, 0, 0},
         {"simulate-ips", 0, 0 , 0},
+        {"afl-http-request", required_argument, 0 , 0},
+        {"afl-http", required_argument, 0 , 0},
+        {"afl-tls-request", required_argument, 0 , 0},
+        {"afl-tls", required_argument, 0 , 0},
+        {"afl-dns-request", required_argument, 0 , 0},
+        {"afl-dns", required_argument, 0 , 0},
+        {"afl-ssh-request", required_argument, 0 , 0},
+        {"afl-ssh", required_argument, 0 , 0},
+        {"afl-ftp-request", required_argument, 0 , 0},
+        {"afl-ftp", required_argument, 0 , 0},
+        {"afl-smtp-request", required_argument, 0 , 0},
+        {"afl-smtp", required_argument, 0 , 0},
+        {"afl-smb-request", required_argument, 0 , 0},
+        {"afl-smb", required_argument, 0 , 0},
+        {"afl-modbus-request", required_argument, 0 , 0},
+        {"afl-modbus", required_argument, 0 , 0},
 #ifdef BUILD_UNIX_SOCKET
         {"unix-socket", optional_argument, 0, 0},
 #endif
@@ -1324,6 +1347,92 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri)
                     usage(argv[0]);
                     return TM_ECODE_FAILED;
                 }
+#ifdef AFLFUZZ_APPLAYER
+            } else if(strcmp((long_opts[option_index]).name, "afl-http-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterHTPParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_HTTP, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-http") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterHTPParsers();
+                exit(AppLayerParserFromFile(ALPROTO_HTTP, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-tls-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                RegisterSSLParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_TLS, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-tls") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterSSLParsers();
+                exit(AppLayerParserFromFile(ALPROTO_TLS, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-dns-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                RegisterDNSTCPParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_DNS, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-dns") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterDNSTCPParsers();
+                exit(AppLayerParserFromFile(ALPROTO_DNS, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-ssh-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                RegisterSSHParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_SSH, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-ssh") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterSSHParsers();
+                exit(AppLayerParserFromFile(ALPROTO_SSH, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-ftp-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                RegisterFTPParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_FTP, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-ftp") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterFTPParsers();
+                exit(AppLayerParserFromFile(ALPROTO_FTP, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-smtp-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                MpmTableSetup();
+                AppLayerParserSetup();
+                RegisterSMTPParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_SMTP, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-smtp") == 0) {
+                //printf("arg: //%s\n", optarg);
+                MpmTableSetup();
+                AppLayerParserSetup();
+                RegisterSMTPParsers();
+                exit(AppLayerParserFromFile(ALPROTO_SMTP, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-smb-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                RegisterSMBParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_SMB, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-smb") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterSMBParsers();
+                exit(AppLayerParserFromFile(ALPROTO_SMB, optarg));
+
+            } else if(strcmp((long_opts[option_index]).name, "afl-modbus-request") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterModbusParsers();
+                exit(AppLayerParserRequestFromFile(ALPROTO_MODBUS, optarg));
+            } else if(strcmp((long_opts[option_index]).name, "afl-modbus") == 0) {
+                //printf("arg: //%s\n", optarg);
+                AppLayerParserSetup();
+                RegisterModbusParsers();
+                exit(AppLayerParserFromFile(ALPROTO_MODBUS, optarg));
+#endif
             } else if(strcmp((long_opts[option_index]).name, "simulate-ips") == 0) {
                 SCLogInfo("Setting IPS mode");
                 EngineModeSetIPS();