Update to the parsers.

src/app-layer-http.c

@@ -29,6 +29,10 @@ enum {
     HTTP_FIELD_MAX,
 };
 
+typedef struct HttpState_ {
+
+} HttpState;
+
 /** \brief Mapping between HTTP_FIELD_* and AppLayerParsers
  *
  * Map the http fields identifiers to the parsers.
@@ -53,16 +57,17 @@ int HTTPParseRequest(void *http_state, void *parser_state, u_int8_t *input, u_in
         http_state, parser_state, input, input_len);
 
     char done = FALSE;
+    u_int32_t offset = 0;
     AppLayerParserState *pstate = (AppLayerParserState *)parser_state;
 
     printf("HTTPParseRequest: pstate->buflen %u\n", pstate->buflen);
 
     u_int32_t u32 = 0;
-    for ( ; u32 < input_len; u32++) {
+    for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) {
         pstate->buf[pstate->buflen] = input[u32];
         pstate->buflen++;
 
-        if (pstate->buflen > 1 && pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') {
+        if (pstate->buflen >= 2 && pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') {
             printf("HTTPParseRequest: request line done.\n");
             done = TRUE;
             break;
@@ -70,11 +75,64 @@ int HTTPParseRequest(void *http_state, void *parser_state, u_int8_t *input, u_in
     }
 
     if (done == TRUE) {
+        AppLayerParserResultElement *e = AppLayerGetResultElmt();
+        if (e == NULL)
+            return -1;
+
+        e->name_idx = HTTP_FIELD_REQUEST_LINE;
+        e->data_ptr = input;
+        e->data_len = pstate->buflen;
+        output[*output_num] = e;
+
+        (*output_num)++;
+
         printf("HTTPParseRequest: request line:\n");
-        PrintRawDataFp(stdout, pstate->buf,pstate->buflen);
-        pstate->flags |= APP_LAYER_PARSER_DONE;
+        PrintRawDataFp(stdout, e->data_ptr,e->data_len);
+
+        offset += pstate->buflen;
+        pstate->buflen = 0;
+        done = FALSE;
+    } else {
+        /* bail with state update */
+        return 0;
+    }
+
+    printf("HTTPParseRequest: u32 %u, pstate->buflen %u\n", u32, pstate->buflen);
+    for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) {
+        pstate->buf[pstate->buflen] = input[u32];
+        pstate->buflen++;
+
+
+        if (pstate->buflen > 3 &&
+            pstate->buf[pstate->buflen - 4] == '\r' && pstate->buf[pstate->buflen - 3] == '\n' &&
+            pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') {
+            printf("HTTPParseRequest: request headers done @ u32 %u, pstate->buflen %u\n", u32, pstate->buflen);
+            done = TRUE;
+            break;
+        }
+    }
+
+    if (done == TRUE) {
+        AppLayerParserResultElement *e = AppLayerGetResultElmt();
+        if (e == NULL)
+            return -1;
+
+        e->name_idx = HTTP_FIELD_REQUEST_HEADERS;
+        e->data_ptr = input + offset;
+        e->data_len = pstate->buflen;
+        output[*output_num] = e;
+
+        (*output_num)++;
+
+        printf("HTTPParseRequest: request headers:\n");
+        PrintRawDataFp(stdout, e->data_ptr,e->data_len);
 
+        offset += pstate->buflen;
         pstate->buflen = 0;
+        done = FALSE;
+    } else {
+        /* bail with state update */
+        return 0;
     }
     return 1;
 }
@@ -88,6 +146,6 @@ void RegisterHTTPParsers(void) {
     AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOSERVER, HTTPParseRequest);
     AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOCLIENT, HTTPParseResponse);
 
-    AppLayerRegisterParser("http.request_line", HTTPParseRequestLine, "http");
+    AppLayerRegisterParser("http.request_line", ALPROTO_HTTP, HTTP_FIELD_REQUEST_LINE, HTTPParseRequestLine, "http");
 }
 

src/app-layer-parser.c

@@ -14,6 +14,24 @@
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
 
+static Pool *al_result_pool = NULL;
+
+void* AppLayerParserResultElementAlloc(void *null) {
+    AppLayerParserResultElement *e = (AppLayerParserResultElement *)malloc(sizeof(AppLayerParserResultElement));
+    if (e == NULL) {
+        return NULL;
+    }
+
+    memset(e, 0, sizeof(AppLayerParserResultElement));
+    return e;
+}
+#define AppLayerParserResultElementFree free
+
+AppLayerParserResultElement *AppLayerGetResultElmt(void) {
+    AppLayerParserResultElement *e = (AppLayerParserResultElement *)PoolGet(al_result_pool);
+    return e;
+}
+
 static u_int16_t app_layer_sid = 0;
 static AppLayerProto al_proto_table[ALPROTO_MAX];
 
@@ -39,12 +57,17 @@ u_int16_t AppLayerParserGetStorageId(void) {
  * \retval 0 on success
  * \retval -1 on error
  */
-int AppLayerRegisterParser(char *name, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num), char *dependency) {
+int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num), char *dependency) {
 
     al_max_parsers++;
 
     al_parser_table[al_max_parsers].name = name;
+    al_parser_table[al_max_parsers].proto = proto;
+    al_parser_table[al_max_parsers].parser_local_id = parser_id;
     al_parser_table[al_max_parsers].AppLayerParser = AppLayerParser;
+
+    printf("AppLayerRegisterParser: registered %p at proto %u, al_proto_table idx %u, storage_id %u, parser_local_id %u\n",
+        AppLayerParser, proto, al_max_parsers, al_proto_table[proto].storage_id, parser_id);
     return 0;
 }
 
@@ -117,12 +140,12 @@ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_in
             parser_idx = p->to_client;
         }
     } else {
-        printf("L7Parse: using parser %u we stored before\n", parser_state->cur_parser);
+        printf("AppLayerParse: using parser %u we stored before\n", parser_state->cur_parser);
         parser_idx = parser_state->cur_parser;
     }
 
     if (parser_idx == 0) {
-        printf("L7Parse: no parser for protocol %u\n", proto);
+        printf("AppLayerParse: no parser for protocol %u\n", proto);
         return 0;
     }
 
@@ -135,10 +158,31 @@ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_in
         }
     }
 
-    int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, NULL, NULL);
+    AppLayerParserResultElement *result_tbl[256];
+    memset(&result_tbl,0,sizeof(result_tbl));
+    u_int16_t output_num = 0;
+    int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, result_tbl, &output_num);
     if (r < 0)
         return -1;
 
+    printf("AppLayerParse: output_num %u\n", output_num);
+    u_int16_t u = 0;
+    for (u = 0; u < output_num; u++) {
+        AppLayerParserResultElement *e = result_tbl[u];
+        printf("AppLayerParse: e->name_idx %u, e->data_ptr %p, e->data_len %u, map_size %u\n", e->name_idx, e->data_ptr, e->data_len, al_proto_table[proto].map_size);
+
+        /* no parser defined for this field. */
+        if (e->name_idx >= al_proto_table[proto].map_size || al_proto_table[proto].map[e->name_idx] == NULL) {
+            printf("AppLayerParse: no parser for proto %u, parser_local_id %u\n", proto, e->name_idx);
+            continue;
+        }
+
+        parser_idx = al_proto_table[proto].map[e->name_idx]->parser_id;
+        int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, e->data_ptr, e->data_len, result_tbl, &output_num);
+        if (r < 0)
+            return -1;
+    }
+
     return 0;
 }
 
@@ -148,5 +192,81 @@ void RegisterAppLayerParsers(void) {
     memset(&al_parser_table, 0, sizeof(al_parser_table));
 
     app_layer_sid = StreamL7RegisterModule();
+
+    /** setup result pool
+     * \todo Per thread pool */
+    al_result_pool = PoolInit(100,10,AppLayerParserResultElementAlloc,NULL,AppLayerParserResultElementFree);
+}
+
+void AppLayerParsersInitPostProcess(void) {
+    printf("AppLayerParsersInitPostProcess: start\n");
+    u_int16_t u16 = 0;
+
+    /* build local->global mapping */
+    for (u16 = 1; u16 <= al_max_parsers; u16++) {
+        /* no local parser */
+        if (al_parser_table[u16].parser_local_id == 0)
+            continue;
+
+        if (al_parser_table[u16].parser_local_id > al_proto_table[al_parser_table[u16].proto].map_size)
+            al_proto_table[al_parser_table[u16].proto].map_size = al_parser_table[u16].parser_local_id;
+
+        printf("AppLayerParsersInitPostProcess: map_size %u\n", al_proto_table[al_parser_table[u16].proto].map_size);
+    }
+
+    /* for each proto, alloc the map array */
+    for (u16 = 0; u16 < ALPROTO_MAX; u16++) {
+        if (al_proto_table[u16].map_size == 0)
+            continue;
+
+        al_proto_table[u16].map_size++;
+        al_proto_table[u16].map = (AppLayerLocalMap **)malloc(al_proto_table[u16].map_size * sizeof(AppLayerLocalMap *));
+        if (al_proto_table[u16].map == NULL) {
+            printf("XXX memory error\n");
+            exit(1);
+        }
+        memset(al_proto_table[u16].map, 0, al_proto_table[u16].map_size * sizeof(AppLayerLocalMap *));
+
+        u_int16_t u = 0;
+        u_int16_t x = 0;
+        for (u = 1; u <= al_max_parsers; u++) {
+            /* no local parser */
+            if (al_parser_table[u].parser_local_id == 0)
+                continue;
+
+            if (al_parser_table[u].proto != u16)
+                continue;
+
+            printf("al_proto_table[%u].map_size %u, x %u, %p %p\n", u16, al_proto_table[u16].map_size, x, al_proto_table[u16].map[x], al_proto_table[u16].map);
+            u_int16_t parser_local_id = al_parser_table[u].parser_local_id;
+            printf("parser_local_id: %u\n", parser_local_id);
+
+            if (parser_local_id < al_proto_table[u16].map_size) {
+                al_proto_table[u16].map[parser_local_id] = malloc(sizeof(AppLayerLocalMap));
+                if (al_proto_table[u16].map[parser_local_id] == NULL) {
+                    printf("XXX memory error\n");
+                    exit(1);
+                }
+
+                al_proto_table[u16].map[parser_local_id]->parser_id = u;
+            }
+        }
+    }
+
+    for (u16 = 0; u16 < ALPROTO_MAX; u16++) {
+        if (al_proto_table[u16].map_size == 0)
+            continue;
+
+        if (al_proto_table[u16].map == NULL)
+            continue;
+
+        u_int16_t x = 0;
+        for (x = 0; x < al_proto_table[u16].map_size; x++) {
+            if (al_proto_table[u16].map[x] == NULL)
+                continue;
+
+            printf("al_proto_table[%u].map[%u]->parser_id: %u\n", u16, x, al_proto_table[u16].map[x]->parser_id);
+        }
+    }
 }
 

src/app-layer-parser.h

@@ -1,6 +1,12 @@
 #ifndef __APP_LAYER_PARSER_H__
 #define __APP_LAYER_PARSER_H__
 
+/** Mapping between local parser id's (e.g. HTTP_FIELD_REQUEST_URI) and
+  * the dynamically assigned (at registration) global parser id. */
+typedef struct AppLayerLocalMap_ {
+    u_int16_t parser_id;
+} AppLayerLocalMap;
+
 /** \brief Mapping between ALPROTO_* and L7Parsers
  *
  * Map the proto to the parsers for the to_client and to_server directions.
@@ -9,6 +15,9 @@ typedef struct AppLayerProto_ {
     u_int16_t to_server;
     u_int16_t to_client;
     u_int8_t storage_id;
+
+    AppLayerLocalMap **map;
+    u_int16_t map_size;
 } AppLayerProto;
 
 typedef struct AppLayerParserResultElement_ {
@@ -22,6 +31,8 @@ typedef struct AppLayerParserResultElement_ {
 
 typedef struct AppLayerParserTableElement_ {
     char *name;
+    u_int16_t proto;
+    u_int16_t parser_local_id; /** local id of the parser in the parser itself. */
     u_int8_t flags;
     int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num);
     u_int16_t max_outputs; /* rationele is that if we know the max outputs of all parsers, we
@@ -38,7 +49,7 @@ typedef struct AppLayerParserState_ {
 
     /** \todo this needs to become dynamic */
     u_int8_t buf[1024];
-    u_int8_t buflen;
+    u_int32_t buflen;
 } AppLayerParserState;
 
 #endif /* __APP_LAYER_PARSER_H__ */

src/eidps.c

@@ -902,6 +902,7 @@ int main(int argc, char **argv)
 
 RegisterAppLayerParsers();
 RegisterHTTPParsers();
+AppLayerParsersInitPostProcess();
 
     TmModuleReceiveNFQRegister();
     TmModuleVerdictNFQRegister();