From 9f78d47c2aaec42aa63d6314c8c96457dea1b251 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 15 Aug 2009 10:07:34 +0200 Subject: [PATCH] Further work on the stream L7 parser, it's api and the http stub implementation. --- src/app-layer-http.c | 948 +++++++++++++++++++++++++++++++++++++---- src/app-layer-http.h | 2 + src/app-layer-parser.c | 436 ++++++++++++++++--- src/app-layer-parser.h | 59 ++- src/eidps.c | 10 +- 5 files changed, 1303 insertions(+), 152 deletions(-) diff --git a/src/app-layer-http.c b/src/app-layer-http.c index b0a635e25c..e2f3a79274 100644 --- a/src/app-layer-http.c +++ b/src/app-layer-http.c @@ -14,6 +14,18 @@ #include "app-layer-protos.h" #include "app-layer-parser.h" +#include "util-binsearch.h" +#include "util-unittest.h" + +typedef enum { + HTTP_METHOD_UNKNOWN = 0, + HTTP_METHOD_GET, + HTTP_METHOD_POST, + /** \todo more.. */ +} HttpRequestMethod; + +typedef u_int16_t HttpResponseCode; + enum { HTTP_FIELD_NONE = 0, @@ -25,121 +37,347 @@ enum { HTTP_FIELD_REQUEST_URI, HTTP_FIELD_REQUEST_VERSION, + HTTP_FIELD_RESPONSE_LINE, + HTTP_FIELD_RESPONSE_HEADERS, + HTTP_FIELD_RESPONSE_BODY, + + HTTP_FIELD_RESPONSE_VERSION, + HTTP_FIELD_RESPONSE_CODE, + HTTP_FIELD_RESPONSE_MSG, + /* must be last */ HTTP_FIELD_MAX, }; typedef struct HttpState_ { + HttpRequestMethod method; + HttpResponseCode response_code; } HttpState; -/** \brief Mapping between HTTP_FIELD_* and AppLayerParsers - * - * Map the http fields identifiers to the parsers. - */ -typedef struct HTTPParser_ { - u_int16_t parser_idx; -} HTTPParser; +static int HTTPParseRequestMethod(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + HttpState *hstate = (HttpState *)http_state; -static HTTPParser http_field_table[HTTP_FIELD_MAX]; + if (input_len == 4 && memcmp(input, "POST", 4) == 0) { + //printf("HTTPParseRequestMethod: POST\n"); + hstate->method = HTTP_METHOD_POST; + } else if (input_len == 3 && memcmp(input, "GET", 3) == 0) { + //printf("HTTPParseRequestMethod: GET\n"); + hstate->method = HTTP_METHOD_GET; + } + return 1; +} -int HTTPParseRequestLine(void *http_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num) { - printf("HTTPParseRequestLine: http_state %p, parser_state %p, input %p, input_len %u\n", - http_state, parser_state, input, input_len); - PrintRawDataFp(stdout, input,input_len); +static int HTTPParseResponseCode(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + HttpState *hstate = (HttpState *)http_state; - return 0; -} + if (input_len > 3) + return 1; -int HTTPParseRequest(void *http_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num) { - printf("HTTPParseRequest: http_state %p, parser_state %p, input %p, input_len %u\n", - http_state, parser_state, input, input_len); + char code[4] = { 0x0, 0x0, 0x0, 0x0, }; + u_int32_t u = 0; + for ( ; u < input_len; u++) { + code[u] = input[u]; + } - char done = FALSE; - u_int32_t offset = 0; - AppLayerParserState *pstate = (AppLayerParserState *)parser_state; + unsigned long ul = strtoul(code, (char **)NULL, 10); + if (ul >= 1000) { /** \todo what is the max HTTP code */ + return 1; + } - printf("HTTPParseRequest: pstate->buflen %u\n", pstate->buflen); + hstate->response_code = (HttpResponseCode)ul; + return 1; +} - u_int32_t u32 = 0; - for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) { - pstate->buf[pstate->buflen] = input[u32]; - pstate->buflen++; +static int HTTPParseRequestLine(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + //printf("HTTPParseRequestLine: http_state %p, pstate %p, input %p, input_len %u\n", + // http_state, pstate, input, input_len); + //PrintRawDataFp(stdout, input,input_len); - if (pstate->buflen >= 2 && pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') { - printf("HTTPParseRequest: request line done.\n"); - done = TRUE; - break; + u_int16_t max_fields = 3; + u_int16_t u = 0; + u_int32_t offset = 0; + + if (pstate == NULL) + return -1; + + for (u = pstate->parse_field; u < max_fields; u++) { + //printf("HTTPParseRequestLine: u %u\n", u); + + switch(u) { + case 0: /* REQUEST METHOD */ + { + //printf("HTTPParseRequestLine: request method\n"); + + const u_int8_t delim[] = { 0x20, }; + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_REQUEST_METHOD, delim, sizeof(delim), input, input_len, &offset); + //printf("HTTPParseRequestLine: r = %d\n", r); + + if (r == 0) { + pstate->parse_field = 0; + return 0; + } + break; + } + case 1: /* REQUEST URI */ + { + const u_int8_t delim[] = { 0x20, }; + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_REQUEST_URI, delim, sizeof(delim), data, data_len, &offset); + if (r == 0) { + pstate->parse_field = 1; + return 0; + } + break; + } + case 2: /* REQUEST VERSION */ + { + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + //printf("HTTPParseRequestLine: request version\n"); + //PrintRawDataFp(stdout, data, data_len); + + int r = AlpParseFieldByEOF(output, pstate, HTTP_FIELD_REQUEST_VERSION, data, data_len); + if (r == 0) { + pstate->parse_field = 2; + return 0; + } + + break; + } } } - if (done == TRUE) { - AppLayerParserResultElement *e = AppLayerGetResultElmt(); - if (e == NULL) - return -1; - - e->name_idx = HTTP_FIELD_REQUEST_LINE; - e->data_ptr = input; - e->data_len = pstate->buflen; - output[*output_num] = e; + pstate->parse_field = 0; + return 1; +} - (*output_num)++; +static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + //printf("HTTPParseRequest: http_state %p, state %p, input %p, input_len %u\n", + // http_state, pstate, input, input_len); + //PrintRawDataFp(stdout, input,input_len); - printf("HTTPParseRequest: request line:\n"); - PrintRawDataFp(stdout, e->data_ptr,e->data_len); + u_int16_t max_fields = 3; + u_int16_t u = 0; + u_int32_t offset = 0; - offset += pstate->buflen; - pstate->buflen = 0; - done = FALSE; - } else { - /* bail with state update */ - return 0; + if (pstate == NULL) + return -1; + + //printf("HTTPParseRequest: pstate->parse_field %u\n", pstate->parse_field); + + for (u = pstate->parse_field; u < max_fields; u++) { + switch(u) { + case 0: /* REQUEST LINE */ + { + //printf("HTTPParseRequest: request line (1)\n"); + //PrintRawDataFp(stdout, pstate->store, pstate->store_len); + + const u_int8_t delim[] = { 0x0D, 0x0A }; + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_REQUEST_LINE, delim, sizeof(delim), input, input_len, &offset); + if (r == 0) { + pstate->parse_field = 0; + //printf("HTTPParseRequest: request line (4)\n"); + return 0; + } + //printf("HTTPParseRequest: request line (2)\n"); + //if (pstate->store_len) PrintRawDataFp(stdout, pstate->store, pstate->store_len); + //printf("HTTPParseRequest: request line (3)\n"); + break; + } + case 1: /* HEADERS */ + { + //printf("HTTPParseRequest: request headers (offset %u, pstate->store_len %u)\n", offset, pstate->store_len); + //if (pstate->store_len) PrintRawDataFp(stdout, pstate->store, pstate->store_len); + + const u_int8_t delim[] = { 0x0D, 0x0A, 0x0D, 0x0A }; + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_REQUEST_HEADERS, delim, sizeof(delim), data, data_len, &offset); + if (r == 0) { + pstate->parse_field = 1; + return 0; + } + break; + } + case 2: + { + //printf("HTTPParseRequest: request body\n"); + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByEOF(output, pstate, HTTP_FIELD_REQUEST_BODY, data, data_len); + if (r == 0) { + pstate->parse_field = 2; + return 0; + } + + break; + } + } } - printf("HTTPParseRequest: u32 %u, pstate->buflen %u\n", u32, pstate->buflen); - for ( ; u32 < input_len && pstate->buflen < sizeof(pstate->buf); u32++) { - pstate->buf[pstate->buflen] = input[u32]; - pstate->buflen++; + pstate->parse_field = 0; + return 1; +} + +static int HTTPParseResponseLine(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + //printf("HTTPParseResponseLine: http_state %p, pstate %p, input %p, input_len %u\n", + // http_state, pstate, input, input_len); + //PrintRawDataFp(stdout, input,input_len); + u_int16_t max_fields = 3; + u_int16_t u = 0; + u_int32_t offset = 0; - if (pstate->buflen >= 4 && - pstate->buf[pstate->buflen - 4] == '\r' && pstate->buf[pstate->buflen - 3] == '\n' && - pstate->buf[pstate->buflen - 2] == '\r' && pstate->buf[pstate->buflen - 1] == '\n') { - printf("HTTPParseRequest: request headers done @ u32 %u, pstate->buflen %u\n", u32, pstate->buflen); - done = TRUE; - break; + if (pstate == NULL) + return -1; + + for (u = pstate->parse_field; u < max_fields; u++) { + //printf("HTTPParseResponseLine: u %u\n", u); + + switch(u) { + case 0: /* RESPONSE VERSION */ + { + //printf("HTTPParseResponseLine: request method\n"); + + const u_int8_t delim[] = { 0x20, }; + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_RESPONSE_VERSION, delim, sizeof(delim), input, input_len, &offset); + //printf("HTTPParseResponseLine: r = %d\n", r); + + if (r == 0) { + pstate->parse_field = 0; + return 0; + } + break; + } + case 1: /* RESPONSE CODE */ + { + const u_int8_t delim[] = { 0x20, }; + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_RESPONSE_CODE, delim, sizeof(delim), data, data_len, &offset); + if (r == 0) { + pstate->parse_field = 1; + return 0; + } + break; + } + case 2: /* RESPONSE MSG */ + { + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + //printf("HTTPParseResponseLine: request version\n"); + //PrintRawDataFp(stdout, data, data_len); + + int r = AlpParseFieldByEOF(output, pstate, HTTP_FIELD_RESPONSE_MSG, data, data_len); + if (r == 0) { + pstate->parse_field = 2; + return 0; + } + + break; + } } } - if (done == TRUE) { - AppLayerParserResultElement *e = AppLayerGetResultElmt(); - if (e == NULL) - return -1; - - e->name_idx = HTTP_FIELD_REQUEST_HEADERS; - e->data_ptr = input + offset; - e->data_len = pstate->buflen; - output[*output_num] = e; + pstate->parse_field = 0; + return 1; +} - (*output_num)++; +static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output) { + //printf("HTTPParseResponse: http_state %p, pstate %p, input %p, input_len %u\n", + // http_state, pstate, input, input_len); - printf("HTTPParseRequest: request headers:\n"); - PrintRawDataFp(stdout, e->data_ptr,e->data_len); + u_int16_t max_fields = 3; + u_int16_t u = 0; + u_int32_t offset = 0; - offset += pstate->buflen; - pstate->buflen = 0; - done = FALSE; - } else { - /* bail with state update */ - return 0; + if (pstate == NULL) + return -1; + + //printf("HTTPParseReponse: pstate->parse_field %u\n", pstate->parse_field); + + for (u = pstate->parse_field; u < max_fields; u++) { + switch(u) { + case 0: /* RESPONSE LINE */ + { + //printf("HTTPParseResponse: response line (1)\n"); + //PrintRawDataFp(stdout, pstate->store, pstate->store_len); + + const u_int8_t delim[] = { 0x0D, 0x0A }; + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_RESPONSE_LINE, delim, sizeof(delim), input, input_len, &offset); + if (r == 0) { + pstate->parse_field = 0; + //printf("HTTPParseResponse: response line (4)\n"); + return 0; + } + //printf("HTTPParseResponse: response line (2)\n"); + //if (pstate->store_len) PrintRawDataFp(stdout, pstate->store, pstate->store_len); + //printf("HTTPParseResponse: response line (3)\n"); + break; + } + case 1: /* HEADERS */ + { + //printf("HTTPParseResponse: response headers (offset %u, pstate->store_len %u)\n", offset, pstate->store_len); + //if (pstate->store_len) PrintRawDataFp(stdout, pstate->store, pstate->store_len); + + const u_int8_t delim[] = { 0x0D, 0x0A, 0x0D, 0x0A }; + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByDelimiter(output, pstate, HTTP_FIELD_RESPONSE_HEADERS, delim, sizeof(delim), data, data_len, &offset); + if (r == 0) { + pstate->parse_field = 1; + return 0; + } + break; + } + case 2: + { + //printf("HTTPParseResponse: response body\n"); + + u_int8_t *data = input + offset; + u_int32_t data_len = input_len - offset; + + int r = AlpParseFieldByEOF(output, pstate, HTTP_FIELD_RESPONSE_BODY, data, data_len); + if (r == 0) { + pstate->parse_field = 2; + return 0; + } + + break; + } + } } + + pstate->parse_field = 0; return 1; } -int HTTPParseResponse(void *http_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num) { - printf("HTTPParseResponse: http_state %p, parser_state %p, input %p, input_len %u\n", - http_state, parser_state, input, input_len); +static void *HTTPStateAlloc(void) { + void *s = malloc(sizeof(HttpState)); + if (s == NULL) + return NULL; + + memset(s, 0, sizeof(HttpState)); + return s; +} + +static void HTTPStateFree(void *s) { + free(s); } void RegisterHTTPParsers(void) { @@ -147,5 +385,559 @@ void RegisterHTTPParsers(void) { AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOCLIENT, HTTPParseResponse); AppLayerRegisterParser("http.request_line", ALPROTO_HTTP, HTTP_FIELD_REQUEST_LINE, HTTPParseRequestLine, "http"); + AppLayerRegisterParser("http.request.method", ALPROTO_HTTP, HTTP_FIELD_REQUEST_METHOD, HTTPParseRequestMethod, "http.request_line"); + + AppLayerRegisterParser("http.response_line", ALPROTO_HTTP, HTTP_FIELD_RESPONSE_LINE, HTTPParseResponseLine, "http"); + AppLayerRegisterParser("http.response.code", ALPROTO_HTTP, HTTP_FIELD_RESPONSE_CODE, HTTPParseResponseCode, "http.response_line"); + + AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTTPStateAlloc, HTTPStateFree); +} + +/* UNITTESTS */ +#ifdef UNITTESTS + +/** \test Send a get request in one chunk. */ +int HTTPParserTest01(void) { + int result = 1; + Flow f; + u_int8_t httpbuf[] = "GET / HTTP/1.1\r\nUser-Agent: Victor/1.0\r\n\r\n"; + u_int32_t httplen = sizeof(httpbuf) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_GET) { + printf("expected method %u, got %u: ", HTTP_METHOD_GET, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test Send a post request in one chunk. */ +int HTTPParserTest02(void) { + int result = 1; + Flow f; + u_int8_t httpbuf[] = "POST / HTTP/1.1\r\nUser-Agent: Victor/1.0\r\n\r\nPost Data Is c0oL!"; + u_int32_t httplen = sizeof(httpbuf) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_POST) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test Send a get request. */ +int HTTPParserTest03(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "GET / HTTP"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "/1.1\r\n"; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + u_int8_t httpbuf3[] = "User-Agent: Victor/1.0\r\n\r\n"; + u_int32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_GET) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test Send a get request in 3 chunks, splitting up the request line. */ +int HTTPParserTest04(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "GET / HTTP"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "/1."; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + u_int8_t httpbuf3[] = "1\r\nUser-Agent: Victor/1.0\r\n\r\n"; + u_int32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_GET) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + +end: + return result; } +/** \test Send a post request with data, splitting up after the headers. */ +int HTTPParserTest05(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "POST / HTTP/1.1\r\nUser-Agent: Victor/1.0\r\n\r\n"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "Post D"; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + u_int8_t httpbuf3[] = "ata is c0oL!"; + u_int32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_POST) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test See how it deals with an incomplete request. */ +int HTTPParserTest06(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "POST"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_UNKNOWN) { + printf("expected method %u, got %u: ", HTTP_METHOD_UNKNOWN, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test See how it deals with an incomplete request in multiple chunks. */ +int HTTPParserTest07(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "PO"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "ST"; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_UNKNOWN) { + printf("expected method %u, got %u: ", HTTP_METHOD_UNKNOWN, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +/** \test Test both sides of a http stream mixed up to see if the parser + * properly keeps them separated. */ +int HTTPParserTest08(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "POST / HTTP/1.1\r\nUser-Agent: Victor/1.0\r\n\r\n"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "Post D"; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + u_int8_t httpbuf3[] = "ata is c0oL!"; + u_int32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + u_int8_t httpbuf4[] = "HTTP/1.1 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; + u_int32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + u_int8_t httpbuf5[] = "post R"; + u_int32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ + u_int8_t httpbuf6[] = "esults are tha bomb!"; + u_int32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */ + TcpSession ssn; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + + f.stream = (void *)&ssn; + + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOCLIENT|STREAM_START, httpbuf4, httplen4); + if (r != 0) { + printf("toserver chunk 4 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOCLIENT, httpbuf5, httplen5); + if (r != 0) { + printf("toserver chunk 5 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOCLIENT|STREAM_EOF, httpbuf6, httplen6); + if (r != 0) { + printf("toserver chunk 6 returned %d, expected 0: ", r); + result = 0; + goto end; + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_POST) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + + if (http_state->response_code != 200) { + printf("expected code %u, got %u: ", http_state->response_code, 200); + result = 0; + goto end; + } +end: + return result; +} + +/** \test Feed the parser our HTTP streams one byte at a time and see how + * it copes. */ +int HTTPParserTest09(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "POST / HTTP/1.1\r\nUser-Agent: Victor/1.0\r\n\r\nPost Data is c0oL!"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + u_int8_t httpbuf2[] = "HTTP/1.1 200 OK\r\nServer: VictorServer/1.0\r\n\r\npost Results are tha bomb!"; + u_int32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + TcpSession ssn; + int r = 0; + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + u_int32_t u; + for (u = 0; u < httplen1; u++) { + u_int8_t flags = 0; + + if (u == 0) flags = STREAM_TOSERVER|STREAM_START; + else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF; + else flags = STREAM_TOSERVER; + + r = AppLayerParse(&f, ALPROTO_HTTP, flags, &httpbuf1[u], 1); + if (r != 0) { + printf("toserver chunk %u returned %d, expected 0: ", u, r); + result = 0; + goto end; + } + } + + for (u = 0; u < httplen2; u++) { + u_int8_t flags = 0; + + if (u == 0) flags = STREAM_TOCLIENT|STREAM_START; + else if (u == (httplen2 - 1)) flags = STREAM_TOCLIENT|STREAM_EOF; + else flags = STREAM_TOCLIENT; + + r = AppLayerParse(&f, ALPROTO_HTTP, flags, &httpbuf2[u], 1); + if (r != 0) { + printf("toclient chunk %u returned %d, expected 0: ", u, r); + result = 0; + goto end; + } + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_POST) { + printf("expected method %u, got %u: ", HTTP_METHOD_POST, http_state->method); + result = 0; + goto end; + } + + if (http_state->response_code != 200) { + printf("expected code %u, got %u: ", http_state->response_code, 200); + result = 0; + goto end; + } +end: + return result; +} + +/** \test Test case where chunks are smaller than the delim length and the + * last chunk is supposed to match the delim. */ +int HTTPParserTest10(void) { + int result = 1; + Flow f; + u_int8_t httpbuf1[] = "GET / HTTP/1.0\r\n"; + u_int32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + TcpSession ssn; + int r = 0; + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); + f.stream = (void *)&ssn; + + u_int32_t u; + for (u = 0; u < httplen1; u++) { + u_int8_t flags = 0; + + if (u == 0) flags = STREAM_TOSERVER|STREAM_START; + else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF; + else flags = STREAM_TOSERVER; + + r = AppLayerParse(&f, ALPROTO_HTTP, flags, &httpbuf1[u], 1); + if (r != 0) { + printf("toserver chunk %u returned %d, expected 0: ", u, r); + result = 0; + goto end; + } + } + + HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + if (http_state->method != HTTP_METHOD_GET) { + printf("expected method %u, got %u: ", HTTP_METHOD_GET, http_state->method); + result = 0; + goto end; + } + +end: + return result; +} + +void HTTPParserRegisterTests(void) { + UtRegisterTest("HTTPParserTest01", HTTPParserTest01, 1); + UtRegisterTest("HTTPParserTest02", HTTPParserTest02, 1); + UtRegisterTest("HTTPParserTest03", HTTPParserTest03, 1); + UtRegisterTest("HTTPParserTest04", HTTPParserTest04, 1); + UtRegisterTest("HTTPParserTest05", HTTPParserTest05, 1); + UtRegisterTest("HTTPParserTest06", HTTPParserTest06, 1); + UtRegisterTest("HTTPParserTest07", HTTPParserTest07, 1); + UtRegisterTest("HTTPParserTest08", HTTPParserTest08, 1); + UtRegisterTest("HTTPParserTest09", HTTPParserTest09, 1); + UtRegisterTest("HTTPParserTest10", HTTPParserTest10, 1); +} + +#endif /* UNITTESTS */ + diff --git a/src/app-layer-http.h b/src/app-layer-http.h index 94912d784f..fd4cd22ed9 100644 --- a/src/app-layer-http.h +++ b/src/app-layer-http.h @@ -1,6 +1,8 @@ #ifndef __APP_LAYER_HTTP_H__ #define __APP_LAYER_HTTP_H__ +void RegisterHTTPParsers(void); +void HTTPParserRegisterTests(void); #endif /* __APP_LAYER_HTTP_H__ */ diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 7a8760d8d0..2c63efc3b6 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -14,24 +14,263 @@ #include "app-layer-protos.h" #include "app-layer-parser.h" +#include "util-binsearch.h" + static Pool *al_result_pool = NULL; -void* AppLayerParserResultElementAlloc(void *null) { - AppLayerParserResultElement *e = (AppLayerParserResultElement *)malloc(sizeof(AppLayerParserResultElement)); +/** \brief Alloc a AppLayerParserResultElmt func for the pool */ +static void *AlpResultElmtPoolAlloc(void *null) { + AppLayerParserResultElmt *e = (AppLayerParserResultElmt *)malloc(sizeof(AppLayerParserResultElmt)); if (e == NULL) { return NULL; } - memset(e, 0, sizeof(AppLayerParserResultElement)); + memset(e, 0, sizeof(AppLayerParserResultElmt)); return e; } -#define AppLayerParserResultElementFree free -AppLayerParserResultElement *AppLayerGetResultElmt(void) { - AppLayerParserResultElement *e = (AppLayerParserResultElement *)PoolGet(al_result_pool); +static void AlpResultElmtPoolFree(void *e) { + AppLayerParserResultElmt *re = (AppLayerParserResultElmt *)e; + + if (re->flags & ALP_RESULT_ELMT_ALLOC) { + if (re->data_ptr != NULL) + free(re->data_ptr); + } + free(re); +} + +static AppLayerParserResultElmt *AlpGetResultElmt(void) { + AppLayerParserResultElmt *e = (AppLayerParserResultElmt *)PoolGet(al_result_pool); + e->next = NULL; return e; } +static void AlpReturnResultElmt(AppLayerParserResultElmt *e) { + if (e->flags & ALP_RESULT_ELMT_ALLOC) { + if (e->data_ptr != NULL) + free(e->data_ptr); + } + e->flags = 0; + e->data_ptr = NULL; + e->data_len = 0; + e->next = NULL; + + PoolReturn(al_result_pool, (void *)e); +} + +static void AlpAppendResultElmt(AppLayerParserResult *r, AppLayerParserResultElmt *e) { + if (r->head == NULL) { + r->head = e; + r->tail = e; + r->cnt = 1; + } else { + r->tail->next = e; + r->tail = e; + r->cnt++; + } +} + +/** + * \param alloc Is ptr alloc'd (1) or a ptr to static mem (0). + */ +static void AlpStoreField(AppLayerParserResult *output, u_int16_t idx, u_int8_t *ptr, u_int32_t len, u_int8_t alloc) { + AppLayerParserResultElmt *e = AlpGetResultElmt(); + if (e == NULL) + return; + + if (alloc == 1) + e->flags |= ALP_RESULT_ELMT_ALLOC; + + e->name_idx = idx; + e->data_ptr = ptr; + e->data_len = len; + AlpAppendResultElmt(output, e); + + //printf("FIELD registered %u:\n", idx); + //PrintRawDataFp(stdout, e->data_ptr,e->data_len); +} + +/** \brief Parse a field up to the EOF + * + * \retval 1 Field found and stored. + * \retval 0 Field parsing in progress. + * \retval -1 error + */ +int AlpParseFieldByEOF(AppLayerParserResult *output, AppLayerParserState *pstate, u_int16_t field_idx, u_int8_t *input, u_int32_t input_len) { + if (pstate->store_len == 0) { + if (pstate->flags & APP_LAYER_PARSER_EOF) { + //printf("ParseFieldByEOF: store_len 0 and EOF\n"); + AlpStoreField(output, field_idx, input, input_len, 0); + return 1; + } else { + //printf("ParseFieldByEOF: store_len 0 but no EOF\n"); + /* delimiter field not found, so store the result for the next run */ + pstate->store = malloc(input_len); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store, input, input_len); + pstate->store_len = input_len; + } + } else { + if (pstate->flags & APP_LAYER_PARSER_EOF) { + //printf("ParseFieldByEOF: store_len %u and EOF\n", pstate->store_len); + pstate->store = realloc(pstate->store, (input_len + pstate->store_len)); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store+pstate->store_len, input, input_len); + pstate->store_len += input_len; + + AlpStoreField(output, field_idx, pstate->store, pstate->store_len, 1); + pstate->store = NULL; + pstate->store_len = 0; + return 1; + } else { + //printf("ParseFieldByEOF: store_len %u but no EOF\n", pstate->store_len); + /* delimiter field not found, so store the result for the next run */ + pstate->store = realloc(pstate->store, (input_len + pstate->store_len)); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store+pstate->store_len, input, input_len); + pstate->store_len += input_len; + } + + } + + return 0; +} + +/** \brief Parse a field up to a delimeter. + * + * \retval 1 Field found and stored. + * \retval 0 Field parsing in progress. + * \retval -1 error + */ +int AlpParseFieldByDelimiter(AppLayerParserResult *output, AppLayerParserState *pstate, u_int16_t field_idx, const u_int8_t *delim, u_int8_t delim_len, u_int8_t *input, u_int32_t input_len, u_int32_t *offset) { +// printf("ParseFieldByDelimiter: pstate->store_len %u, delim_len %u\n", pstate->store_len, delim_len); + + if (pstate->store_len == 0) { + u_int8_t *ptr = BinSearch(input, input_len, delim, delim_len); + if (ptr != NULL) { + u_int32_t len = ptr - input; + //printf("ParseFieldByDelimiter: len %u\n", len); + + AlpStoreField(output, field_idx, input, len, 0); + (*offset) += (len + delim_len); + return 1; + } else { + if (pstate->flags & APP_LAYER_PARSER_EOF) { + //printf("ParseFieldByDelimiter: delim not found and EOF\n"); + return 0; + } + + //printf("ParseFieldByDelimiter: delim not found, continue\n"); + + /* delimiter field not found, so store the result for the next run */ + pstate->store = malloc(input_len); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store, input, input_len); + pstate->store_len = input_len; + } + } else { + u_int8_t *ptr = BinSearch(input, input_len, delim, delim_len); + if (ptr != NULL) { + u_int32_t len = ptr - input; + //printf("ParseFieldByDelimiter: len %u + %u = %u\n", len, pstate->store_len, len + pstate->store_len); + + pstate->store = realloc(pstate->store, (len + pstate->store_len)); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store+pstate->store_len, input, len); + pstate->store_len += len; + + AlpStoreField(output, field_idx, pstate->store, pstate->store_len, 1); + pstate->store = NULL; + pstate->store_len = 0; + + (*offset) += (len + delim_len); + return 1; + } else { + if (pstate->flags & APP_LAYER_PARSER_EOF) { + /* if the input len is smaller than the delim len we search the + * pstate->store since we may match there. */ + if (delim_len > input_len) { + /* delimiter field not found, so store the result for the next run */ + pstate->store = realloc(pstate->store, (input_len + pstate->store_len)); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store+pstate->store_len, input, input_len); + pstate->store_len += input_len; + //printf("ParseFieldByDelimiter: input_len < delim_len, checking pstate->store\n"); + + if (pstate->store_len >= delim_len) { + ptr = BinSearch(pstate->store, pstate->store_len, delim, delim_len); + if (ptr != NULL) { + //printf("ParseFieldByDelimiter: now we found the delim\n"); + + u_int32_t len = ptr - pstate->store; + AlpStoreField(output, field_idx, pstate->store, len, 1); + pstate->store = NULL; + pstate->store_len = 0; + + (*offset) += (input_len); + + //printf("ParseFieldByDelimiter: offset %u\n", (*offset)); + return 1; + } + goto free_and_return; + } + goto free_and_return; + } + free_and_return: + //printf("ParseFieldByDelimiter: not found and EOF, so free what we have so far.\n"); + free(pstate->store); + pstate->store = NULL; + pstate->store_len = 0; + return 0; + } + + /* delimiter field not found, so store the result for the next run */ + pstate->store = realloc(pstate->store, (input_len + pstate->store_len)); + if (pstate->store == NULL) + return -1; + + memcpy(pstate->store+pstate->store_len, input, input_len); + pstate->store_len += input_len; + + /* if the input len is smaller than the delim len we search the + * pstate->store since we may match there. */ + if (delim_len > input_len && delim_len <= pstate->store_len) { + //printf("ParseFieldByDelimiter: input_len < delim_len, checking pstate->store\n"); + + ptr = BinSearch(pstate->store, pstate->store_len, delim, delim_len); + if (ptr != NULL) { + //printf("ParseFieldByDelimiter: now we found the delim\n"); + + u_int32_t len = ptr - pstate->store; + AlpStoreField(output, field_idx, pstate->store, len, 1); + pstate->store = NULL; + pstate->store_len = 0; + + (*offset) += (input_len); + + //printf("ParseFieldByDelimiter: offset %u\n", (*offset)); + return 1; + } + } + } + + } + + return 0; +} + static u_int16_t app_layer_sid = 0; static AppLayerProto al_proto_table[ALPROTO_MAX]; @@ -57,7 +296,7 @@ u_int16_t AppLayerParserGetStorageId(void) { * \retval 0 on success * \retval -1 on error */ -int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num), char *dependency) { +int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int (*AppLayerParser)(void *protocol_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output), char *dependency) { al_max_parsers++; @@ -76,12 +315,11 @@ int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int * \param name full parser name, e.g. "http.request_line" * \todo do we need recursive, so a "http" and a "request_line" where the engine knows it's actually "http.request_line"... same difference maybe. * \param AppLayerParser pointer to the parser function - * \param max_outputs max number of unique outputs the parser can generate * * \retval 0 on success * \retval -1 on error */ -int AppLayerRegisterProto(char *name, u_int8_t proto, u_int8_t flags, int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num)) { +int AppLayerRegisterProto(char *name, u_int8_t proto, u_int8_t flags, int (*AppLayerParser)(void *protocol_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output)) { al_max_parsers++; @@ -104,44 +342,143 @@ int AppLayerRegisterProto(char *name, u_int8_t proto, u_int8_t flags, int (*AppL return 0; } -AppLayerParserState* AppLayerParserStateAlloc(void) { - AppLayerParserState *s = (AppLayerParserState *)malloc(sizeof(AppLayerParserState)); +void AppLayerRegisterStateFuncs(u_int16_t proto, void *(*StateAlloc)(void), void (*StateFree)(void *)) { + al_proto_table[proto].StateAlloc = StateAlloc; + al_proto_table[proto].StateFree = StateFree; +} + +u_int16_t AlpGetStateIdx(u_int16_t proto) { + return al_proto_table[proto].storage_id; +} + +AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) { + AppLayerParserStateStore *s = (AppLayerParserStateStore *)malloc(sizeof(AppLayerParserStateStore)); if (s == NULL) return NULL; - memset(s, 0, sizeof(AppLayerParserState)); + memset(s, 0, sizeof(AppLayerParserStateStore)); return s; } +static void AppLayerParserResultCleanup(AppLayerParserResult *result) { + AppLayerParserResultElmt *e = result->head; + while (e != NULL) { + AppLayerParserResultElmt *next_e = e->next; + + result->head = next_e; + if (next_e == NULL) + result->tail = NULL; + result->cnt--; + + AlpReturnResultElmt(e); + e = next_e; + } +} + +static int AppLayerDoParse(void *app_layer_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, u_int16_t parser_idx, u_int16_t proto) { + int retval = 0; + AppLayerParserResult result = { NULL, NULL, 0 }; + + //printf("AppLayerDoParse: parser_idx %u\n", parser_idx); + //PrintRawDataFp(stdout, input,input_len); + + /* invoke the parser */ + int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, &result); + if (r < 0) + return -1; + + /* process the result elements */ + AppLayerParserResultElmt *e = result.head; + for (; e != NULL; e = e->next) { + //printf("AppLayerParse: e %p e->name_idx %u, e->data_ptr %p, e->data_len %u, map_size %u\n", + // e, e->name_idx, e->data_ptr, e->data_len, al_proto_table[proto].map_size); + + /* no parser defined for this field. */ + if (e->name_idx >= al_proto_table[proto].map_size || al_proto_table[proto].map[e->name_idx] == NULL) { + //printf("AppLayerParse: no parser for proto %u, parser_local_id %u\n", proto, e->name_idx); + continue; + } + + u_int16_t idx = al_proto_table[proto].map[e->name_idx]->parser_id; + + /* prepare */ + u_int16_t tmp = parser_state->parse_field; + parser_state->parse_field = 0; + parser_state->flags |= APP_LAYER_PARSER_EOF; + + r = AppLayerDoParse(app_layer_state, parser_state, e->data_ptr, e->data_len, idx, proto); + + /* restore */ + parser_state->flags &= ~APP_LAYER_PARSER_EOF; + parser_state->parse_field = tmp; + + /* bail out on a serious error */ + if (r < 0) { + retval = -1; + break; + } + } + + AppLayerParserResultCleanup(&result); + return retval; +} + /** * \brief Layer 7 Parsing main entry point. * + * \param f Properly initialized and locked flow. + * \param proto L7 proto, e.g. ALPROTO_HTTP + * \param flags Stream flags + * \param input Input L7 data + * \param input_len Length of the input data. + * + * \retval -1 error + * \retval 0 ok */ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_int32_t input_len) { - printf("AppLayerParse: proto %u, flags %02X\n", proto, flags); + //printf("AppLayerParse: proto %u, flags %02X\n", proto, flags); + //PrintRawDataFp(stdout, input,input_len); u_int16_t parser_idx = 0; AppLayerProto *p = &al_proto_table[proto]; TcpSession *ssn = f->stream; if (ssn == NULL) { + printf("AppLayerParse: no stream\n"); return -1; } /* Get the parser state (if any) */ - AppLayerParserState *parser_state = (AppLayerParserState *)ssn->l7data[app_layer_sid]; - /* See if we already have a 'app' state */ - void *app_layer_state = ssn->l7data[p->storage_id]; + AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->l7data[app_layer_sid]; + if (parser_state_store == NULL) { + parser_state_store = AppLayerParserStateStoreAlloc(); + if (parser_state_store == NULL) + return -1; - if (parser_state == NULL) { - if (flags & STREAM_TOSERVER) { + ssn->l7data[app_layer_sid] = (void *)parser_state_store; + } + + AppLayerParserState *parser_state = NULL; + if (flags & STREAM_TOSERVER) { + parser_state = &parser_state_store->to_server; + if (!(parser_state->flags & APP_LAYER_PARSER_USE)) { parser_idx = p->to_server; - } else if (flags & STREAM_TOCLIENT) { - parser_idx = p->to_client; + parser_state->cur_parser = parser_idx; + parser_state->flags |= APP_LAYER_PARSER_USE; + } else { + //printf("AppLayerParse: using parser %u we stored before (to_server)\n", parser_state->cur_parser); + parser_idx = parser_state->cur_parser; } } else { - printf("AppLayerParse: using parser %u we stored before\n", parser_state->cur_parser); - parser_idx = parser_state->cur_parser; + parser_state = &parser_state_store->to_client; + if (!(parser_state->flags & APP_LAYER_PARSER_USE)) { + parser_idx = p->to_client; + parser_state->cur_parser = parser_idx; + parser_state->flags |= APP_LAYER_PARSER_USE; + } else { + //printf("AppLayerParse: using parser %u we stored before (to_client)\n", parser_state->cur_parser); + parser_idx = parser_state->cur_parser; + } } if (parser_idx == 0) { @@ -149,40 +486,24 @@ int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_in return 0; } - if (parser_state == NULL) { - parser_state = AppLayerParserStateAlloc(); - if (parser_state != NULL) { - parser_state->cur_parser = parser_idx; + if (flags & STREAM_EOF) + parser_state->flags |= APP_LAYER_PARSER_EOF; - ssn->l7data[app_layer_sid] = (void *)parser_state; - } + /* See if we already have a 'app layer' state */ + void *app_layer_state = ssn->l7data[p->storage_id]; + if (app_layer_state == NULL) { + app_layer_state = p->StateAlloc(); + if (app_layer_state == NULL) + return -1; + + ssn->l7data[p->storage_id] = app_layer_state; } - AppLayerParserResultElement *result_tbl[256]; - memset(&result_tbl,0,sizeof(result_tbl)); - u_int16_t output_num = 0; - int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, input, input_len, result_tbl, &output_num); + /* invoke the recursive parser */ + int r = AppLayerDoParse(app_layer_state, parser_state, input, input_len, parser_idx, proto); if (r < 0) return -1; - printf("AppLayerParse: output_num %u\n", output_num); - u_int16_t u = 0; - for (u = 0; u < output_num; u++) { - AppLayerParserResultElement *e = result_tbl[u]; - printf("AppLayerParse: e->name_idx %u, e->data_ptr %p, e->data_len %u, map_size %u\n", e->name_idx, e->data_ptr, e->data_len, al_proto_table[proto].map_size); - - /* no parser defined for this field. */ - if (e->name_idx >= al_proto_table[proto].map_size || al_proto_table[proto].map[e->name_idx] == NULL) { - printf("AppLayerParse: no parser for proto %u, parser_local_id %u\n", proto, e->name_idx); - continue; - } - - parser_idx = al_proto_table[proto].map[e->name_idx]->parser_id; - int r = al_parser_table[parser_idx].AppLayerParser(app_layer_state, parser_state, e->data_ptr, e->data_len, result_tbl, &output_num); - if (r < 0) - return -1; - } - return 0; } @@ -195,9 +516,13 @@ void RegisterAppLayerParsers(void) { /** setup result pool * \todo Per thread pool */ - al_result_pool = PoolInit(100,10,AppLayerParserResultElementAlloc,NULL,AppLayerParserResultElementFree); + al_result_pool = PoolInit(100,10,AlpResultElmtPoolAlloc,NULL,AlpResultElmtPoolFree); } +/** \brief Create a mapping between the individual parsers local field id's + * and the global field parser id's. + * + */ void AppLayerParsersInitPostProcess(void) { printf("AppLayerParsersInitPostProcess: start\n"); u_int16_t u16 = 0; @@ -211,7 +536,7 @@ void AppLayerParsersInitPostProcess(void) { if (al_parser_table[u16].parser_local_id > al_proto_table[al_parser_table[u16].proto].map_size) al_proto_table[al_parser_table[u16].proto].map_size = al_parser_table[u16].parser_local_id; - printf("AppLayerParsersInitPostProcess: map_size %u\n", al_proto_table[al_parser_table[u16].proto].map_size); + //printf("AppLayerParsersInitPostProcess: map_size %u\n", al_proto_table[al_parser_table[u16].proto].map_size); } /* for each proto, alloc the map array */ @@ -228,7 +553,6 @@ void AppLayerParsersInitPostProcess(void) { memset(al_proto_table[u16].map, 0, al_proto_table[u16].map_size * sizeof(AppLayerLocalMap *)); u_int16_t u = 0; - u_int16_t x = 0; for (u = 1; u <= al_max_parsers; u++) { /* no local parser */ if (al_parser_table[u].parser_local_id == 0) @@ -237,9 +561,9 @@ void AppLayerParsersInitPostProcess(void) { if (al_parser_table[u].proto != u16) continue; - printf("al_proto_table[%u].map_size %u, x %u, %p %p\n", u16, al_proto_table[u16].map_size, x, al_proto_table[u16].map[x], al_proto_table[u16].map); + //printf("AppLayerParsersInitPostProcess: al_proto_table[%u].map_size %u, %p %p\n", u16, al_proto_table[u16].map_size, al_proto_table[u16].map[x], al_proto_table[u16].map); u_int16_t parser_local_id = al_parser_table[u].parser_local_id; - printf("parser_local_id: %u\n", parser_local_id); + //printf("AppLayerParsersInitPostProcess: parser_local_id: %u\n", parser_local_id); if (parser_local_id < al_proto_table[u16].map_size) { al_proto_table[u16].map[parser_local_id] = malloc(sizeof(AppLayerLocalMap)); @@ -265,7 +589,7 @@ void AppLayerParsersInitPostProcess(void) { if (al_proto_table[u16].map[x] == NULL) continue; - printf("al_proto_table[%u].map[%u]->parser_id: %u\n", u16, x, al_proto_table[u16].map[x]->parser_id); + //printf("AppLayerParsersInitPostProcess: al_proto_table[%u].map[%u]->parser_id: %u\n", u16, x, al_proto_table[u16].map[x]->parser_id); } } } diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index 1744dbb185..48a6d0f50b 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -18,39 +18,72 @@ typedef struct AppLayerProto_ { AppLayerLocalMap **map; u_int16_t map_size; + + void *(*StateAlloc)(void); + void (*StateFree)(void *); } AppLayerProto; -typedef struct AppLayerParserResultElement_ { +/** flags for the result elmts */ +#define ALP_RESULT_ELMT_ALLOC 0x01 + +/** \brief Result elements for the parser */ +typedef struct AppLayerParserResultElmt_ { u_int16_t flags; /* flags. E.g. local alloc */ u_int16_t name_idx; /* idx for names like "http.request_line.uri" */ u_int8_t *data_ptr; /* point to the position in the "input" data * or ptr to new mem if local alloc flag set */ u_int32_t data_len; /* length of the data from the ptr */ -} AppLayerParserResultElement; + struct AppLayerParserResultElmt_ *next; +} AppLayerParserResultElmt; + +/** \brief List head for parser result elmts */ +typedef struct AppLayerParserResult_ { + AppLayerParserResultElmt *head; + AppLayerParserResultElmt *tail; + u_int32_t cnt; +} AppLayerParserResult; + +#define APP_LAYER_PARSER_USE 0x01 +#define APP_LAYER_PARSER_EOF 0x02 + +typedef struct AppLayerParserState_ { + u_int8_t flags; + + u_int16_t cur_parser; /* idx of currently active parser */ + u_int8_t *store; + u_int32_t store_len; + u_int16_t parse_field; +} AppLayerParserState; + +typedef struct AppLayerParserStateStore_ { + AppLayerParserState to_client; + AppLayerParserState to_server; +} AppLayerParserStateStore; typedef struct AppLayerParserTableElement_ { char *name; u_int16_t proto; u_int16_t parser_local_id; /** local id of the parser in the parser itself. */ u_int8_t flags; - int (*AppLayerParser)(void *protocol_state, void *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResultElement **output, u_int16_t *output_num); + int (*AppLayerParser)(void *protocol_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output); u_int16_t max_outputs; /* rationele is that if we know the max outputs of all parsers, we can statically define our output array to be a certain size */ } AppLayerParserTableElement; -#define APP_LAYER_PARSER_DONE 0x01 /** the last parser was done */ -#define APP_LAYER_PARSER_MAYBE 0x02 /** we're not sure if the last parser is done */ -#define APP_LAYER_PARSER_CONT 0x04 /** the last parser is still working */ +/* prototypes */ +void AppLayerParsersInitPostProcess(void); +void RegisterAppLayerParsers(void); -typedef struct AppLayerParserState_ { - u_int8_t flags; - u_int16_t cur_parser; /* idx of currently active parser */ +int AppLayerRegisterProto(char *name, u_int8_t proto, u_int8_t flags, int (*AppLayerParser)(void *protocol_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output)); +int AppLayerRegisterParser(char *name, u_int16_t proto, u_int16_t parser_id, int (*AppLayerParser)(void *protocol_state, AppLayerParserState *parser_state, u_int8_t *input, u_int32_t input_len, AppLayerParserResult *output), char *dependency); +void AppLayerRegisterStateFuncs(u_int16_t proto, void *(*StateAlloc)(void), void (*StateFree)(void *)); - /** \todo this needs to become dynamic */ - u_int8_t buf[1024]; - u_int32_t buflen; -} AppLayerParserState; +int AppLayerParse(Flow *f, u_int8_t proto, u_int8_t flags, u_int8_t *input, u_int32_t input_len); + +int AlpParseFieldByEOF(AppLayerParserResult *, AppLayerParserState *, u_int16_t, u_int8_t *, u_int32_t); +int AlpParseFieldByDelimiter(AppLayerParserResult *, AppLayerParserState *, u_int16_t, const u_int8_t *, u_int8_t, u_int8_t *, u_int32_t, u_int32_t *); +u_int16_t AlpGetStateIdx(u_int16_t); #endif /* __APP_LAYER_PARSER_H__ */ diff --git a/src/eidps.c b/src/eidps.c index 8c9fbf114d..8ea4bcbfdd 100644 --- a/src/eidps.c +++ b/src/eidps.c @@ -897,12 +897,11 @@ int main(int argc, char **argv) PatternMatchPrepare(mpm_ctx); PerfInitCounterApi(); - /* XXX we need an api for this */ + /** \todo we need an api for this */ AppLayerDetectProtoThreadInit(); - -RegisterAppLayerParsers(); -RegisterHTTPParsers(); -AppLayerParsersInitPostProcess(); + RegisterAppLayerParsers(); + RegisterHTTPParsers(); + AppLayerParsersInitPostProcess(); TmModuleReceiveNFQRegister(); TmModuleVerdictNFQRegister(); @@ -942,6 +941,7 @@ AppLayerParsersInitPostProcess(); SigRegisterTests(); PerfRegisterTests(); DecodePPPRegisterTests(); + HTTPParserRegisterTests(); UtRunTests(); UtCleanup(); exit(0);