diff --git a/doc/Basic_Setup.txt b/doc/Basic_Setup.txt index 1769e1d484..97d642c4b5 100644 --- a/doc/Basic_Setup.txt +++ b/doc/Basic_Setup.txt @@ -103,14 +103,14 @@ enter: Followed by: - tail http.log + tail eve.json And: tail -n 50 stats.log -To make sure the information displayed is up-dated in real time, use the - -f option before http.log and stats.log: +To make sure the information displayed is up-dated in real time, use the -f +option before eve.json and stats.log: - tail -f http.log stats.log + tail -f eve.json stats.log diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst index 5c1edf30c9..02f61f1289 100644 --- a/doc/userguide/configuration/suricata-yaml.rst +++ b/doc/userguide/configuration/suricata-yaml.rst @@ -430,43 +430,6 @@ Example: enabled: yes #certs-log-dir: certs # directory to store the certificates files -A line based log of HTTP requests (http.log) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. attention:: http-log is deprecated in Suricata 8.0 and will be - removed in Suricata 9.0. - -This log keeps track of all HTTP-traffic events. It contains the HTTP -request, hostname, URI and the User-Agent. This information will be -stored in the http.log (default name, in the suricata log -directory). This logging can also be performed through the use of the -:ref:`Eve-log capability `. - -Example of a HTTP-log line with non-extended logging: - -:: - - 07/01/2014-04:20:14.338309 vg.no [**] / [**] Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) - AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36 [**] - 192.168.1.6:64685 -> 195.88.54.16:80 - -Example of a HTTP-log line with extended logging: - -:: - - 07/01/2014-04:21:06.994705 vg.no [**] / [**] Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) - AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36 [**] [**] - GET [**] HTTP/1.1 [**] 301 => http://www.vg.no/ [**] 239 bytes [**] 192.168.1.6:64726 -> 195.88.54.16:80 - -:: - - - http-log: #The log-name. - enabled: yes #This log is enabled. Set 'no' to disable. - filename: http.log #The name of the file in the default logging directory. - append: yes/no #If this option is set to yes, the last filled http.log-file will not be - # overwritten while restarting Suricata. - extended: yes # If set to yes more information is written about the event. - .. _suricata_yaml_pcap_log: Packet log (pcap-log) diff --git a/doc/userguide/lua/libs/flowlib.rst b/doc/userguide/lua/libs/flowlib.rst index bdb8ab6782..112a964fd6 100644 --- a/doc/userguide/lua/libs/flowlib.rst +++ b/doc/userguide/lua/libs/flowlib.rst @@ -35,8 +35,7 @@ microseconds since `1970-01-01 00:00:00` UTC, returning 4 numbers:: ^^^^^^^^^^^^^^^^^^^^^ Get the timestamp of the first packet from the flow, as a string in the format: -`11/24/2009-18:57:25.179869`. This is the format used by `fast.log`, `http.log` -and other legacy outputs. +`11/24/2009-18:57:25.179869`. This is the format used by `fast.log`. :: diff --git a/doc/userguide/lua/libs/packetlib.rst b/doc/userguide/lua/libs/packetlib.rst index 8924dba3fa..6587c770d1 100644 --- a/doc/userguide/lua/libs/packetlib.rst +++ b/doc/userguide/lua/libs/packetlib.rst @@ -38,7 +38,7 @@ Get packet timestamp as 2 numbers: seconds & microseconds elapsed since ^^^^^^^^^^^^^^^^^^^^^ Get packet timestamp as a string in the format: `11/24/2009-18:57:25.179869`. -This is the format used by `fast.log`, `http.log` and other legacy outputs. +This is the format used by `fast.log`. :: diff --git a/doc/userguide/output/custom-http-logging.rst b/doc/userguide/output/custom-http-logging.rst deleted file mode 100644 index 3b683337e8..0000000000 --- a/doc/userguide/output/custom-http-logging.rst +++ /dev/null @@ -1,53 +0,0 @@ -Custom http logging -=================== - -.. attention:: http-log is deprecated in Suricata 8.0 and will be - removed in Suricata 9.0. - -In your Suricata.yaml, find the http-log section and edit as follows: - - -:: - - - - http-log: - enabled: yes - filename: http.log - custom: yes # enable the custom logging format (defined by custom format) - customformat: "%{%D-%H:%M:%S}t.%z %{X-Forwarded-For}i %{User-agent}i %H %m %h %u %s %B %a:%p -> %A:%P" - append: no - #extended: yes # enable this for extended logging information - #filetype: regular # 'regular', 'unix_stream' or 'unix_dgram' - -And in your http.log file you would get the following, for example: - -:: - - 8/28/12-22:14:21.101619 - Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:11.0) Gecko/20100101 Firefox/11.0 HTTP/1.1 GET us.cnn.com /video/data/3.0/video/world/2012/08/28/hancocks-korea-typhoon-bolavan.cnn/index.xml 200 16856 192.168.1.91:45111 -> 157.166.255.18:80 - -:: - - 08/28/12-22:14:30.693856 - Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:11.0) Gecko/20100101 Firefox/11.0 HTTP/1.1 GET us.cnn.com /video/data/3.0/video/showbiz/2012/08/28/conan-reports-from-rnc-convention.teamcoco/index.xml 200 15789 192.168.1.91:45108 -> 157.166.255.18:80 - -The list of supported format strings is the following: - -* %h - Host HTTP Header (remote host name). ie: google.com -* %H - Request Protocol. ie: HTTP/1.1 -* %m - Request Method. ie: GET -* %u - URL including query string. ie: /search?q=suricata -* %{header_name}i - contents of the defined HTTP Request Header name. ie: - - * %{User-agent}i: Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:11.0) Gecko/20100101 Firefox/11.0 - * %{X-Forwarded-For}i: outputs the IP address contained in the X-Forwarded-For HTTP header (inserted by a reverse proxy) - -* %s - return status code. In the case of 301 and 302 it will print the url in brackets. ie: 200 -* %B - response size in bytes. ie: 15789 -* %{header_name}o - contents of the defined HTTP Response Header name -* %{strftime_format]t - timestamp of the HTTP transaction in the selected strftime format. ie: 08/28/12-22:14:30 -* %z - precision time in useconds. ie: 693856 -* %a - client IP address -* %p - client port number -* %A - server IP address -* %P - server port number - -Any non printable character will be represented by its byte value in hexadecimal format (\|XX\|, where XX is the hex code) diff --git a/doc/userguide/output/index.rst b/doc/userguide/output/index.rst index dd44536701..c82224b48d 100644 --- a/doc/userguide/output/index.rst +++ b/doc/userguide/output/index.rst @@ -6,6 +6,5 @@ Output eve/index.rst lua-output syslog-alerting-comp - custom-http-logging custom-tls-logging log-rotation diff --git a/doc/userguide/performance/ignoring-traffic.rst b/doc/userguide/performance/ignoring-traffic.rst index 25db399d79..506b7085ba 100644 --- a/doc/userguide/performance/ignoring-traffic.rst +++ b/doc/userguide/performance/ignoring-traffic.rst @@ -54,8 +54,8 @@ Example:: pass ip 1.2.3.4 any <> any any (msg:"pass all traffic from/to 1.2.3.4"; sid:1;) -A big difference with capture filters is that logs such as Eve or http.log -are still generated for this traffic. +A big difference with capture filters is that logs such as Eve are still +generated for this traffic. suppress -------- diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst index c835bb1c69..0110b611e4 100644 --- a/doc/userguide/upgrade.rst +++ b/doc/userguide/upgrade.rst @@ -1,3 +1,4 @@ + Upgrading ========= @@ -62,6 +63,12 @@ Logging Changes ``stats.app_layer.*.ftp-data`` becomes ``stats.app_layer.*.ftp_data``, and same for bittorrent_dht +Removals +~~~~~~~~ + +- The deprecated ``http-log`` output has been removed. Use ``eve-log`` + with the ``http`` event type for HTTP logging. + Other Changes ~~~~~~~~~~~~~ - ``engine-analysis`` output has been changed for keywords that now use the generic integers framework diff --git a/qa/wirefuzz.pl b/qa/wirefuzz.pl index 759cd9a04a..2c6a9966eb 100755 --- a/qa/wirefuzz.pl +++ b/qa/wirefuzz.pl @@ -318,7 +318,7 @@ sub printhelp { -x=<(optional) regex for excluding certain files incase something blows up but we want to continue fuzzing .> -z=<(optional) regex for excluding certain files from fuzzing but still process them note: the original files will be processed and not removed.> -y - -k + -k Example usage: First thing to do is download and build suricata from git with -O0 so vars don't get optimized out. See the example below: git clone git://phalanx.openinfosecfoundation.org/oisf.git suricatafuzz1 && cd suricatafuzz1 && ./autogen.sh && CFLAGS=\"-g -O0\" ./configure && make @@ -674,14 +674,6 @@ sub keep_logs { . "-fast.log"; system($savecmd); } - if (-e $logdir . "http.log"){ - $savecmd = "mv -f " . $logdir - . "http.log " - . $logdir - . $saveme - . "-http.log"; - system($savecmd); - } if (-e $logdir . "stats.log"){ $savecmd = "mv -f " . $logdir . "stats.log " diff --git a/src/Makefile.am b/src/Makefile.am index 6b0039a2f6..0ed1409559 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -339,7 +339,6 @@ noinst_HEADERS = \ ippair.h \ log-cf-common.h \ log-flush.h \ - log-httplog.h \ log-pcap.h \ log-stats.h \ log-tcp-data.h \ @@ -914,7 +913,6 @@ libsuricata_c_a_SOURCES = \ ippair.c \ log-cf-common.c \ log-flush.c \ - log-httplog.c \ log-pcap.c \ log-stats.c \ log-tcp-data.c \ diff --git a/src/log-httplog.c b/src/log-httplog.c deleted file mode 100644 index c776520cc1..0000000000 --- a/src/log-httplog.c +++ /dev/null @@ -1,620 +0,0 @@ -/* Copyright (C) 2007-2013 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Victor Julien - * \author Ignacio Sanchez - * - * Implements http logging portion of the engine. - */ - -#include "suricata-common.h" -#include "detect.h" -#include "pkt-var.h" -#include "conf.h" - -#include "threads.h" -#include "threadvars.h" -#include "tm-threads.h" - -#include "util-print.h" -#include "util-unittest.h" - -#include "util-debug.h" - -#include "output.h" -#include "log-httplog.h" -#include "app-layer-htp.h" -#include "app-layer.h" -#include "app-layer-parser.h" -#include "util-privs.h" -#include "util-buffer.h" - -#include "util-logopenfile.h" -#include "util-time.h" -#include "log-cf-common.h" - -#define DEFAULT_LOG_FILENAME "http.log" - -#define MODULE_NAME "LogHttpLog" - -#define OUTPUT_BUFFER_SIZE 65535 - -TmEcode LogHttpLogThreadInit(ThreadVars *, const void *, void **); -TmEcode LogHttpLogThreadDeinit(ThreadVars *, void *); -static void LogHttpLogDeInitCtx(OutputCtx *); - -int LogHttpLogger(ThreadVars *tv, void *thread_data, const Packet *, Flow *f, void *state, void *tx, uint64_t tx_id); - -void LogHttpLogRegister (void) -{ - OutputRegisterTxModule(LOGGER_HTTP, MODULE_NAME, "http-log", LogHttpLogInitCtx, ALPROTO_HTTP1, - LogHttpLogger, LogHttpLogThreadInit, LogHttpLogThreadDeinit); -} - -#define LOG_HTTP_CF_REQUEST_HOST 'h' -#define LOG_HTTP_CF_REQUEST_PROTOCOL 'H' -#define LOG_HTTP_CF_REQUEST_METHOD 'm' -#define LOG_HTTP_CF_REQUEST_URI 'u' -#define LOG_HTTP_CF_REQUEST_HEADER 'i' -#define LOG_HTTP_CF_REQUEST_COOKIE 'C' -#define LOG_HTTP_CF_REQUEST_LEN 'b' -#define LOG_HTTP_CF_RESPONSE_STATUS 's' -#define LOG_HTTP_CF_RESPONSE_HEADER 'o' -#define LOG_HTTP_CF_RESPONSE_LEN 'B' - - -typedef struct LogHttpFileCtx_ { - LogFileCtx *file_ctx; - uint32_t flags; /** Store mode */ - LogCustomFormat *cf; -} LogHttpFileCtx; - -#define LOG_HTTP_DEFAULT 0 -#define LOG_HTTP_EXTENDED 1 -#define LOG_HTTP_CUSTOM 2 - -typedef struct LogHttpLogThread_ { - LogHttpFileCtx *httplog_ctx; - /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ - uint32_t uri_cnt; - - MemBuffer *buffer; -} LogHttpLogThread; - -/* Retrieves the selected cookie value */ -static uint32_t GetCookieValue(const uint8_t *rawcookies, uint32_t rawcookies_len, char *cookiename, - const uint8_t **cookievalue) -{ - const uint8_t *p = rawcookies; - const uint8_t *cn = p; /* ptr to cookie name start */ - const uint8_t *cv = NULL; /* ptr to cookie value start */ - while (p < rawcookies + rawcookies_len) { - if (cv == NULL && *p == '=') { - cv = p + 1; - } else if (cv != NULL && (*p == ';' || p == rawcookies + rawcookies_len - 1) ) { - /* Found end of cookie */ - p++; - if (strlen(cookiename) == (unsigned int) (cv-cn-1) && - strncmp(cookiename, (char *) cn, cv-cn-1) == 0) { - *cookievalue = cv; - return (uint32_t) (p-cv); - } - cv = NULL; - cn = p + 1; - } - p++; - } - return 0; -} - -/* Custom format logging */ -static void LogHttpLogCustom(LogHttpLogThread *aft, htp_tx_t *tx, const SCTime_t ts, char *srcip, - Port sp, char *dstip, Port dp) -{ - LogHttpFileCtx *httplog_ctx = aft->httplog_ctx; - uint32_t i; - size_t datalen; - char buf[128]; - - const uint8_t *cvalue = NULL; - uint32_t cvalue_len = 0; - - const htp_header_t *h_request_hdr; - const htp_header_t *h_response_hdr; - - for (i = 0; i < httplog_ctx->cf->cf_n; i++) { - h_request_hdr = NULL; - h_response_hdr = NULL; - - LogCustomFormatNode * node = httplog_ctx->cf->cf_nodes[i]; - if (! node) /* Should never happen */ - continue; - - switch (node->type){ - case LOG_CF_LITERAL: - /* LITERAL */ - MemBufferWriteString(aft->buffer, "%s", node->data); - break; - case LOG_CF_TIMESTAMP: - /* TIMESTAMP */ - LogCustomFormatWriteTimestamp(aft->buffer, node->data, ts); - break; - case LOG_CF_TIMESTAMP_U: - /* TIMESTAMP USECONDS */ - snprintf(buf, sizeof(buf), "%06u", (unsigned int)SCTIME_USECS(ts)); - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)buf, MIN(strlen(buf), 6)); - break; - case LOG_CF_CLIENT_IP: - /* CLIENT IP ADDRESS */ - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, - aft->buffer->size, (uint8_t *)srcip,strlen(srcip)); - break; - case LOG_CF_SERVER_IP: - /* SERVER IP ADDRESS */ - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, - aft->buffer->size, (uint8_t *)dstip,strlen(dstip)); - break; - case LOG_CF_CLIENT_PORT: - /* CLIENT PORT */ - MemBufferWriteString(aft->buffer, "%" PRIu16 "", sp); - break; - case LOG_CF_SERVER_PORT: - /* SERVER PORT */ - MemBufferWriteString(aft->buffer, "%" PRIu16 "", dp); - break; - case LOG_HTTP_CF_REQUEST_METHOD: - /* METHOD */ - if (htp_tx_request_method(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_method(tx)), - bstr_len(htp_tx_request_method(tx))); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_URI: - /* URI */ - if (htp_tx_request_uri(tx) != NULL) { - datalen = node->maxlen; - if (datalen == 0 || datalen > bstr_len(htp_tx_request_uri(tx))) { - datalen = bstr_len(htp_tx_request_uri(tx)); - } - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_uri(tx)), datalen); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_HOST: - /* HOSTNAME */ - if (htp_tx_request_hostname(tx) != NULL) { - datalen = node->maxlen; - if (datalen == 0 || datalen > bstr_len(htp_tx_request_hostname(tx))) { - datalen = bstr_len(htp_tx_request_hostname(tx)); - } - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_hostname(tx)), datalen); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_PROTOCOL: - /* PROTOCOL */ - if (htp_tx_request_protocol(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_protocol(tx)), - bstr_len(htp_tx_request_protocol(tx))); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_HEADER: - /* REQUEST HEADER */ - h_request_hdr = htp_tx_request_header(tx, node->data); - if (h_request_hdr != NULL) { - datalen = node->maxlen; - if (datalen == 0 || datalen > htp_header_value_len(h_request_hdr)) { - datalen = htp_header_value_len(h_request_hdr); - } - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - htp_header_value_ptr(h_request_hdr), datalen); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_COOKIE: - /* REQUEST COOKIE */ - if (htp_tx_request_headers(tx) != NULL) { - h_request_hdr = htp_tx_request_header(tx, "Cookie"); - if (h_request_hdr != NULL) { - cvalue_len = GetCookieValue(htp_header_value_ptr(h_request_hdr), - (uint32_t)htp_header_value_len(h_request_hdr), (char *)node->data, - &cvalue); - } - } - if (cvalue_len > 0 && cvalue != NULL) { - datalen = node->maxlen; - if (datalen == 0 || datalen > cvalue_len) { - datalen = cvalue_len; - } - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, - aft->buffer->size, cvalue, datalen); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_REQUEST_LEN: - /* REQUEST LEN */ - MemBufferWriteString( - aft->buffer, "%" PRIuMAX "", (uintmax_t)htp_tx_request_message_len(tx)); - break; - case LOG_HTTP_CF_RESPONSE_STATUS: - /* RESPONSE STATUS */ - if (htp_tx_response_status(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_response_status(tx)), - bstr_len(htp_tx_response_status(tx))); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_RESPONSE_HEADER: - /* RESPONSE HEADER */ - if (htp_tx_response_headers(tx) != NULL) { - h_response_hdr = htp_tx_response_header(tx, node->data); - } - if (h_response_hdr != NULL) { - datalen = node->maxlen; - if (datalen == 0 || datalen > htp_header_value_len(h_response_hdr)) { - datalen = htp_header_value_len(h_response_hdr); - } - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, - aft->buffer->size, htp_header_value_ptr(h_response_hdr), datalen); - } else { - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - } - break; - case LOG_HTTP_CF_RESPONSE_LEN: - /* RESPONSE LEN */ - MemBufferWriteString( - aft->buffer, "%" PRIuMAX "", (uintmax_t)htp_tx_response_message_len(tx)); - break; - default: - /* NO MATCH */ - MemBufferWriteString(aft->buffer, LOG_CF_NONE); - SCLogDebug("No matching parameter %%%c for custom http log.", node->type); - break; - } - } - MemBufferWriteString(aft->buffer, "\n"); -} - -static void LogHttpLogExtended(LogHttpLogThread *aft, htp_tx_t *tx) -{ - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* referer */ - const htp_header_t *h_referer = htp_tx_request_header(tx, "referer"); - - if (h_referer != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - htp_header_value_ptr(h_referer), htp_header_value_len(h_referer)); - } else { - MemBufferWriteString(aft->buffer, ""); - } - - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* method */ - if (htp_tx_request_method(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_method(tx)), - bstr_len(htp_tx_request_method(tx))); - } - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* protocol */ - if (htp_tx_request_protocol(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_protocol(tx)), - bstr_len(htp_tx_request_protocol(tx))); - } else { - MemBufferWriteString(aft->buffer, ""); - } - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* response status */ - if (htp_tx_response_status(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_response_status(tx)), - bstr_len(htp_tx_response_status(tx))); - /* Redirect? */ - if ((htp_tx_response_status_number(tx) > 300) && - ((htp_tx_response_status_number(tx)) < 303)) { - const htp_header_t *h_location = htp_tx_response_header(tx, "location"); - if (h_location != NULL) { - MemBufferWriteString(aft->buffer, " => "); - - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - htp_header_value_ptr(h_location), htp_header_value_len(h_location)); - } - } - } else { - MemBufferWriteString(aft->buffer, ""); - } - - /* length */ - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - MemBufferWriteString( - aft->buffer, "%" PRIuMAX " bytes", (uintmax_t)htp_tx_response_message_len(tx)); -} - -static TmEcode LogHttpLogIPWrapper(ThreadVars *tv, void *data, const Packet *p, Flow *f, HtpState *htp_state, htp_tx_t *tx, uint64_t tx_id, int ipproto) -{ - SCEnter(); - - LogHttpLogThread *aft = (LogHttpLogThread *)data; - LogHttpFileCtx *hlog = aft->httplog_ctx; - char timebuf[64]; - - /* check if we have HTTP state or not */ - CreateTimeString(p->ts, timebuf, sizeof(timebuf)); - - char srcip[46], dstip[46]; - Port sp, dp; - if ((PKT_IS_TOSERVER(p))) { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->sp; - dp = p->dp; - } else { - switch (ipproto) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), dstip, sizeof(dstip)); - break; - default: - goto end; - } - sp = p->dp; - dp = p->sp; - } - - SCLogDebug("got a HTTP request and now logging !!"); - - /* reset */ - MemBufferReset(aft->buffer); - - if (hlog->flags & LOG_HTTP_CUSTOM) { - LogHttpLogCustom(aft, tx, p->ts, srcip, sp, dstip, dp); - } else { - /* time */ - MemBufferWriteString(aft->buffer, "%s ", timebuf); - - /* hostname */ - if (htp_tx_request_hostname(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_hostname(tx)), - bstr_len(htp_tx_request_hostname(tx))); - } else { - MemBufferWriteString(aft->buffer, ""); - } - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* uri */ - if (htp_tx_request_uri(tx) != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - (uint8_t *)bstr_ptr(htp_tx_request_uri(tx)), bstr_len(htp_tx_request_uri(tx))); - } - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - - /* user agent */ - const htp_header_t *h_user_agent = htp_tx_request_header(tx, "user-agent"); - if (h_user_agent != NULL) { - PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size, - htp_header_value_ptr(h_user_agent), htp_header_value_len(h_user_agent)); - } else { - MemBufferWriteString(aft->buffer, ""); - } - if (hlog->flags & LOG_HTTP_EXTENDED) { - LogHttpLogExtended(aft, tx); - } - - /* ip/tcp header info */ - LOG_CF_WRITE_STAR_SEPARATOR(aft->buffer); - MemBufferWriteString(aft->buffer, - "%s:%" PRIu16 " -> %s:%" PRIu16 "\n", - srcip, sp, dstip, dp); - } - - aft->uri_cnt ++; - - hlog->file_ctx->Write((const char *)MEMBUFFER_BUFFER(aft->buffer), - MEMBUFFER_OFFSET(aft->buffer), hlog->file_ctx); - -end: - SCReturnInt(0); - -} - -int LogHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *state, void *tx, uint64_t tx_id) -{ - SCEnter(); - - if (!(PacketIsTCP(p))) { - SCReturnInt(TM_ECODE_OK); - } - - int r = 0; - if (PacketIsIPv4(p)) { - r = LogHttpLogIPWrapper(tv, thread_data, p, f, (HtpState *)state, (htp_tx_t *)tx, tx_id, AF_INET); - } else if (PacketIsIPv6(p)) { - r = LogHttpLogIPWrapper(tv, thread_data, p, f, (HtpState *)state, (htp_tx_t *)tx, tx_id, AF_INET6); - } - - SCReturnInt(r); -} - -TmEcode LogHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data) -{ - LogHttpLogThread *aft = SCCalloc(1, sizeof(LogHttpLogThread)); - if (unlikely(aft == NULL)) - return TM_ECODE_FAILED; - - if(initdata == NULL) - { - SCLogDebug("Error getting context for LogHTTPLog. \"initdata\" argument NULL"); - SCFree(aft); - return TM_ECODE_FAILED; - } - - aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); - if (aft->buffer == NULL) { - SCFree(aft); - return TM_ECODE_FAILED; - } - - /* Use the Output Context (file pointer and mutex) */ - aft->httplog_ctx= ((OutputCtx *)initdata)->data; - - *data = (void *)aft; - return TM_ECODE_OK; -} - -TmEcode LogHttpLogThreadDeinit(ThreadVars *t, void *data) -{ - LogHttpLogThread *aft = (LogHttpLogThread *)data; - if (aft == NULL) { - return TM_ECODE_OK; - } - - MemBufferFree(aft->buffer); - /* clear memory */ - memset(aft, 0, sizeof(LogHttpLogThread)); - - SCFree(aft); - return TM_ECODE_OK; -} - -/** \brief Create a new http log LogFileCtx. - * \param conf Pointer to ConfNode containing this loggers configuration. - * \return NULL if failure, LogFileCtx* to the file_ctx if succesful - * */ -OutputInitResult LogHttpLogInitCtx(SCConfNode *conf) -{ - SCLogWarning("The http-log output has been deprecated and will be removed in Suricata 9.0."); - OutputInitResult result = { NULL, false }; - LogFileCtx* file_ctx = LogFileNewCtx(); - if(file_ctx == NULL) { - SCLogError("couldn't create new file_ctx"); - return result; - } - - if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) { - LogFileFreeCtx(file_ctx); - return result; - } - - LogHttpFileCtx *httplog_ctx = SCCalloc(1, sizeof(LogHttpFileCtx)); - if (unlikely(httplog_ctx == NULL)) { - LogFileFreeCtx(file_ctx); - return result; - } - - httplog_ctx->file_ctx = file_ctx; - - const char *extended = SCConfNodeLookupChildValue(conf, "extended"); - const char *custom = SCConfNodeLookupChildValue(conf, "custom"); - const char *customformat = SCConfNodeLookupChildValue(conf, "customformat"); - - /* If custom logging format is selected, lets parse it */ - if (custom != NULL && customformat != NULL && SCConfValIsTrue(custom)) { - - httplog_ctx->cf = LogCustomFormatAlloc(); - if (!httplog_ctx->cf) { - goto errorfree; - } - - httplog_ctx->flags |= LOG_HTTP_CUSTOM; - - /* Parsing */ - if ( ! LogCustomFormatParse(httplog_ctx->cf, customformat)) { - goto parsererror; - } - } else { - if (extended == NULL) { - httplog_ctx->flags |= LOG_HTTP_DEFAULT; - } else { - if (SCConfValIsTrue(extended)) { - httplog_ctx->flags |= LOG_HTTP_EXTENDED; - } - } - } - - OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); - if (unlikely(output_ctx == NULL)) { - goto parsererror; - } - - output_ctx->data = httplog_ctx; - output_ctx->DeInit = LogHttpLogDeInitCtx; - - SCLogDebug("HTTP log output initialized"); - - /* enable the logger for the app layer */ - SCAppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP1); - - result.ctx = output_ctx; - result.ok = true; - return result; - -parsererror: - SCLogError("Syntax error in custom http log format string."); -errorfree: - LogCustomFormatFree(httplog_ctx->cf); - LogFileFreeCtx(file_ctx); - SCFree(httplog_ctx); - return result; - -} - -static void LogHttpLogDeInitCtx(OutputCtx *output_ctx) -{ - LogHttpFileCtx *httplog_ctx = (LogHttpFileCtx *)output_ctx->data; - LogCustomFormatFree(httplog_ctx->cf); - LogFileFreeCtx(httplog_ctx->file_ctx); - SCFree(httplog_ctx); - SCFree(output_ctx); -} diff --git a/src/log-httplog.h b/src/log-httplog.h deleted file mode 100644 index 8ec6ba2171..0000000000 --- a/src/log-httplog.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Victor Julien - */ - -#ifndef SURICATA_LOG_HTTPLOG_H -#define SURICATA_LOG_HTTPLOG_H - -void LogHttpLogRegister(void); -OutputInitResult LogHttpLogInitCtx(SCConfNode *); - -#endif /* SURICATA_LOG_HTTPLOG_H */ diff --git a/src/output-file.c b/src/output-file.c index e468f14d25..f8c376edbd 100644 --- a/src/output-file.c +++ b/src/output-file.c @@ -40,7 +40,7 @@ bool g_file_logger_enabled = false; /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputFileLogger_ { SCFileLogger LogFunc; void *initdata; diff --git a/src/output-filedata.c b/src/output-filedata.c index 1e07902fdc..1bdd161b84 100644 --- a/src/output-filedata.c +++ b/src/output-filedata.c @@ -38,7 +38,7 @@ bool g_filedata_logger_enabled = false; /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputFiledataLogger_ { SCFiledataLogger LogFunc; void *initdata; diff --git a/src/output-flow.c b/src/output-flow.c index 3f9cbd318e..a2562637fd 100644 --- a/src/output-flow.c +++ b/src/output-flow.c @@ -37,7 +37,7 @@ typedef struct OutputFlowLoggerThreadData_ { /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputFlowLogger_ { FlowLogger LogFunc; diff --git a/src/output-stats.c b/src/output-stats.c index 00fe3aaae7..76eed1549f 100644 --- a/src/output-stats.c +++ b/src/output-stats.c @@ -36,7 +36,7 @@ typedef struct OutputStatsLoggerThreadData_ { /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputStatsLogger_ { StatsLogger LogFunc; OutputCtx *output_ctx; diff --git a/src/output-streaming.c b/src/output-streaming.c index 51906dc9dd..2adbd77ca4 100644 --- a/src/output-streaming.c +++ b/src/output-streaming.c @@ -46,7 +46,7 @@ typedef struct OutputStreamingLoggerThreadData_ { /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputStreamingLogger_ { SCStreamingLogger LogFunc; void *initdata; diff --git a/src/output-tx.c b/src/output-tx.c index 7a1a6ca234..829e6b4f0d 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -45,7 +45,7 @@ typedef struct OutputTxLoggerThreadData_ { /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same - * log module (e.g. http.log) with different output ctx'. */ + * log module (e.g. fast.log) with different output ctx'. */ typedef struct OutputTxLogger_ { AppProto alproto; TxLogger LogFunc; diff --git a/src/output.c b/src/output.c index f512de031b..ed38790756 100644 --- a/src/output.c +++ b/src/output.c @@ -52,7 +52,6 @@ #include "log-cf-common.h" #include "output-json-drop.h" #include "output-eve-stream.h" -#include "log-httplog.h" #include "output-json-http.h" #include "output-json-dns.h" #include "output-json-mdns.h" @@ -1089,7 +1088,6 @@ void OutputRegisterLoggers(void) /* email logs */ JsonSmtpLogRegister(); /* http log */ - LogHttpLogRegister(); JsonHttpLogRegister(); OutputRegisterTxSubModuleWithProgress(LOGGER_JSON_TX, "eve-log", "LogHttp2Log", "eve-log.http2", OutputJsonLogInitSub, ALPROTO_HTTP2, JsonGenericDirFlowLogger, HTTP2StateClosed, diff --git a/src/runmode-af-packet.c b/src/runmode-af-packet.c index d72b33e368..8fbce6d964 100644 --- a/src/runmode-af-packet.c +++ b/src/runmode-af-packet.c @@ -37,7 +37,6 @@ #include "runmodes.h" #include "runmode-af-packet.h" #include "output.h" -#include "log-httplog.h" #include "detect-engine-mpm.h" #include "alert-fastlog.h" diff --git a/src/runmode-af-xdp.c b/src/runmode-af-xdp.c index 606921c3a0..017b1d782f 100644 --- a/src/runmode-af-xdp.c +++ b/src/runmode-af-xdp.c @@ -36,7 +36,6 @@ #include "runmodes.h" #include "runmode-af-xdp.h" #include "output.h" -#include "log-httplog.h" #include "detect-engine-mpm.h" #include "alert-fastlog.h" diff --git a/src/suricata-common.h b/src/suricata-common.h index 29987beb02..39becde648 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -486,7 +486,6 @@ typedef enum LoggerId { LOGGER_UNDEFINED, /* TX loggers first for low logger IDs */ - LOGGER_HTTP, LOGGER_TLS_STORE, LOGGER_TLS_STORE_CLIENT, LOGGER_TLS, diff --git a/src/tests/fuzz/confyaml.c b/src/tests/fuzz/confyaml.c index a34c9632ef..0a7b80ddbb 100644 --- a/src/tests/fuzz/confyaml.c +++ b/src/tests/fuzz/confyaml.c @@ -70,10 +70,6 @@ outputs:\n\ - flow\n\ - netflow\n\ - metadata\n\ - - http-log:\n\ - enabled: yes\n\ - filename: /dev/null\n\ - extended: yes\n\ - tls-log:\n\ enabled: yes\n\ filename: /dev/null\n\ diff --git a/src/util-lua-flowlib.c b/src/util-lua-flowlib.c index e39a1575f8..49887c2c7e 100644 --- a/src/util-lua-flowlib.c +++ b/src/util-lua-flowlib.c @@ -178,7 +178,7 @@ static int LuaFlowTimestringIso8601(lua_State *luastate) } /** \internal - * \brief legacy format as used by fast.log, http.log, etc. + * \brief legacy format as used by fast.log, etc. */ static int LuaFlowTimestringLegacy(lua_State *luastate) { diff --git a/src/util-lua-packetlib.c b/src/util-lua-packetlib.c index 6ba04910a7..32780147e5 100644 --- a/src/util-lua-packetlib.c +++ b/src/util-lua-packetlib.c @@ -87,7 +87,7 @@ static int LuaPacketPcapCnt(lua_State *luastate) } /** \internal - * \brief legacy format as used by fast.log, http.log, etc. + * \brief legacy format as used by fast.log, etc. */ static int LuaPacketTimestringLegacy(lua_State *luastate) { diff --git a/src/util-profiling.c b/src/util-profiling.c index 6af51fcc23..b6ec465495 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -1266,7 +1266,6 @@ const char *PacketProfileLoggerIdToString(LoggerId id) { switch (id) { CASE_CODE(LOGGER_UNDEFINED); - CASE_CODE(LOGGER_HTTP); CASE_CODE(LOGGER_TLS_STORE); CASE_CODE(LOGGER_TLS_STORE_CLIENT); CASE_CODE(LOGGER_TLS); diff --git a/src/util-runmodes.c b/src/util-runmodes.c index 4336687b06..c01d476a8a 100644 --- a/src/util-runmodes.c +++ b/src/util-runmodes.c @@ -30,7 +30,6 @@ #include "runmodes.h" #include "runmode-af-packet.h" #include "output.h" -#include "log-httplog.h" #include "detect-engine.h" #include "detect-engine-mpm.h"