diff --git a/doc/sphinx/.gitignore b/doc/sphinx/.gitignore new file mode 100644 index 0000000000..8b77e79c66 --- /dev/null +++ b/doc/sphinx/.gitignore @@ -0,0 +1,2 @@ +!Makefile +_build diff --git a/doc/sphinx/Makefile b/doc/sphinx/Makefile new file mode 100644 index 0000000000..12d28b7a11 --- /dev/null +++ b/doc/sphinx/Makefile @@ -0,0 +1,192 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Suricata.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Suricata.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Suricata" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Suricata" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/sphinx/_static/.gitignore b/doc/sphinx/_static/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py new file mode 100644 index 0000000000..a5af0169ea --- /dev/null +++ b/doc/sphinx/conf.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +# +# Suricata documentation build configuration file, created by +# sphinx-quickstart on Fri Nov 6 10:12:25 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Suricata' +copyright = u'2015, OISF' +author = u'OISF' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '2.1dev' +# The full version, including alpha/beta/rc tags. +release = '3.0dev' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Suricatadoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'Suricata.tex', u'Suricata Documentation', + u'OISF', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'suricata', u'Suricata Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Suricata', u'Suricata Documentation', + author, 'Suricata', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/doc/sphinx/fast-pattern.rst b/doc/sphinx/fast-pattern.rst new file mode 100644 index 0000000000..17be899879 --- /dev/null +++ b/doc/sphinx/fast-pattern.rst @@ -0,0 +1,4 @@ +Fast Pattern +============ + +Just a place holder now to demontrate linking. diff --git a/doc/sphinx/http-keywords.rst b/doc/sphinx/http-keywords.rst new file mode 100644 index 0000000000..e4363c79ce --- /dev/null +++ b/doc/sphinx/http-keywords.rst @@ -0,0 +1,363 @@ +:tocdepth: 2 + +HTTP Keywords +============= + +There are additional content modifiers that can provide +protocol-specific capabilities at the application layer (if you are +unfamiliar with content modifiers, please visit the page [[Payload +keywords]]). These keywords make sure the signature checks only +specific parts of the network traffic. For instance, to check +specifically on the request URI, cookies, or the HTTP request or +response body, etc. + + +Use ``http_method`` to match on the HTTP request method, ``http_uri`` +or ``http_raw_uri`` to match on the request URI, ``http_stat_code`` to +match on the response status code and ``http_stat_msg`` to match on the +response status message. + +It is important to understand the structure of HTTP requests and +responses. A simple example of a HTTP request and response follows: + +HTTP request +------------ + +:: + + GET /index.html HTTP/1.0\r\n + +GET is a request **method**. Examples of methods are: GET, POST, PUT, +HEAD, etc. The URI path is ``/index.html`` and the HTTP version is +``HTTP/1.0``. Several HTTP versions have been used over the years; of +the versions 0.9, 1.0 and 1.1, 1.0 and 1.1 are the most commonly used +today. + +HTTP response +------------- + +:: + + HTTP/1.0 200 OK\r\n + + some page + + +In this example, HTTP/1.0 is the HTTP version, 200 the response status +code and OK the response status message. + +Another more detailed example: + +Request: + +.. image:: http-keywords/request.png + +Response: + +.. image:: http-keywords/response1.png + +Request: + +.. image:: http-keywords/request2.png + +Although cookies are sent in an HTTP header, you can not match on them +with the ``http_header`` keyword. Cookies are matched with their own +keyword, namely ``http_cookie``. + +Each part of the table belongs to a so-called *buffer*. The HTTP +method belongs to the method buffer, HTTP headers to the header buffer +etc. A buffer is a specific portion of the request or response that +Suricata extracts in memory for inspection. + +All previous described keywords can be used in combination with a +buffer in a signature. The keywords ``distance`` and ``within`` are +relative modifiers, so they may only be used within the same +buffer. You can not relate content matches against different buffers +with relative modifiers. + +http_method +----------- + +With the ``http_method`` content modifier, it is possible to match +specifically and only on the HTTP method buffer. The keyword can be +used in combination with all previously mentioned content modifiers +such as: ``depth``, ``distance``, ``offset``, ``nocase`` and ``within``. + +Methods are: **GET**, **POST**, **PUT**, **HEAD**, **DELETE**, **TRACE**, +**OPTIONS**, **CONNECT** and **PATCH**. + +Example of a method in a HTTP request: + +.. image:: http-keywords/method2.png + +Example of the purpose of method: + +.. image:: http-keywords/method.png + +.. image:: http-keywords/Legenda_rules.png + +.. image:: http-keywords/method1.png + + +http_uri and http_raw_uri +------------------------- + +With the ``http_uri`` and the ``http_raw_uri`` content modifiers, it +is possible to match specifically and only on the request URI +buffer. The keyword can be used in combination with all previously +mentioned content modifiers like ``depth``, ``distance``, ``offset``, +``nocase`` and ``within``. + +To learn more about the difference between ``http_uri`` and +``http_raw_uri``, please read the information about [[HTTP-uri +normalization]]. + +Example of the URI in a HTTP request: + +.. image:: http-keywords/uri1.png + +Example of the purpose of ``http_uri``: + +.. image:: http-keywords/uri.png + +Example of the purpose of ``http_raw_uri``: + +#.. image:: http-keywords/raw_uri.png + +uricontent +---------- + +The ``uricontent`` keyword has the exact same effect as the +``http_uri`` content modifier. ``uricontent`` is a deprecated +(although still supported) way to match specifically and only on the +request URI buffer. + +Example of ``uricontent``: + +.. image:: http-keywords/uricontent.png + +The difference between ``http_uri`` and ``uricontent`` is the syntax: + +.. image:: http-keywords/uricontent1.png + +.. image:: http-keywords/http_uri.png + +When authoring new rules, it is recommended that the ``http_uri`` +content modifier be used rather than the deprecated ``uricontent`` +keyword. + +http_header and http_raw_header +------------------------------- + +With the ``http_header`` content modifier, it is possible to match +specifically and only on the HTTP header buffer. This contains all of +the extracted headers in a single buffer, except for those indicated +in the documentation that are not able to match by this buffer and +have their own content modifier (e.g. ``http_cookie``). The modifier +can be used in combination with all previously mentioned content +modifiers, like ``depth``, ``distance``, ``offset``, ``nocase`` and +``within``. + + **Note**: the header buffer is *normalized*. Any trailing + whitespace and tab characters are removed. See: + http://lists.openinfosecfoundation.org/pipermail/oisf-users/2011-October/000935.html. Match + on the ``http_raw_header`` buffer if you require the raw + (non-normalized) headers. + +Example of a header in a HTTP request: + +.. image:: http-keywords/header.png + +Example of the purpose of ``http_header``: + +.. image:: http-keywords/header1.png + +http_cookie +----------- + +With the ``http_cookie`` content modifier, it is possible to match +specifically and only on the cookie buffer. The keyword can be used in +combination with all previously mentioned content modifiers like +``depth``, ``distance``, ``offset``, ``nocase`` and ``within``. + +Note that cookies are passed in HTTP headers, but are extracted to a +dedicated buffer and matched using their own specific content +modifier. + +Example of a cookie in a HTTP request: + +.. image:: http-keywords/cookie.png + +Example of the purpose of ``http_cookie``: + +.. image:: http-keywords/cookie1.png + +http_user_agent +--------------- + +The ``http_user_agent`` content modifier is part of the HTTP request +header. It makes it possible to match specifically on the value of the +User-Agent header. It is normalized in the sense that it does not +include the _"User-Agent: "_ header name and separator, nor does it +contain the trailing carriage return and line feed (CRLF). The keyword +can be used in combination with all previously mentioned content +modifiers like ``depth``, ``distance``, ``offset``, ``nocase`` and +``within``. Note that the ``pcre`` keyword can also inspect this +buffer when using the ``/V`` modifier. + +An analysis into the performance of ``http_user_agent`` +vs. ``http_header`` is found at: +http://blog.inliniac.net/2012/07/09/suricata-http_user_agent-vs-http_header/ + +Normalization: leading spaces **are not** part of this buffer. So +"User-Agent: \r\n" will result in an empty ``http_user_agent`` buffer. + +Example of the User-Agent header in a HTTP request: + +.. image:: http-keywords/user_agent.png + +Example of the purpose of ``http_user_agent``: + +.. image:: http-keywords/user_agent_match.png + +http_client_body +---------------- + +With the ``http_client_body`` content modifier, it is possible to +match specifically and only on the HTTP request body. The keyword can +be used in combination with all previously mentioned content modifiers +like ``distance``, ``offset``, ``nocase``, ``within``, etc. + +Example of ``http_client_body`` in a HTTP request: + +.. image:: http-keywords/client_body.png + +Example of the purpose of ``http_client_body``: + +.. image:: http-keywords/client_body1.png + +Note: how much of the request/client body is inspected is controlled +in the [[suricata.yaml#Configure-Libhtp]], in the "libhtp" section, +via the ``request-body-limit`` setting. + +http_stat_code +-------------- + +With the ``http_stat_code`` content modifier, it is possible to match +specifically and only on the HTTP status code buffer. The keyword can +be used in combination with all previously mentioned content modifiers +like ``distance``, ``offset``, ``nocase``, ``within``, etc. + +Example of ``http_stat_code`` in a HTTP response: + +.. image:: http-keywords/stat_code.png + +Example of the purpose of ``http_stat_code``: + +.. image:: http-keywords/stat-code1.png + +http_stat_msg +------------- + +With the ``http_stat_msg`` content modifier, it is possible to match +specifically and only on the HTTP status message buffer. The keyword +can be used in combination with all previously mentioned content +modifiers like ``depth``, ``distance``, ``offset``, ``nocase`` and +``within``. + +Example of ``http_stat_msg`` in a HTTP response: + +.. image:: http-keywords/stat_msg.png + +Example of the purpose of ``http_stat_msg``: + +.. image:: http-keywords/stat_msg_1.png + +http_server_body +---------------- + +With the ``http_server_body`` content modifier, it is possible to +match specifically and only on the HTTP response body. The keyword can +be used in combination with all previously mentioned content modifiers +like ``distance``, ``offset``, ``nocase``, ``within``, etc. If the +response body is *gzip* encoded, it is first uncompressed for +inspection. + +Note: how much of the response/server body is inspected is controlled +in your [[suricata.yaml#Configure-Libhtp]], in the "libhtp" section, +via the ``response-body-limit`` setting. + +file_data +--------- + +With ``file_data``, the HTTP response body is inspected, just like +with ``http_server_body``. The ``file_data`` keyword works a bit +differently from the normal content modifiers; when used in a rule, +all content matches following it in the rule are affected (modified) +by it. + +Example:: + + alert http any any -> any any (file_data; content:"abc"; content:"xyz";) + +.. image:: http-keywords/file_data.png + +The ``file_data`` keyword affects all following content matches, until +the ``pkt_data`` keyword is encountered or it reaches the end of the +rule. This makes it a useful shortcut for applying many content +matches to the HTTP response body, eliminating the need to modify each +content match individually. If the response body is *gzip* encoded, it +is first uncompressed for inspection. + +Note: how much of the response/server body is inspected is controlled +in your [[suricata.yaml#Configure-Libhtp]], in the "libhtp" section +via the ``response-body-limit`` setting. + +**NOTE:** In 2.0.x, ``file_data`` is only supported for HTTP server +bodies (specified as flow direction **to_client**). Starting with +2.1-dev and above, ``file_data`` is handled more intelligently +per-protocol, supported with **to_server** for SMTP and **to_client** +for HTTP. + +urilen +------ + +The ``urilen`` keyword is used to match on the length of the request +URI. It is possible to use the ``<`` and ``>`` operators, which +indicate respectively *smaller than* and *larger than*. + +The format of ``urilen`` is:: + + urilen:3; + +Other possibilities are:: + + urilen:1; + urilen:>1; + urilen:<10; + urilen:10<>20; (bigger than 10, smaller than 20) + +Example: + +.. image:: http-keywords/urilen.png + +Example of ``urilen`` in a signature: + +.. image:: http-keywords/urilen1.png + +You can specify whether the inspected URI buffer is either the +normalized or the raw buffer by appending ``norm`` or ``raw``:: + + urilen:<5,raw; + +pcre +---- + +For information about the ``pcre`` keyword, check the [[pcre (Perl +Compatible Regular Expressions)]] page. + +fast_pattern +------------ + +For information about the ``fast_pattern`` keyword, check the +:doc:`fast-pattern` page. diff --git a/doc/sphinx/http-keywords/Legenda_rules.png b/doc/sphinx/http-keywords/Legenda_rules.png new file mode 100644 index 0000000000..c3e913340f Binary files /dev/null and b/doc/sphinx/http-keywords/Legenda_rules.png differ diff --git a/doc/sphinx/http-keywords/client_body.png b/doc/sphinx/http-keywords/client_body.png new file mode 100644 index 0000000000..d4ea6dba47 Binary files /dev/null and b/doc/sphinx/http-keywords/client_body.png differ diff --git a/doc/sphinx/http-keywords/client_body1.png b/doc/sphinx/http-keywords/client_body1.png new file mode 100644 index 0000000000..5b9a749632 Binary files /dev/null and b/doc/sphinx/http-keywords/client_body1.png differ diff --git a/doc/sphinx/http-keywords/cookie.png b/doc/sphinx/http-keywords/cookie.png new file mode 100644 index 0000000000..8e5c26262e Binary files /dev/null and b/doc/sphinx/http-keywords/cookie.png differ diff --git a/doc/sphinx/http-keywords/cookie1.png b/doc/sphinx/http-keywords/cookie1.png new file mode 100644 index 0000000000..12293c3f41 Binary files /dev/null and b/doc/sphinx/http-keywords/cookie1.png differ diff --git a/doc/sphinx/http-keywords/fast_pattern.png b/doc/sphinx/http-keywords/fast_pattern.png new file mode 100644 index 0000000000..97163a50ab Binary files /dev/null and b/doc/sphinx/http-keywords/fast_pattern.png differ diff --git a/doc/sphinx/http-keywords/file_data.png b/doc/sphinx/http-keywords/file_data.png new file mode 100644 index 0000000000..d604c0738e Binary files /dev/null and b/doc/sphinx/http-keywords/file_data.png differ diff --git a/doc/sphinx/http-keywords/header.png b/doc/sphinx/http-keywords/header.png new file mode 100644 index 0000000000..d78714c276 Binary files /dev/null and b/doc/sphinx/http-keywords/header.png differ diff --git a/doc/sphinx/http-keywords/header1.png b/doc/sphinx/http-keywords/header1.png new file mode 100644 index 0000000000..a255a0e61c Binary files /dev/null and b/doc/sphinx/http-keywords/header1.png differ diff --git a/doc/sphinx/http-keywords/http_server_body.png b/doc/sphinx/http-keywords/http_server_body.png new file mode 100644 index 0000000000..8fc84e1235 Binary files /dev/null and b/doc/sphinx/http-keywords/http_server_body.png differ diff --git a/doc/sphinx/http-keywords/http_uri.png b/doc/sphinx/http-keywords/http_uri.png new file mode 100644 index 0000000000..60a678609a Binary files /dev/null and b/doc/sphinx/http-keywords/http_uri.png differ diff --git a/doc/sphinx/http-keywords/method.png b/doc/sphinx/http-keywords/method.png new file mode 100644 index 0000000000..b718c8b7c2 Binary files /dev/null and b/doc/sphinx/http-keywords/method.png differ diff --git a/doc/sphinx/http-keywords/method1.png b/doc/sphinx/http-keywords/method1.png new file mode 100644 index 0000000000..70d8c78f9e Binary files /dev/null and b/doc/sphinx/http-keywords/method1.png differ diff --git a/doc/sphinx/http-keywords/method2.png b/doc/sphinx/http-keywords/method2.png new file mode 100644 index 0000000000..1084618167 Binary files /dev/null and b/doc/sphinx/http-keywords/method2.png differ diff --git a/doc/sphinx/http-keywords/request.png b/doc/sphinx/http-keywords/request.png new file mode 100644 index 0000000000..7298234e93 Binary files /dev/null and b/doc/sphinx/http-keywords/request.png differ diff --git a/doc/sphinx/http-keywords/request2.png b/doc/sphinx/http-keywords/request2.png new file mode 100644 index 0000000000..b59b00c249 Binary files /dev/null and b/doc/sphinx/http-keywords/request2.png differ diff --git a/doc/sphinx/http-keywords/response1.png b/doc/sphinx/http-keywords/response1.png new file mode 100644 index 0000000000..40430c9e1f Binary files /dev/null and b/doc/sphinx/http-keywords/response1.png differ diff --git a/doc/sphinx/http-keywords/stat-code1.png b/doc/sphinx/http-keywords/stat-code1.png new file mode 100644 index 0000000000..77758ee6c9 Binary files /dev/null and b/doc/sphinx/http-keywords/stat-code1.png differ diff --git a/doc/sphinx/http-keywords/stat_code.png b/doc/sphinx/http-keywords/stat_code.png new file mode 100644 index 0000000000..a865fe10bd Binary files /dev/null and b/doc/sphinx/http-keywords/stat_code.png differ diff --git a/doc/sphinx/http-keywords/stat_msg.png b/doc/sphinx/http-keywords/stat_msg.png new file mode 100644 index 0000000000..f6f8ac8c76 Binary files /dev/null and b/doc/sphinx/http-keywords/stat_msg.png differ diff --git a/doc/sphinx/http-keywords/stat_msg_1.png b/doc/sphinx/http-keywords/stat_msg_1.png new file mode 100644 index 0000000000..f26a075b65 Binary files /dev/null and b/doc/sphinx/http-keywords/stat_msg_1.png differ diff --git a/doc/sphinx/http-keywords/uri.png b/doc/sphinx/http-keywords/uri.png new file mode 100644 index 0000000000..e43b4f792b Binary files /dev/null and b/doc/sphinx/http-keywords/uri.png differ diff --git a/doc/sphinx/http-keywords/uri1.png b/doc/sphinx/http-keywords/uri1.png new file mode 100644 index 0000000000..d9fab6c14d Binary files /dev/null and b/doc/sphinx/http-keywords/uri1.png differ diff --git a/doc/sphinx/http-keywords/uricontent.png b/doc/sphinx/http-keywords/uricontent.png new file mode 100644 index 0000000000..b9e8aad274 Binary files /dev/null and b/doc/sphinx/http-keywords/uricontent.png differ diff --git a/doc/sphinx/http-keywords/uricontent1.png b/doc/sphinx/http-keywords/uricontent1.png new file mode 100644 index 0000000000..fa0cfac200 Binary files /dev/null and b/doc/sphinx/http-keywords/uricontent1.png differ diff --git a/doc/sphinx/http-keywords/urilen.png b/doc/sphinx/http-keywords/urilen.png new file mode 100644 index 0000000000..1697db1ca8 Binary files /dev/null and b/doc/sphinx/http-keywords/urilen.png differ diff --git a/doc/sphinx/http-keywords/urilen1.png b/doc/sphinx/http-keywords/urilen1.png new file mode 100644 index 0000000000..8a11075fca Binary files /dev/null and b/doc/sphinx/http-keywords/urilen1.png differ diff --git a/doc/sphinx/http-keywords/user_agent.png b/doc/sphinx/http-keywords/user_agent.png new file mode 100644 index 0000000000..f4fa2e835e Binary files /dev/null and b/doc/sphinx/http-keywords/user_agent.png differ diff --git a/doc/sphinx/http-keywords/user_agent_match.png b/doc/sphinx/http-keywords/user_agent_match.png new file mode 100644 index 0000000000..d99d85be0b Binary files /dev/null and b/doc/sphinx/http-keywords/user_agent_match.png differ diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst new file mode 100644 index 0000000000..e67347d17e --- /dev/null +++ b/doc/sphinx/index.rst @@ -0,0 +1,20 @@ +.. Suricata documentation master file, created by + sphinx-quickstart on Fri Nov 6 10:12:25 2015. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Suricata User Guide +=================== + +.. toctree:: + :maxdepth: 3 + + what-is-suricata + rules + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/sphinx/meta.rst b/doc/sphinx/meta.rst new file mode 100644 index 0000000000..6b5fbf5a71 --- /dev/null +++ b/doc/sphinx/meta.rst @@ -0,0 +1,168 @@ +Meta-settings +============= + +Meta-settings have no effect on Suricata's inspection; they have an +effect on the way Suricata reports events. + +msg (message) +------------- + +The keyword msg gives more information about the signature and the +possible alert. The first part shows the filename of the +signature. It is a convention that part is written in uppercase +characters. + +The format of msg is:: + + msg: “...”; + +Example:: + + msg:"ATTACK-RESPONSES 403 Forbidden"; + msg:"ET EXPLOIT SMB-DS DCERPC PnP bind attempt"; + +*It is a convention that msg is always the first keyword of a signature.* + +Another example of msg in a signature: + + alert tcp $HOME_NET any -> $EXTERNAL_NET any (**msg:"ET TROJAN Likely Bot Nick in IRC (USA +..)";** flow:established,to_server; content:"NICK "; depth:5; content:"USA"; within:10; reference:url,doc.emergingthreats.net/2008124; classtype:trojan-activity; sid:2008124; rev:5;) + +In this example the red, bold-faced part is the msg. + +Sid (signature id) +------------------ + +The keyword sid gives every signature its own id. This id is stated +with a number. + +The format of sid is:: + + sid:123; + +Example of sid in a signature: + + alert tcp $HOME_NET any -> $EXTERNAL_NET any (msg:"ET TROJAN Likely Bot Nick in IRC (USA +..)"; flow:established,to_server; content:"NICK "; depth:5; content:"USA"; within:10; reference:url,doc.emergingthreats.net/2008124; classtype:trojan-activity; **sid:2008124;** rev:5;) + +Rev (Revision) +-------------- + +The sid keyword is almost every time accompanied by rev. Rev +represents the version of the signature. If a signature is modified, +the number of rev will be incremented by the signature writers. + +The format of rev is:: + + rev:123; + +*It is a convention that sid comes before rev, and both are the last of +all keywords.* + +Example of rev in a signature: + + alert tcp $HOME_NET any -> $EXTERNAL_NET any (msg:"ET TROJAN Likely Bot Nick in IRC (USA +..)"; flow:established,to_server; content:"NICK "; depth:5; content:"USA"; within:10; reference:url,doc.emergingthreats.net/2008124; classtype:trojan-activity; sid:2008124; **rev:5;**) + +Gid (group id) +-------------- + +The gid keyword can be used to give different groups of signatures +another id value (like in sid). Suricata uses by default gid 1. It is +possible to modify this. It is not usual that it will be changed, and +changing it has no technical implications. You can only notice it in +the alert. + + 10/01/2014-05:14:43.926704 [**] [**1**:2016903:5] ET USER_AGENTS Suspicious User-Agent (DownloadMR) [**] [Classification: A Network Trojan was detected] [Priority: 1] {TCP} 192.168.81.10:1032 -> 95.211.39.161:80 + +This is an example from the fast.log. In the part [1:2008124:2], 1 is +the gid (2008124 is the the sid and 2 the rev). + +Classtype +--------- + +The classtype keyword gives information about the classification of +rules and alerts. It consists of a short name, a long name and a +priority. It can tell for example whether a rule is just informational +or is about a hack etcetera. For each classtype, the +classification.config has a priority which will be used in the rule. + +*It is a convention that classtype comes before sid and rev and after +the rest of the keywords.* + +Example classtype:: + + config classification: web-application-attack,Web Application Attack,1 + config classification: not-suspicious,Not Suspicious Traffic,3 + +============== =================================== ====================== +Signature classification.config Alert +============== =================================== ====================== +web-attack web-attack, Web Application Attack, Web Application Attack + priority:1 +not-suspicious not-suspicious, Not Suspiscious Not Suspicious Traffic + Traffic, priority:3 +============== =================================== ====================== + +In the above table you see how classtype appears in signatures, the +classification.config and the alert. + +Another example of classtype in a signature: + + alert tcp $HOME_NET any -> $EXTERNAL_NET any (msg:"ET TROJAN Likely Bot Nick in IRC (USA +..)" flow:established,to_server; content:"NICK "; depth:5; content:"USA"; within:10; reference:url,doc.emergingthreats.net/2008124; **classtype:trojan-activity;** sid:2008124; rev:5;) + +Reference +--------- + +The reference keywords direct to places where information about the +signature and about the problem the signature tries to address, can be +found. The reference keyword can appear multiple times in a signature. +This keyword is meant for signature-writers and analysts who +investigate why a signature has matched. It has the following format:: + + reference: url, www.info.nl + +In this example url is the type of reference. After that comes the +actual reference (notice here you can not use http before the url). + +There are different types of references: + +========= =============================================== +system URL Prefix +========= =============================================== +bugtraq ``http://www.securityfocus.com/bid`` +cve ``http://cve.mitre.org/cgi-bin/cvename.cgi?name=`` +nessus ``http://cgi.nessus.org/plugins/dump.php3?id=`` +arachnids ``http://www.whitehats.com/info/IDS`` +mcafee ``http://vil.nai.com/vil/dispVirus.asp?virus_k=`` +url ``http://`` +========= =============================================== + +*Note that ararchnids is no longer available but may still be +encountered in signatures.* + +For example bugtraq will be replaced by the full url:: + + reference: bugtraq, 123; http://www.securityfocus.com/bid + +Example of reference in a signature: + + alert tcp $HOME_NET any -> $EXTERNAL_NET any (msg:"ET TROJAN Likely Bot Nick in IRC (USA +..)" flow:established,to_server; content:"NICK "; depth:5; content:"USA"; within:10; **reference:url,doc.emergingthreats.net/2008124;** classtype:trojan-activity; sid:2008124; rev:5;) + +Priority +-------- + +The priority keyword comes with a mandatory numeric value which can +range from 1 till 255. The numbers 1 to 4 are most often used. +Signatures with a higher priority will be examined first. The highest +priority is 1. Normally signatures have already a priority through +class type. This can be overruled with the keyword priority. The +format of priority is:: + + priority:1; + +Metadata +-------- + +Suricata ignores the words behind meta data. +Suricata supports this keyword because it is part of the signature language. +The format is:: + + metadata:...; diff --git a/doc/sphinx/payload-keywords.rst b/doc/sphinx/payload-keywords.rst new file mode 100644 index 0000000000..6481766d6c --- /dev/null +++ b/doc/sphinx/payload-keywords.rst @@ -0,0 +1,7 @@ +Payload Keywords +================ + +.. toctree:: + :maxdepth: 2 + + fast-pattern diff --git a/doc/sphinx/rules.rst b/doc/sphinx/rules.rst new file mode 100644 index 0000000000..59d2a432ae --- /dev/null +++ b/doc/sphinx/rules.rst @@ -0,0 +1,10 @@ +Rules +===== + + +.. toctree:: + :maxdepth: 2 + + meta + http-keywords + payload-keywords diff --git a/doc/sphinx/what-is-suricata.rst b/doc/sphinx/what-is-suricata.rst new file mode 100644 index 0000000000..6ae7ba1fd8 --- /dev/null +++ b/doc/sphinx/what-is-suricata.rst @@ -0,0 +1,68 @@ +What is Suricata +================ + +The Suricata Engine is an Open Source Next Generation Intrusion +Detection and Prevention Engine. This engine is not intended to just +replace or emulate the existing tools in the industry, but will bring +new ideas and technologies to the field. The Suricata Engine and the +HTP Library are available to use under the GPLv2. + + +IDS/IPS +------- + +Suricata is a rule-based ID/PS engine that utilises externally +developed rule sets to monitor network traffic and provide alerts to +the system administrator when suspicious events occur. Designed to be +compatible with existing network security components, Suricata +features unified output functionality and pluggable library options to +accept calls from other applications. The initial release of Suricata +runs on a Linux 2.6 platform that supports inline and passive traffic +monitoring configuration capable of handling multiple gigabit traffic +levels. Linux 2.4 is supported with reduced configuration +functionality, such as no inline option. Available under Version 2 of +the General Public License, Suricata eliminates the ID/PS engine cost +concerns while providing a scalable option for the most complex +network security architectures. + + +Multi-threading +--------------- + +As a multi-threaded engine, Suricata offers increased speed and +efficiency in network traffic analysis. In addition to hardware +acceleration (with hardware and network card limitations), the engine +is build to utilise the increased processing power offered by the +latest multi-core CPU chip sets. Suricata is developed for ease of +implementation and accompanied by a step-by-step getting started +documentation and user manual. + +Development and features +------------------------ + +The goal of the Suricata Project Phase 1 was to have a distributable +and functional ID/PS engine. The initial beta release was made +available for download on January 1, 2010. The engine supports or +provides the following functionality: the latest Snort VRT, Snort +logging, rule language options, multi-threading, hardware acceleration +(with hardware and network card dependencies/limitations), unified +output enabling interaction with external log management systems, +IPv6, rule-based IP reputation, library plug-ability for interaction +with other applications, performance statistics output, and a simple +and effective getting started user manual. + +By engaging the open source community and the leading ID/PS rule set +resources available, OISF has built the Suricata engine to simplify +the process of maintaining optimum security levels. Through strategic +partnerships, OISF is leveraging the expertise of Emerging Threats +(www.emergingthreats.net) and other prominent resources in the +industry to provide the most current and comprehensive rule sets +available. + +The HTP Library is an HTTP normaliser and parser written by Ivan +Ristic of Mod Security fame for the OISF. This integrates and provides +very advanced processing of HTTP streams for Suricata. The HTP library +is required by the engine, but may also be used independently in a +range of applications and tools. + +[[Major Features|Next]]