mirror of https://github.com/OISF/suricata
prscript: read buildbot config from source
This patch change the logic of the Docker buildbot system. The buildbot configuration is now the one available in qa/docker directory. This way, developers can test features in docker buildbot that could require some specific flags to be set. They just need to edit the buildbot.cfg for instance to enable a new feature. In the same way, the tested pcap files are the one which are in the qa/docker/pcaps/ directory. So to test some private ones it is enough to put them in that directory. To take into consideration a buildbot.cfg modification or a new pcap, it is enough to stop and restart the container: sudo qa/prscript.py -l -d -S master sudo qa/prscript.py -l -d -s master This patch also fixes the container update issue. A local modification to the buildbot will be kept. It is also fixing the issue when working on old code that could possibly not support the same build flag as the one of buildbot. Here the configuration will remains in sync.pull/1405/head
Eric Leblond
10 years ago
committed by
Victor Julien
parent
be473fa712
commit
f12e6fdcda
@ -0,0 +1,235 @@
|
||||
# -*- python -*-
|
||||
# ex: set syntax=python:
|
||||
|
||||
# This is a sample buildmaster config file. It must be installed as
|
||||
# 'master.cfg' in your buildmaster's base directory.
|
||||
|
||||
# This is the dictionary that the buildmaster pays attention to. We also use
|
||||
# a shorter alias to save typing.
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
####### BUILDSLAVES
|
||||
|
||||
# The 'slaves' list defines the set of recognized buildslaves. Each element is
|
||||
# a BuildSlave object, specifying a unique slave name and password. The same
|
||||
# slave name and password must be configured on the slave.
|
||||
from buildbot.buildslave import BuildSlave
|
||||
c['slaves'] = [BuildSlave("buildslave", "Suridocker")]
|
||||
|
||||
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
|
||||
# This must match the value configured into the buildslaves (with their
|
||||
# --master option)
|
||||
c['slavePortnum'] = 9989
|
||||
|
||||
####### CHANGESOURCES
|
||||
|
||||
# the 'change_source' setting tells the buildmaster how it should find out
|
||||
# about source code changes. Here we point to the buildbot clone of pyflakes.
|
||||
|
||||
from buildbot.changes.gitpoller import GitPoller
|
||||
c['change_source'] = []
|
||||
c['change_source'].append(GitPoller(
|
||||
'/data/oisf/.git/',
|
||||
workdir='gitpoller-workdir', branches = ['master'],
|
||||
pollinterval=300, project='suricata'))
|
||||
|
||||
####### SCHEDULERS
|
||||
|
||||
# Configure the Schedulers, which decide how to react to incoming changes. In this
|
||||
# case, just kick off a 'runtests' build
|
||||
|
||||
from buildbot.schedulers.basic import SingleBranchScheduler
|
||||
#from buildbot.schedulers.forcesched import ForceScheduler
|
||||
from buildbot.changes import filter
|
||||
c['schedulers'] = []
|
||||
c['schedulers'].append(SingleBranchScheduler(
|
||||
name="master",
|
||||
change_filter=filter.ChangeFilter(branch='master'),
|
||||
treeStableTimer=None,
|
||||
builderNames=["features","profiling","clang"]))
|
||||
|
||||
#c['schedulers'].append(ForceScheduler(
|
||||
# name="force",
|
||||
# builderNames=["builds","debug"]))
|
||||
|
||||
####### BUILDERS
|
||||
|
||||
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
|
||||
# what steps, and which slaves can execute them. Note that any particular build will
|
||||
# only take place on one slave.
|
||||
|
||||
from buildbot.process.factory import BuildFactory
|
||||
#from buildbot.steps.source.git import Git
|
||||
from buildbot.steps.source import Git
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
|
||||
def SuriBuildFactory(repo='/data/oisf/'):
|
||||
factory = BuildFactory()
|
||||
factory.addStep(Git(repourl=repo, mode='copy'))
|
||||
factory.addStep(ShellCommand(command=["rm", "-rf", "libhtp"]))
|
||||
factory.addStep(ShellCommand(command=["git", "clone", "-b", "0.5.x", "/data/oisf/libhtp/.git/", "libhtp"]))
|
||||
return factory
|
||||
|
||||
factory = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory.addStep(ShellCommand(command=["./configure"]))
|
||||
factory.addStep(ShellCommand(command=["make"]))
|
||||
factory.addStep(ShellCommand(command=["make", "clean"]))
|
||||
#factory.addStep(ShellCommand(command=["make", "distcheck"]))
|
||||
|
||||
factory_devel = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_devel.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory_devel.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests"]))
|
||||
factory_devel.addStep(ShellCommand(command=["make"]))
|
||||
factory_devel.addStep(ShellCommand(command=["make", "clean"]))
|
||||
#factory_devel.addStep(ShellCommand(command=["make", "distcheck"], env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-unittests"}))
|
||||
|
||||
factory_profiling = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_profiling.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory_profiling.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-profiling","--enable-unittests"]))
|
||||
factory_profiling.addStep(ShellCommand(command=["make"]))
|
||||
factory_profiling.addStep(ShellCommand(command=["make", "clean"]))
|
||||
#factory_profiling.addStep(ShellCommand(command=["make", "distcheck"],env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-profiling --enable-unittests"}))
|
||||
|
||||
factory_clang = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_clang.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
#factory_clang.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","CFLAGS=-fsanitize=address"]))
|
||||
factory_clang.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","ac_cv_func_malloc_0_nonnull=yes","ac_cv_func_realloc_0_nonnull=yes"]))
|
||||
factory_clang.addStep(ShellCommand(command=["make"]))
|
||||
factory_clang.addStep(ShellCommand(command=["make", "clean"]))
|
||||
|
||||
factory_clang_32 = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_clang_32.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory_clang_32.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","CFLAGS=-fsanitize=address","ac_cv_func_malloc_0_nonnull=yes","ac_cv_func_realloc_0_nonnull=yes"]))
|
||||
factory_clang_32.addStep(ShellCommand(command=["make"]))
|
||||
factory_clang_32.addStep(ShellCommand(command=["make", "clean"]))
|
||||
|
||||
factory_features = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_features.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory_features.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","--enable-nfqueue","--enable-nflog", "--enable-lua", "--enable-prelude"]))
|
||||
factory_features.addStep(ShellCommand(command=["make"]))
|
||||
factory_features.addStep(ShellCommand(command=["make", "clean"]))
|
||||
import psutil
|
||||
factory_features.addStep(ShellCommand(command=["make", "distcheck"],env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-unittests --enable-nfqueue --enable-nflog --enable-lua --enable-prelude", "CONCURRENCY_LEVEL": str(psutil.cpu_count())}))
|
||||
|
||||
import os
|
||||
PCAP_PATH='/data/oisf/qa/docker/pcaps/'
|
||||
(_, _, pcaps_list) = os.walk(PCAP_PATH).next()
|
||||
pcaps_list = [ os.path.join(PCAP_PATH, pcap) for pcap in pcaps_list if pcap.endswith(".pcap") ]
|
||||
|
||||
factory_stress_pcap = SuriBuildFactory()
|
||||
# run the tests (note that this will require that 'trial' is installed)
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["./autogen.sh"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["./configure","--enable-debug-validation"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["make"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "make","install"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "rm", "-f", "/usr/local/etc/suricata/suricata.yaml"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "make","install-conf"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["make","clean"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "ldconfig"]))
|
||||
for pfile in pcaps_list:
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "/usr/local/bin/suricata","-r",pfile,"--init-errors-fatal","-S","/data/oisf/rules/http-events.rules"]))
|
||||
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "rm", "-rf", "/usr/local/var/log/suricata/"]))
|
||||
|
||||
from buildbot.config import BuilderConfig
|
||||
|
||||
def SuriBuilderConfig(*args, **kwargs):
|
||||
if not kwargs.has_key('category'):
|
||||
kwargs['category']='default'
|
||||
return BuilderConfig(*args, **kwargs)
|
||||
|
||||
c['builders'] = []
|
||||
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="gcc",
|
||||
slavename="buildslave",
|
||||
factory=factory))
|
||||
c['schedulers'].append(SingleBranchScheduler(
|
||||
name="build",
|
||||
change_filter=filter.ChangeFilter(branch='master'),
|
||||
treeStableTimer=None,
|
||||
builderNames=["gcc"]))
|
||||
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="debug",
|
||||
slavename="buildslave",
|
||||
factory=factory_devel))
|
||||
c['schedulers'].append(SingleBranchScheduler(
|
||||
name="debug",
|
||||
change_filter=filter.ChangeFilter(branch='master'),
|
||||
treeStableTimer=None,
|
||||
builderNames=["debug"]))
|
||||
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="profiling",
|
||||
slavename="buildslave",
|
||||
factory=factory_profiling))
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="clang",
|
||||
slavename="buildslave",
|
||||
factory=factory_clang_32))
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="features",
|
||||
slavename="buildslave",
|
||||
factory=factory_features))
|
||||
c['builders'].append(
|
||||
SuriBuilderConfig(name="pcaps",
|
||||
slavename="buildslave",
|
||||
factory=factory_stress_pcap))
|
||||
|
||||
from buildbot import locks
|
||||
build_lock = locks.SlaveLock("slave_builds", maxCount = 1)
|
||||
|
||||
|
||||
from buildbot.schedulers.forcesched import *
|
||||
c['schedulers'].append(ForceScheduler(name="force", builderNames = [ builder.getConfigDict()['name'] for builder in c['builders'] ]))
|
||||
|
||||
c['status'] = []
|
||||
|
||||
from buildbot.status import html
|
||||
from buildbot.status.web import authz, auth
|
||||
|
||||
authz_cfg=authz.Authz(
|
||||
# change any of these to True to enable; see the manual for more
|
||||
# options
|
||||
#auth=auth.BasicAuth(users),
|
||||
gracefulShutdown = False,
|
||||
forceBuild = True, # use this to test your slave once it is set up
|
||||
forceAllBuilds = True,
|
||||
pingBuilder = True,
|
||||
stopBuild = True,
|
||||
stopAllBuilds = True,
|
||||
cancelPendingBuild = True,
|
||||
)
|
||||
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
||||
|
||||
####### PROJECT IDENTITY
|
||||
|
||||
# the 'title' string will appear at the top of this buildbot
|
||||
# installation's html.WebStatus home page (linked to the
|
||||
# 'titleURL') and is embedded in the title of the waterfall HTML page.
|
||||
|
||||
c['title'] = "Suricata"
|
||||
c['titleURL'] = "https://redmine.openinfosecfoundation.org/projects/suricata"
|
||||
|
||||
# the 'buildbotURL' string should point to the location where the buildbot's
|
||||
# internal web server (usually the html.WebStatus page) is visible. This
|
||||
# typically uses the port number set in the Waterfall 'status' entry, but
|
||||
# with an externally-visible host name which the buildbot cannot figure out
|
||||
# without some help.
|
||||
|
||||
c['buildbotURL'] = "http://localhost:8010/"
|
||||
|
||||
####### DB URL
|
||||
|
||||
c['db'] = {
|
||||
# This specifies what database buildbot uses to store its state. You can leave
|
||||
# this at its default for all but the largest installations.
|
||||
'db_url' : "sqlite:///state.sqlite",
|
||||
}
|
||||
Binary file not shown.
Loading…
Reference in New Issue