[GNUnet-SVN] r23223 - in gnunet-planetlab/gplmt: contrib scripts

[gnunet]

Author: wachs
Date: 2012-08-14 14:45:59 +0200 (Tue, 14 Aug 2012)
New Revision: 23223

Added:
   gnunet-planetlab/gplmt/contrib/master_template.cfg
Modified:
   gnunet-planetlab/gplmt/scripts/create_buildbot_configuration.py
Log:
master cfg working

Added: gnunet-planetlab/gplmt/contrib/master_template.cfg
===================================================================
--- gnunet-planetlab/gplmt/contrib/master_template.cfg                          
(rev 0)
+++ gnunet-planetlab/gplmt/contrib/master_template.cfg  2012-08-14 12:45:59 UTC 
(rev 23223)
@@ -0,0 +1,203 @@
+# -*- python -*-
+# ex: set syntax=python:
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory (although the filename
+# can be changed with the --basedir option to 'mktap buildbot master').
+
+# It has one job: define a dictionary named BuildmasterConfig. This
+# dictionary has a variety of keys to control different aspects of the
+# buildmaster. They are documented in docs/config.xhtml .
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+
+####### BUILDSLAVES
+
+# the 'slaves' list defines the set of allowable buildslaves. Each element is
+# a tuple of bot-name and bot-password. These correspond to values given to
+# the buildslave's mktap invocation.
+from buildbot.buildslave import BuildSlave
+c['slaves'] =  [%GPLMT_SLAVES]
+
+# 'slavePortnum' defines the TCP port to listen on. This must match the value
+# configured into the buildslaves (with their --master option)
+
+c['slavePortnum'] = 9989
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should find out
+# about source code changes. Any class which implements IChangeSource can be
+# put here: there are several in buildbot/changes/*.py to choose from.
+
+from buildbot.changes.pb import PBChangeSource
+c['change_source'] = PBChangeSource()
+
+# For example, if you had CVSToys installed on your repository, and your
+# CVSROOT/freshcfg file had an entry like this:
+#pb = ConfigurationSet([
+#    (None, None, None, PBService(userpass=('foo', 'bar'), port=4519)),
+#    ])
+
+# then you could use the following buildmaster Change Source to subscribe to
+# the FreshCVS daemon and be notified on every commit:
+#
+#from buildbot.changes.freshcvs import FreshCVSSource
+#fc_source = FreshCVSSource("cvs.example.com", 4519, "foo", "bar")
+#c['change_source'] = fc_source
+
+# or, use a PBChangeSource, and then have your repository's commit script run
+# 'buildbot sendchange', or use contrib/svn_buildbot.py, or
+# contrib/arch_buildbot.py :
+#
+#from buildbot.changes.pb import PBChangeSource
+#c['change_source'] = PBChangeSource()
+
+
+####### SCHEDULERS
+
+## configure the Schedulers
+
+from buildbot.scheduler import Scheduler
+c['schedulers'] = []
+c['schedulers'].append(Scheduler(name="gnunet", branch=None,
+                                 treeStableTimer=30*60,
+                                categories=["GNUnet"],
+                                 builderNames=[%GPLMT_SCHEDULER_BUILDERS]))
+
+
+####### BUILDERS
+
+# the 'builders' list defines the Builders. Each one is configured with a
+# dictionary, using the following keys:
+#  name (required): the name used to describe this bilder
+#  slavename (required): which slave to use, must appear in c['bots']
+#  builddir (required): which subdirectory to run the builder in
+#  factory (required): a BuildFactory to define how the build is run
+#  periodicBuildTime (optional): if set, force a build every N seconds
+
+# buildbot/process/factory.py provides several BuildFactory classes you can
+# start with, which implement build processes for common targets (GNU
+# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
+# base class, and is configured with a series of BuildSteps. When the build
+# is run, the appropriate buildslave is told to execute each Step in turn.
+
+# the first BuildStep is typically responsible for obtaining a copy of the
+# sources. There are source-obtaining Steps in buildbot/steps/source.py for
+# CVS, SVN, and others.
+
+svnurl = "https://gnunet.org/svn/gnunet";
+
+from buildbot.changes.svnpoller import SVNPoller
+from buildbot.process import factory
+from buildbot.steps import source, shell
+from buildbot.steps.shell import Compile
+import os
+#rm-nat
+GNUNET_LOG_LEVEL=';;;;ERROR'
+tmp = "/tmp/gnbuild"
+le_tmp = "/tmp/lebuild"
+c['change_source'] = [SVNPoller(svnurl = "https://gnunet.org/svn/gnunet/";, 
category="GNUnet")]
+
+f = factory.BuildFactory()
+f.addStep(source.SVN(mode='update', svnurl = "https://gnunet.org/svn/gnunet/";))
+#f.addStep(shell.ShellCommand, command=["sh", "-c", "killall 
gnunet-service-arm || true; svn -R revert *; rm -rf /tmp/gnunet*; rm -rf 
/tmp/test*;"],
+#      description="clean up from previous runs",
+#      name="killall",
+#      haltOnFailure=False,
+#      timeout=30)
+#f.addStep(shell.ShellCommand, command=["sh", "-c", "./bootstrap"],
+#      description="bootstrap",
+#      name="bootstrap",
+#      haltOnFailure=True)
+#f.addStep(shell.Configure, command=["sh", "-c", "./configure 
--enable-logging=verbose --enable-benchmarks --enable-expensive-tests 
--prefix=" + tmp],
+#      description="configure",
+#      name="configure",
+#      haltOnFailure=True)
+#f.addStep(shell.Compile,command=["sh", "-c", "make install"])
+                  
+%GPLMT_BUILDER_DEFINITION 
+
+c['builders'] = [%GPLMT_BUILDER_SUMMARY]
+
+####### STATUS TARGETS
+
+# 'status' is a list of Status Targets. The results of each build will be
+# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+# including web pages, email senders, and IRC bots.
+
+c['status'] = []
+
+from buildbot.status import html
+from buildbot.status.web.authz import Authz
+from buildbot.status.web.auth import BasicAuth
+
+
+users = [('team', 'gnunet')]
+authz = Authz(auth=BasicAuth(users),
+       forceAllBuilds='auth',
+       forceBuild='auth', # only authenticated users
+        stopBuild='auth', # but anyone can do this
+        stopAllBuilds='auth',
+     )
+
+c['status'].append(html.WebStatus(http_port=8010, authz=authz))
+
+
+
+
+#c['status'].append(html.WebStatus(http_port=8010, allowForce=True))
+
+# from buildbot.status import mail
+# c['status'].append(mail.MailNotifier(fromaddr="address@hidden",
+#                                      extraRecipients=["address@hidden"],
+#                                      sendToInterestedUsers=False))
+#
+#from buildbot.status import words
+#c['status'].append(words.IRC(host="irc.freenode.net", nick="gnunet-plb",
+#                             channels=["#gnunet"]))
+#
+# from buildbot.status import client
+# c['status'].append(client.PBListener(9988))
+
+
+####### DEBUGGING OPTIONS
+
+# if you set 'debugPassword', then you can connect to the buildmaster with
+# the diagnostic tool in contrib/debugclient.py . From this tool, you can
+# manually force builds and inject changes, which may be useful for testing
+# your buildmaster without actually commiting changes to your repository (or
+# before you have a functioning 'sources' set up). The debug tool uses the
+# same port number as the slaves do: 'slavePortnum'.
+
+#c['debugPassword'] = "f1955c29e336834e88476f74c1825cdb"
+
+# if you set 'manhole', you can ssh into the buildmaster and get an
+# interactive python shell, which may be useful for debugging buildbot
+# internals. It is probably only useful for buildbot developers. You can also
+# use an authorized_keys file, or plain telnet.
+#from buildbot import manhole
+#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
+#                                       "admin", "password")
+
+
+####### PROJECT IDENTITY
+
+# the 'projectName' string will be used to describe the project that this
+# buildbot is working on. For example, it is used as the title of the
+# waterfall HTML page. The 'projectURL' string will be used to provide a link
+# from buildbot HTML pages to your project's home page.
+
+c['projectName'] = "GNUnet PlanetLab Testbed"
+c['projectURL'] = "https://gnunet.org/gplmt";
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.Waterfall page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = "http://gnunet.org:8010/";

Modified: gnunet-planetlab/gplmt/scripts/create_buildbot_configuration.py
===================================================================
--- gnunet-planetlab/gplmt/scripts/create_buildbot_configuration.py     
2012-08-14 11:20:34 UTC (rev 23222)
+++ gnunet-planetlab/gplmt/scripts/create_buildbot_configuration.py     
2012-08-14 12:45:59 UTC (rev 23223)
@@ -24,12 +24,13 @@
 plslice = None
 filename = None
 master = None
-template = None
+template_file = None
 out_cmd_file = "cmdfile"
+out_cfg_file = "master.cfg"
 buildslave_cmd = "buildslave create-slave ./bot "
 
 try:
-    opts, args = getopt.getopt(sys.argv[1:], "hu:p:s:f:c:m:t:", ["help", 
"user=", "password=", "slice=", "file=", "cmd_file", "master", "template"])
+    opts, args = getopt.getopt(sys.argv[1:], "hu:p:s:f:c:m:t:o:", ["help", 
"user=", "password=", "slice=", "file=", "cmd_file", "master", "template", 
"cfg_output"])
 except getopt.GetoptError, err:
     # print help information and exit:
     print str(err) # will print something like "option -a not recognized"
@@ -51,8 +52,10 @@
         out_cmd_file = a
     elif o in ("-m", "--master"):
         master = a
+    elif o in ("-o", "--cfg_output"):
+        out_cfg_file = a        
     elif o in ("-t", "--template"):
-        template = a              
+        template_file = a              
     else:
         assert False, "unhandled option"
 
@@ -66,7 +69,7 @@
     usage()
     sys.exit(2)    
 
-if (template == None):
+if (template_file == None):
     print "No buildbot configuration template given!"
     usage()
     sys.exit(2)
@@ -104,7 +107,6 @@
         print "Could not open: " + str(e) 
         sys.exit (2)        
     for line in f:
-        print line
         node_hostnames.append(line)        
     f.close()
 else: 
@@ -112,8 +114,13 @@
     sys.exit (2)
     
 # Writing files
-master_cfg_buildslave = "c['slaves'] = ["
+template = {}
+master_cfg_slaves = ""
+master_cfg_builder_definition = ""
+master_cfg_builder_summary = ""
+master_cfg_scheduler_builders = ""
 slave_cmds = ""
+c = 0
 
 try:
     f_cmd = open(out_cmd_file, 'w')
@@ -122,18 +129,59 @@
     sys.exit (2)
     
 for node in node_hostnames:
+    node = node.strip()
+    print "Found node '" + node + "'" 
     password = 
''.join([random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890')
 for i in range(64)])
-    master_cfg_buildslave += 'BuildSlave("'+node+'","'+password+'"),\n'
-    f_cmd.writelines (node +";" + buildslave_cmd + " " + master + " " + 
password + "\n")
+    # slaves
+    master_cfg_slaves += 'BuildSlave("'+node+'","'+password+'"),\n'
+    # builder definition
+    master_cfg_builder_definition += "builder"+str(c)+" = {'name': \"" + node 
+"\"," +  "'slavename': \"" + node + "\",'builddir':\"" + node +"\",'factory': 
f,'category': \"GNUnet\",}\n"
+    # scheduler
+    master_cfg_scheduler_builders += '"'+node+'", '
+    # builder summary
+    master_cfg_builder_summary += "builder"+str(c)+", "
+    # cmd
+    f_cmd.writelines (node +";" + buildslave_cmd + " " + master + " " + node + 
" "+ password + "\n")
+    c += 1
 
-master_cfg_buildslave += "]"
 try:
     f_cmd.close()
 except IOError as e:
     print "Cannot close output command file "+ out_cmd_file +": "+str(e)
     sys.exit (2)
 
-print master_cfg_buildslave
+print master_cfg_slaves
+print master_cfg_builder_definition
+print master_cfg_builder_summary
+
+master_file = ""
+
+# Create master.cfg
+try:
+    f_tmpl = open(template_file, 'r')
+except IOError as e:
+    print "Cannot open template file "+ template_file +": "+str(e)
+    sys.exit (2)
+
+for line in f_tmpl:
+    line.strip()
+    line = line.replace ("%GPLMT_BUILDER_DEFINITION",  
master_cfg_builder_definition)
+    line = line.replace ("%GPLMT_BUILDER_SUMMARY",  master_cfg_builder_summary)
+    line = line.replace ("%GPLMT_SLAVES",  master_cfg_slaves)
+    line = line.replace ("%GPLMT_SCHEDULER_BUILDERS",  
master_cfg_scheduler_builders)
+    master_file += line
     
+f_tmpl.close()    
+        
+try:
+    f_cfg = open(out_cfg_file, 'w')
+except IOError as e:
+    print "Cannot open template file "+ template_file +": "+str(e)
+    sys.exit (2)    
 
+f_cfg.writelines(master_file)
+f_cfg.close()  
+    
+    
+