[prev in list] [next in list] [prev in thread] [next in thread]
List: pyamf-commits
Subject: [pyamf-commits] r2914 - pyamf/sandbox/buildbot
From: SVN commit logs for PyAMF <commits () pyamf ! org>
Date: 2009-10-03 23:58:59
Message-ID: 20091003235859.2CD467C8015 () mail ! collab ! com
[Download RAW message or body]
Author: thijs
Date: 2009-10-04 01:58:58 +0200 (Sun, 04 Oct 2009)
New Revision: 2914
Modified:
pyamf/sandbox/buildbot/pyamf.py
Log:
refactor build script
Modified: pyamf/sandbox/buildbot/pyamf.py
===================================================================
--- pyamf/sandbox/buildbot/pyamf.py 2009-09-29 18:08:21 UTC (rev 2913)
+++ pyamf/sandbox/buildbot/pyamf.py 2009-10-03 23:58:58 UTC (rev 2914)
@@ -23,9 +23,6 @@
raise ImportError('This script requires Buildbot 0.7.11 or newer')
-DISABLE_EXT = '--disable-ext'
-
-
def getInterpreter(os='unix', version='2.5'):
"""
Get Python or Jython interpreter string.
@@ -49,216 +46,297 @@
return [interpreter]
-class GAECompile(ShellCommand):
- """
- Google App Engine buildstep.
- """
+class Builder(object):
- def __init__(self, slaveScript=None, **kwargs):
+ def __init__(self, version, os):
"""
- @param slaveScript: Location of the buildslave script.
- @type slaveScript: C{str}
+ @param version: Python interpreter version number.
+ @type version: C{str}
+ @param os: String containing the operating system name.
+ @type os: C{str}
"""
- self.name = 'google-app-engine'
- self.descriptionDone = 'google-app-engine punit'
- self.slaveScript = slaveScript
+ self.version = version
+ self.os = os
+ self.command = []
- ShellCommand.__init__(self, **kwargs)
- self.addFactoryArguments(slaveScript=slaveScript)
+ def slave_step(self, **buildstep_kwargs):
+ """
+ Create slave buildstep.
+
+ @return: The step.
+ """
+ step = self.type(name=self.name, descriptionDone=self.descriptionDone,
+ command=self.command, **buildstep_kwargs)
+
+ return step
- def start(self):
- command = ['python2.5', self.slaveScript]
- if self.getProperty('branch') is not None:
- command.append(WithProperties('--branch=%(branch)s'))
+ def master_step(self, **buildstep_kwargs):
+ """
+ Create master buildstep.
+
+ @return: The step.
+ """
+ step = MasterShellCommand(name=self.name, command=self.command,
+ **buildstep_kwargs)
+
+ return step
- self.setCommand(command)
- ShellCommand.start(self)
+ def setup_step(self, action, **buildstep_kwargs):
+ """
+ Create Python setuptools step.
-class UnpackSQLAlchemy(ShellCommand):
- """
- Put SQLAlchemy source files in place on the buildslave.
- """
+ @param action
+ @type action: C{str}
+ """
+ self.name = 'python%s-%s' % (self.version, action)
+ self.descriptionDone = 'python%s %s' % (self.version, action)
+ self.command = getInterpreter(self.os, self.version) + ['./setup.py', action, self.command]
- def __init__(self, version=None, **kwargs):
+ if self.ext is not True:
+ self.command.append('--disable-ext')
+
+ return self.slave_step(**buildstep_kwargs)
+
+
+ def decompress_step(self, file, **buildstep_kwargs):
"""
- @param version: SQLAlchemy version number, ie. '0.4.8'.
- @type version: C{str}
+ Decompress a tar file.
+
+ @param file: Name of the tarball file.
+ @type file: C{str}
+
+ @return: A slave step.
"""
- self.version = version
- self.name = 'unpack-sqlalchemy-' + self.version
- self.descriptionDone = 'Unpacked SQLAlchemy ' + self.version
+ self.type = ShellCommand
+ self.command = ['tar', 'xvf', file] + self.command
+
+ return self.slave_step(**buildstep_kwargs)
- ShellCommand.__init__(self, **kwargs)
- self.addFactoryArguments(version=version)
+ def compress_step(self, file, **buildstep_kwargs):
+ """
+ Compress a tar file with gzip encoding.
- def start(self):
+ @param file: Name of the output tarball.
+ @type file: C{str}
+
+ @return: A slave step.
"""
- Unpack compressed SQLAlchemy tarball.
+ self.type = ShellCommand
+ self.command = ['tar', 'zcvf', file] + self.command
+
+ return self.slave_step(**buildstep_kwargs)
+
+
+ def compile(self, ext=False, **buildstep_kwargs):
"""
- command = ['tar', 'zxvf', 'sqlalchemy-%s.tar.gz' % self.version]
- self.setCommand(command)
- ShellCommand.start(self)
+ Build the code.
+
+ @param ext: Enable C-extension build.
+ @type ext: C{bool}
+ """
+ self.type = Compile
+ self.ext = ext
+ return self.setup_step('build', **buildstep_kwargs)
-class PythonCompile(Compile):
- """
- Build the Python and C-extension code.
- """
- def __init__(self, version=None, os=None, ext=None, **kwargs):
+ def test(self, ext=False, **buildstep_kwargs):
"""
- @param version: Python interpreter version number.
- @type version: C{str}
- @param os: String containing the operating system name.
- @type os: C{str}
- @param ext: Enable extension build.
+ Test the code.
+
+ @param ext: Enable C-extension build.
@type ext: C{bool}
"""
- self.action = 'build'
- self.version = version
- self.os = os
+ self.type = Test
self.ext = ext
- self.name = 'python%s-%s' % (self.version, self.action)
- self.descriptionDone = 'python%s %s' % (self.version, self.action)
+ #step.evaluateCommand = evaluateCommand
- Compile.__init__(self, **kwargs)
+ return self.setup_step('test', **buildstep_kwargs)
- self.addFactoryArguments(version=version, os=os, ext=ext)
+ def install(self, ext=False, **buildstep_kwargs):
+ """
+ Install the code.
- def start(self):
+ @param ext: Enable C-extension build.
+ @type ext: C{bool}
"""
- Start the build.
+ self.type = Compile
+ self.ext = ext
+ self.command = ['--root=./install']
+
+ return self.setup_step('install', **buildstep_kwargs)
+
+
+ def dist(self, ext=True, **buildstep_kwargs):
"""
- command = getInterpreter(self.os, self.version) + ['./setup.py', self.action]
+ Build platform-specific .egg file.
- if self.ext is not True:
- command.append(DISABLE_EXT)
+ @param ext: Enable C-extension build.
+ @type ext: C{bool}
+ """
+ self.type = ShellCommand
+ self.ext = ext
+ self.command = ['--dist-dir=./build/dist']
+
+ return self.setup_step('bdist_egg', **buildstep_kwargs)
- self.setCommand(command)
- Compile.start(self)
+ def publish(self, src, dest, **buildstep_kwargs):
+ """
+ Publish the .egg file to the web on the master.
-class PythonTest(Test):
- """
- Test the pure Python and C-extension code.
- """
- total = 0
+ @param src: Location of the .egg file.
+ @type src: C{str}
+ @param dest: Destination folder for the .egg file.
+ @type dest: C{str}
+ """
+ self.name = 'Publish .egg to web'
+ self.descriptionDone = 'Published .egg to web'
+ self.command = ['mv ' + src + ' ' + dest]
+ return self.master_step(**buildstep_kwargs)
- def __init__(self, version=None, os=None, ext=None, **kwargs):
+
+ def parse_dumps(self, **buildstep_kwargs):
"""
- @param version: Python interpreter version number.
+ Run the parse_dump script on the AMF dump files.
+ """
+ self.type = ShellCommand
+ self.name = 'Parsing AMF dumps'
+ self.descriptionDone = 'Parsed AMF dumps'
+ self.command = [getInterpreter(self.os, self.version),
+ './build/parse_dump.py', './build/dumps/*']
+
+ return self.slave_step(**buildstep_kwargs)
+
+
+ def unpack_dumps(self, src, **buildstep_kwargs):
+ """
+ Decompresses the tarball's .amf* dump files.
+
+ @param src: Location of the compressed dumps tarball.
+ @type src: C{str}Decompresses the tarball's .amf* dump files.
+ """
+ self.name = 'Decompress AMF dump files'
+ self.descriptionDone = 'Decompressed AMF dump files'
+
+ return self.decompress_step(src, **buildstep_kwargs)
+
+
+ def unpack_sqlalchemy(self, version, **buildstep_kwargs):
+ """
+ Put SQLAlchemy source files in place on the buildslave.
+
+ @param version: SQLAlchemy version number, ie. '0.4.8'.
@type version: C{str}
- @param os: String containing the operating system name.
- @type os: C{str}
- @param ext: Enable extension build.
- @type ext: C{bool}
"""
- self.action = 'test'
- self.version = version
- self.os = os
- self.ext = ext
- self.name = 'python%s-%s' % (self.version, self.action)
- self.descriptionDone = 'python%s %s' % (self.version, self.action)
+ if version is None:
+ return
- Test.__init__(self, **kwargs)
+ self.name = 'unpack-sqlalchemy-' + version
+ self.descriptionDone = 'Unpacked SQLAlchemy ' + version
- self.addFactoryArguments(version=version, os=os, ext=ext)
+ src = 'sqlalchemy-%s.tar.gz' % version
+ return self.decompress_step(src, **buildstep_kwargs)
- def start(self):
+
+ def compress_egg(self, src, **buildstep_kwargs):
"""
- Run the test suite.
+ Compresses the .egg file into a tarball for transfer to the buildmaster.
+
+ @param src: Location of the tarball.
+ @type src: C{str}
"""
- command = getInterpreter(self.os, self.version) + ['./setup.py', self.action]
+ self.name = 'Compress .egg file'
+ self.descriptionDone = 'Compressed .egg file'
+ self.command = ['./build/dist']
- if self.ext is not True:
- command.append(DISABLE_EXT)
+ return self.compress_step(src, **buildstep_kwargs)
- self.setCommand(command)
- Test.start(self)
+ def upload_egg(self, src, dest, **buildstep_kwargs):
+ """
+ Transfer the compressed egg to the buildmaster.
+
+ @param src: Name of the tarball.
+ @type src: C{str}
+ @param dest: Target folder on the buildmaster.
+ @type dest: C{str}
+ """
+ return FileUpload(slavesrc=src, masterdest=dest, **buildstep_kwargs)
+
- def evaluateCommand(self, cmd):
- r = Test.evaluateCommand(self, cmd)
- if r is FAILURE:
- return r
+def evaluateCommand(cmd):
+ r = Test.evaluateCommand(self, cmd)
- lines = self.getLog('stdio').readlines()
+ if r is FAILURE:
+ return r
- re_test_result = re.compile("FAILED \((failures=(\d*), )?(errors=(\d*), )?successes=(\d*)\)")
+ lines = self.getLog('stdio').readlines()
- mos = map(lambda line: re_test_result.search(line), lines)
- test_result_lines = [mo.groups() for mo in mos if mo]
+ re_test_result = re.compile("FAILED \((failures=(\d*), )?(errors=(\d*), )?successes=(\d*)\)")
- if not test_result_lines:
- return cmd.rc
+ mos = map(lambda line: re_test_result.search(line), lines)
+ test_result_lines = [mo.groups() for mo in mos if mo]
- results = test_result_lines[0]
+ if not test_result_lines:
+ return cmd.rc
- failures = errors = passed = 0
+ results = test_result_lines[0]
- if results[1] is not None:
- failures = int(results[1])
+ failures = errors = passed = 0
- if results[3] is not None:
- errors = int(results[3])
+ if results[1] is not None:
+ failures = int(results[1])
- passed = int(results[4])
+ if results[3] is not None:
+ errors = int(results[3])
- if [failures, errors] == [0, 0]:
- rc = SUCCESS
- else:
- rc = FAILURE
+ passed = int(results[4])
- self.setTestResults(total=failures + errors + passed, failed=failures + errors, passed=passed)
+ if [failures, errors] == [0, 0]:
+ rc = SUCCESS
+ else:
+ rc = FAILURE
- return rc
+ self.setTestResults(total=failures + errors + passed, failed=failures + errors, passed=passed)
+ return rc
-class PythonInstall(Compile):
+
+class GAECompile(ShellCommand):
"""
- Install pure Python and C-extension code.
+ Google App Engine buildstep.
"""
- def __init__(self, version=None, os=None, ext=None, **kwargs):
+ def __init__(self, slaveScript=None, **kwargs):
"""
- @param version: Python interpreter version number.
- @type version: C{str}
- @param os: String containing the operating system name.
- @type os: C{str}
- @param ext: Enable extension build.
- @type ext: C{bool}
+ @param slaveScript: Location of the buildslave script.
+ @type slaveScript: C{str}
"""
- self.action = 'install'
- self.version = version
- self.os = os
- self.ext = ext
- self.name = 'python%s-%s' % (self.version, self.action)
- self.descriptionDone = 'python%s %s' % (self.version, self.action)
+ self.name = 'google-app-engine'
+ self.descriptionDone = 'google-app-engine punit'
+ self.slaveScript = slaveScript
- Compile.__init__(self, **kwargs)
+ ShellCommand.__init__(self, **kwargs)
- self.addFactoryArguments(version=version, os=os, ext=ext)
+ self.addFactoryArguments(slaveScript=slaveScript)
def start(self):
- """
- Install the code in a local folder called 'install'.
- """
- command = getInterpreter(self.os, self.version) + ['./setup.py',
- self.action, '--root=./install']
+ command = ['python2.5', self.slaveScript]
- if self.ext is not True:
- command.append(DISABLE_EXT)
+ if self.getProperty('branch') is not None:
+ command.append(WithProperties('--branch=%(branch)s'))
self.setCommand(command)
- Compile.start(self)
+ ShellCommand.start(self)
class GAEBuilder(object):
@@ -266,18 +344,18 @@
Google App Engine build factory.
"""
- def __init__(self, name, slaveName, slaveScript):
+ def __init__(self, builder, slave, src):
"""
- @param name: Builder name.
- @type name: C{str}
- @param slaveName: Slave name.
- @type slaveName: C{str}
- @param slaveScript: Location of the buildslave script.
- @type slaveScript: C{str}
+ @param builder: Builder name.
+ @type builder: C{str}
+ @param slave: Slave name.
+ @type slave: C{str}
+ @param src: Location of the buildslave script.
+ @type src: C{str}
"""
- self.name = name
- self.slaveName = slaveName
- self.script = slaveScript
+ self.builder = builder
+ self.slave = slave
+ self.script = src
def createBuilder(self):
@@ -292,18 +370,18 @@
f = factory.BuildFactory()
f.addStep(compile)
- b = {'name': self.name,
- 'slavename': self.slaveName,
- 'builddir': self.name,
+ b = {'name': self.builder,
+ 'slavename': self.slave,
+ 'builddir': self.builder,
'factory': f,
}
return b
-class PythonBuilder(object):
+class AppBuilder(object):
"""
- Builder for Python code.
+ Builder for PyAMF.
"""
def __init__(self, name, slaveName, scm_step, destFolder, webFolder,
@@ -317,10 +395,10 @@
@type scm_step: L{buildbot.steps.source.*}
@param destFolder: Destination folder on buildmaster for .egg file.
@type destFolder: C{str}
- @param webFolder: Destination folder on buildmaster for nightly file.
- @type webFolder: C{str}
- @param saFolder: Source folder on buildmaster with SQLAlchemy tarballs.
- @type saFolder: C{str}
+ @param webFolder: Destination folder on buildmaster for nightly file.
+ @type webFolder: C{str}
+ @param saFolder: Source folder on buildmaster with SQLAlchemy tarballs.
+ @type saFolder: C{str}
@param sa04: SQLAlchemy 0.4.x version number, ie. '0.4.8'.
@type sa04: C{str}
@param sa05: SQLAlchemy 0.5.x version number, ie. '0.5.0'.
@@ -333,8 +411,8 @@
self.slaveName = slaveName
self.scm_step = scm_step
self.destFolder = destFolder
- self.webFolder = webFolder
- self.saFolder = saFolder
+ self.webFolder = webFolder
+ self.saFolder = saFolder
self.sa04 = sa04
self.sa05 = sa05
self.dumps = dumps
@@ -351,7 +429,7 @@
"""
# Download source to slave for SQLAlchemy 0.4.x and 0.5.x
sa_tarball = 'sqlalchemy-%s.tar.gz'
- sa_path = os.path.join(self.saFolder, sa_tarball)
+ sa_path = os.path.join(self.saFolder, sa_tarball)
sqlalchemy04 = FileDownload(mastersrc=sa_path % self.sa04,
slavedest=sa_tarball % self.sa04)
sqlalchemy05 = FileDownload(mastersrc=sa_path % self.sa05,
@@ -366,22 +444,25 @@
# Create build factory
f = factory.BuildFactory()
+ builder = Builder(self.pythonVersion, self.os)
# Setup build steps
- pure_compile = PythonCompile(ext=False, version=self.pythonVersion, os=self.os)
- pure_test = PythonTest(ext=False, version=self.pythonVersion, os=self.os)
- pure_install = PythonInstall(ext=False, version=self.pythonVersion, os=self.os)
- c_compile = PythonCompile(ext=True, version=self.pythonVersion, os=self.os)
- c_test = PythonTest(ext=True, version=self.pythonVersion, os=self.os)
- c_install = PythonInstall(ext=True, version=self.pythonVersion, os=self.os)
- c_egg = PythonEgg(version=self.pythonVersion, os=self.os)
+ pure_compile = builder.compile()
+ pure_test = builder.test()
+ pure_install = builder.install()
+ c_compile = builder.compile(True)
+ c_test = builder.test(True)
+ c_install = builder.install(True)
+ c_egg = builder.dist()
+ unpack_sqlalchemy04 = builder.unpack_sqlalchemy(self.sa04)
+ unpack_sqlalchemy05 = builder.unpack_sqlalchemy(self.sa05)
# Pure Python - SQLAlchemy 0.4.x
f.addStep(self.scm_step)
if self.sa04:
f.addStep(sqlalchemy04)
- f.addStep(UnpackSQLAlchemy, version=self.sa04)
+ f.addStep(unpack_sqlalchemy04)
f.addStep(pure_compile)
f.addStep(pure_test)
@@ -395,7 +476,7 @@
if self.sa05:
f.addStep(self.scm_step)
f.addStep(sqlalchemy05)
- f.addStep(UnpackSQLAlchemy, version=self.sa05)
+ f.addStep(unpack_sqlalchemy05)
f.addStep(pure_compile)
f.addStep(pure_test)
f.addStep(pure_install)
@@ -405,7 +486,7 @@
if self.sa04:
f.addStep(sqlalchemy04)
- f.addStep(UnpackSQLAlchemy, version=self.sa04)
+ f.addStep(unpack_sqlalchemy04)
f.addStep(c_compile)
f.addStep(c_test)
@@ -415,24 +496,24 @@
if self.sa05:
f.addStep(self.scm_step)
f.addStep(sqlalchemy05)
- f.addStep(UnpackSQLAlchemy, version=self.sa05)
+ f.addStep(unpack_sqlalchemy05)
f.addStep(c_compile)
f.addStep(c_test)
f.addStep(c_install)
# Run parser script on AMF dumps
f.addStep(amf_dumps)
- f.addStep(DecompressDumps, location=self.dumps)
- f.addStep(ParseDumps, version=self.pythonVersion, os=self.os)
+ f.addStep(builder.unpack_dumps(self.dumps))
+ f.addStep(builder.parse_dumps())
# Build .egg file for trunk and upload to the master
egg_path = os.path.join(self.destFolder, eggTarball)
f.addStep(self.scm_step)
f.addStep(c_egg)
- f.addStep(CompressEgg, packageName=eggTarball)
- f.addStep(FileUpload(slavesrc=eggTarball, masterdest=egg_path))
+ f.addStep(builder.compress_egg(eggTarball))
+ f.addStep(builder.upload_egg(eggTarball, egg_path))
# Master step instruction from slave
- f.addStep(PublishEgg, location=egg_path, destination=self.webFolder)
+ f.addStep(builder.publish(egg_path, self.webFolder))
# Create and return builder
b = {'name': self.name,
@@ -445,136 +526,3 @@
return b
-
-class PythonEgg(ShellCommand):
- """
- Build platform-specific .egg file.
- """
-
- def __init__(self, version=None, os=None, **kwargs):
- """
- @param version: Python interpreter version number.
- @type version: C{str}
- @param os: String containing the operating system name.
- @type os: C{str}
- """
- self.action = 'bdist_egg'
- self.version = version
- self.os = os
- self.name = 'python%s-%s' % (self.version, self.action)
- self.descriptionDone = 'python%s %s' % (self.version, self.action)
-
- ShellCommand.__init__(self, **kwargs)
-
- self.addFactoryArguments(version=version, os=os)
-
- def start(self):
- command = getInterpreter(self.os, self.version) + ['./setup.py',
- self.action, '--dist-dir=./build/dist']
- self.setCommand(command)
- ShellCommand.start(self)
-
-
-class CompressEgg(ShellCommand):
- """
- Compresses the .egg file into a tarball for transfer to the buildmaster.
- """
-
- def __init__(self, packageName=None, **kwargs):
- """
- @param packageName: Name of the tarball.
- @type packageName: C{str}
- """
- self.name = 'Compress .egg file'
- self.descriptionDone = 'Compressed .egg file'
- self.eggTarball = packageName
-
- ShellCommand.__init__(self, **kwargs)
-
- self.addFactoryArguments(packageName=packageName)
-
- def start(self):
- """
- Compress egg tarball.
- """
- command = ['tar', 'zcvf', self.eggTarball, './build/dist']
- self.setCommand(command)
- ShellCommand.start(self)
-
-
-class PublishEgg(MasterShellCommand):
- """
- Publish the .egg file to the web.
- """
-
- def __init__(self, location=None, destination=None, **kwargs):
- """
- @param destination: Destination folder for the .egg file.
- @type destination: C{str}
- @param location: Location of the .egg file.
- @type location: C{str}
- """
- self.name = 'Publish .egg to web'
- self.descriptionDone = 'Published .egg to web'
- self.destination = destination
- self.location = location
-
- command = 'mv ' + self.location + ' ' + self.destination
-
- MasterShellCommand.__init__(self, command, **kwargs)
-
- self.addFactoryArguments(destination=destination, location=location)
-
-
-class DecompressDumps(ShellCommand):
- """
- Decompresses the tarball's .amf* dump files.
- """
-
- def __init__(self, location=None, **kwargs):
- """
- @param location: Location of the compressed dumps tarball.
- @type location: C{str}
- """
- self.name = 'Decompress AMF dump files'
- self.descriptionDone = 'Decompressed AMF dump files'
- self.location = location
-
- ShellCommand.__init__(self, **kwargs)
-
- self.addFactoryArguments(location=location)
-
- def start(self):
- """
- Decompress AMF dumps tarball.
- """
- command = ['tar', 'xvf', self.location]
- self.setCommand(command)
- ShellCommand.start(self)
-
-
-class ParseDumps(ShellCommand):
- """
- Run the parse_dump script on the AMF dump files.
- """
- def __init__(self, version=None, os=None, **kwargs):
- """
- @param version: Python interpreter version number.
- @type version: C{str}
- @param os: String containing the operating system name.
- @type os: C{str}
- """
- self.version = version
- self.os = os
- self.name = 'Parsing AMF dumps'
- self.descriptionDone = 'Parsed AMF dumps'
-
- ShellCommand.__init__(self, **kwargs)
-
- self.addFactoryArguments(version=version, os=os)
-
- def start(self):
- command = getInterpreter(self.os, self.version) + ['./build/parse_dump.py',
- './build/dumps/*']
- self.setCommand(command)
- ShellCommand.start(self)
_______________________________________________
PyAMF commits mailing list - commits@pyamf.org
http://lists.pyamf.org/mailman/listinfo/commits
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic