[prev in list] [next in list] [prev in thread] [next in thread]
List: pykde
Subject: [PyKDE] [PATCH] threaded build.py for PyQt 3.6
From: Hans-Peter Jansen <hpj () urpla ! net>
Date: 2003-04-28 10:56:57
[Download RAW message or body]
Hi Phil,
with the attached patch, build.py is able to saturate multiple CPUs on
module code and Makefile generation. Since we're running out of command
line switches, it's triggered by a -j# option from the MAKEFLAGS
environment variable.
Downsides:
- on a sufficiently fast SMP system (dual Athlon MP 1700+), it's
necessary to refine the make call a bit, in order to get the qt module
build first, since all other modules depend on it, aka:
cd qt && make && cd .. && make
- KeyboardInterrupt doesn't kill the build.py process reliable while
threads are running.
If you're going to accept this patch, I will update the docs
accordingly.
Would you accept a patch allowing to build sip and PyQt into an
independant path like I've done it for PyKDE before?
Pete
["PyQt-3.6-build-mp.diff" (text/x-diff)]
--- build.py.orig 2003-04-27 19:30:09.000000000 +0200
+++ build.py 2003-04-28 12:53:34.000000000 +0200
@@ -57,10 +57,11 @@
usingTmake = 0
enableOldPython = 0
debugMode = "release"
catCppFiles = 0
catSplit = 1
+nrThreads = 0
qpeTag = None
licType = None
gccFlags = []
@@ -1116,18 +1117,35 @@
runMake("clean")
inform("Generated the features file.")
-def generateSource(mname,plattag,qttag,xtrtag = None):
+def addBuildDir(file,wd):
+ """Add an optional build path to a given file.
+
+ file is the file name.
+ wd is an optional build directory.
+
+ returns the file name.
+ """
+ if wd:
+ file = wd + os.sep + file
+ return file
+
+
+def generateSource(mname,plattag,qttag,xtrtag = None,cpu_pool = None):
"""Generate the C++ source code for a particular PyQt module.
mname is the name of the module.
plattag is the SIP tag for the platform.
qttag is the SIP tag for the Qt version.
xtrtag is an optional extra SIP tag.
+ cpu_pool is an optional bounded semaphore to control threading.
"""
+ if cpu_pool:
+ cpu_pool.acquire()
+
inform("Generating the C++ source for the %s module." % (mname))
try:
shutil.rmtree(mname)
except:
@@ -1156,59 +1174,56 @@
runProgram(sipBin,argv)
# Python names binary modules differently on different platforms.
global proPatches, makefilePatches, sipVersion
+ # deep copy the patch dicts for threading
+ _makepat = makefilePatches.copy()
+ _propat = proPatches.copy()
+
# We don't use the plain module name with SIP v4, even though that is the
# name of the module we want to generate, because qmake doesn't like
# targets called "qt".
target = mname + "c"
if sipVersion == 4:
# Change the name of the target to Python module standards.
if sys.platform == "win32":
- makefilePatches["TARGET"] = [re.compile(target + "\\.dll",re.M), mname + ".pyd"]
+ _makepat["TARGET"] = [re.compile(target + "\\.dll",re.M), mname + ".pyd"]
else:
- makefilePatches["TARGET"] = [re.compile("lib" + target,re.M), mname]
+ _makepat["TARGET"] = [re.compile("lib" + target,re.M), mname]
else:
if sys.platform == "win32":
target = "lib" + target
# We want to change the module extension on Windows.
- makefilePatches["TARGET"] = [re.compile(target + "\\.dll",re.M), target + ".pyd"]
+ _makepat["TARGET"] = [re.compile(target + "\\.dll",re.M), target + ".pyd"]
else:
target = target + "module"
# Generate the Makefile.
inform("Generating the Makefile for the %s module." % (mname))
- olddir = pushDir(mname)
-
- proPatches["TARGET"] = [re.compile("@BL_TARGET@",re.M), target]
+ _propat["TARGET"] = [re.compile("@BL_TARGET@",re.M), target]
global debugMode
- proPatches["DEBUG"] = [re.compile("@BL_DEBUG@",re.M), debugMode]
+ _propat["DEBUG"] = [re.compile("@BL_DEBUG@",re.M), debugMode]
global catCppFiles
- buildMakefile(pro,catCppFiles)
+ buildMakefile(pro,catCppFiles,_propat,_makepat,mname)
fixInstallTarget(mname)
- del proPatches["TARGET"]
-
- try:
- del makefilePatches["TARGET"]
- except:
- pass
if sipVersion == 3:
# Compile the Python part of the module.
pyname = mname + ".py"
inform("Compiling %s." % (pyname))
- py_compile.compile(pyname)
+ py_compile.compile(addBuildDir(pyname,mname))
- popDir(olddir)
+ if cpu_pool:
+ cpu_pool.release()
def pushDir(newdir):
"""Change to a new directory and return the old one.
@@ -1226,27 +1241,33 @@
olddir is the old directory.
"""
os.chdir(olddir)
-def runProgram(prog,argv = [],fatal = 1):
+def runProgram(prog,argv = [],fatal = 1,wd = None):
"""Execute a program.
prog is the name of the program.
argv is the optional list of program arguments.
fatal is non-zero if an error is considered fatal.
+ wd is an optional build directory.
Returns the error code.
"""
# Use the basename to avoid problems with pathnames containing spaces.
- argv.insert(0,os.path.basename(prog))
+ #argv.insert(0,os.path.basename(prog))
try:
- rc = os.spawnv(os.P_WAIT,prog,argv)
- except AttributeError:
- argv[0] = prog
- rc = os.system(string.join(argv))
+ # rc = os.spawnv(os.P_WAIT,prog,argv)
+ run = prog + ' ' + string.join(argv)
+ if wd:
+ run = 'cd ' + wd + ' >/dev/null && ' + run
+ #print "run: <%s>" % run
+ rc = os.system (run)
+ #except AttributeError:
+ # argv[0] = prog
+ # rc = os.system(string.join(argv))
except:
raise
if rc != 0 and fatal:
error("%s failed with an exit code of %d." % (prog,rc))
@@ -1274,46 +1295,63 @@
time.sleep(1)
return runProgram(makeBin,argv,fatal)
-def buildMakefile(pro,catcpp = 0):
+def buildMakefile(pro,catcpp = 0,_propat = None,_makepat = None,wd = None):
+
"""Run qmake or tmake to create a Makefile from a project file.
pro is the name of the project file.
catcpp is non-zero if the project file is for a module whose C++ files are
to be concatenated.
+ _propat is an optional project patch dictionary.
+ _makepat is an optional project patch dictionary.
+ wd is an optional build directory.
"""
- global makefileGen, proPatches, makefilePatches
+ global makefileGen
+
+ if not _propat:
+ global proPatches
+ _propat = proPatches
+ if not _makepat:
+ global makefilePatches
+ _makepat = makefilePatches
+ pf = addBuildDir(pro,wd)
# Preserve the original project file.
- probak = pro + ".bak"
- patchFile(pro,proPatches,probak)
+ probak = pf + ".bak"
+ patchFile(pf,_propat,probak)
if catcpp:
- catFiles(pro)
+ catFiles(pro,wd)
# Make sure that the Makefile has a later timestamp than the project file.
time.sleep(1)
- runProgram(makefileGen,['-o', 'Makefile', pro])
- os.remove(pro)
- os.rename(probak,pro)
+ runProgram(makefileGen,['-o', 'Makefile', pro],1,wd)
+ # save Makefile for debug purposes
+ #os.rename(pf, pf + ".patched")
+ os.remove(pf)
+ os.rename(probak,pf)
- patchFile("Makefile",makefilePatches)
+ mf = addBuildDir("Makefile",wd)
+ patchFile(mf,_makepat)
-def catFiles(pro):
+def catFiles(pro,wd = None):
"""Concatenate a modules C++ source files.
pro is the name of the project file.
+ wd is an optional build directory.
"""
# Extract the module name from the project file name.
mname = os.path.splitext(pro)[0]
inform("Concatenating the C++ files for the %s module." % (mname))
+ pro = addBuildDir(pro,wd)
f = open(pro,"r")
buf = f.read()
f.close()
pat = re.compile("^SOURCES =.*.cpp\n",re.M | re.S)
@@ -1333,19 +1371,22 @@
# Do each piece.
while srclist:
pname = "%shuge%d.cpp" % (mname, p)
plist.append(pname)
- d = open(pname,"w")
+ d = open(addBuildDir(pname,wd),"w")
for cppfile in srclist[:nrinpiece]:
- f = open(cppfile,"r")
+ f = open(addBuildDir(cppfile,wd),"r")
d.write(f.read())
f.close()
d.close()
+ # avoid dangling last module
+ if nrinpiece + 1 == len(srclist):
+ nrinpiece = nrinpiece + 1
srclist = srclist[nrinpiece:]
p = p + 1
# Replace the C++ file names in the project file.
buf = re.sub(pat,"SOURCES = " + string.join(plist," ") + "\n",buf)
@@ -1364,11 +1405,15 @@
mname is the name of the PyQt module, or None if the target isn't for a
module.
"""
global sipVersion
- f = open("Makefile","a")
+ if mname:
+ mf = mname + os.sep + "Makefile"
+ else:
+ mf = "Makefile"
+ f = open(mf,"a")
f.write("\ninstall:\n")
if mname and sipVersion == 3:
global platCopy, modDir
@@ -1417,10 +1462,21 @@
# Parse the command line.
global progName
progName = os.path.basename(argv[0])
+ # check, if threading is available and desired.
+ try:
+ from threading import BoundedSemaphore, Thread
+ makeopts = os.environ["MAKEFLAGS"]
+ i = string.find(makeopts, "-j")
+ if i != -1:
+ global nrThreads
+ nrThreads = int(string.split(makeopts[i+2:])[0])
+ except:
+ pass
+
initGlobals()
try:
optlist, args = getopt.getopt(argv[1:],"ha:b:cd:e:f:g:i:j:l:m:n:o:p:q:r:s:t:uw")
except getopt.GetoptError:
@@ -1553,52 +1609,72 @@
# We don't need the temporary build directory anymore.
mkTempBuildDir(maindir)
global buildModules
- subdirs = ""
- for mname in buildModules:
- generateSource(mname,plattag,qttag)
- subdirs = subdirs + " " + mname
+ subdirs = []
+ if not nrThreads:
+ for mname in buildModules:
+ generateSource(mname, plattag, qttag)
+ subdirs.append(mname)
+ else:
+ cpu_pool = BoundedSemaphore(nrThreads)
+ threads = []
+ inform("Starting %d builder threads for %s" % (nrThreads, string.join(buildModules, " ")))
+ for mname in buildModules:
+ t = Thread(target=generateSource, name=mname, args=(mname, plattag, qttag, None, cpu_pool))
+ threads.append(t)
+ t.start()
+ subdirs.append(mname)
+
+ while 1:
+ for t in threads:
+ if t.isAlive():
+ t.join(1.0)
+ else:
+ del threads[threads.index(t)]
+ if not threads:
+ break
+ inform("Builder threads finished.")
# We handle the qtpe module explicitly rather than auto-detect. This is
# because it does things a bit differently and I'm too lazy to deal with it
# properly at the moment.
if qpeTag:
generateSource("qtpe",plattag,qttag,qpeTag)
- subdirs = subdirs + " qtpe"
+ subdirs.append("qtpe")
# See which version of pyuic to build.
global proPatches
proPatches["BINDIR"] = [re.compile("@BL_BINDIR@",re.M), escape(platBinDir)]
if qtVersion >= 0x030000:
inform("Creating Makefile for pyuic3.")
- subdirs = subdirs + " pyuic3"
+ subdirs.append("pyuic3")
olddir = pushDir("pyuic3")
elif qtVersion >= 0x020000:
inform("Creating Makefile for pyuic2.")
- subdirs = subdirs + " pyuic2"
+ subdirs.append("pyuic2")
olddir = pushDir("pyuic2")
buildMakefile("pyuic.pro")
fixInstallTarget()
popDir(olddir)
# Build pylupdate if Qt v3.0 or later.
if qtVersion >= 0x030000:
inform("Creating Makefile for pylupdate3.")
- subdirs = subdirs + " pylupdate3"
+ subdirs.append("pylupdate3")
olddir = pushDir("pylupdate3")
buildMakefile("pylupdate.pro")
fixInstallTarget()
popDir(olddir)
# Generate the top-level Makefile.
inform("Creating top level Makefile.")
- copyToFile("PyQt.pro","TEMPLATE = subdirs\nSUBDIRS =" + subdirs + "\n")
+ copyToFile("PyQt.pro","TEMPLATE = subdirs\nSUBDIRS =" + string.join(subdirs, " ") + "\n")
buildMakefile("PyQt.pro")
# Tell the user what to do next.
if explicitMake:
mk = makeBin
_______________________________________________
PyKDE mailing list PyKDE@mats.gmd.de
http://mats.gmd.de/mailman/listinfo/pykde
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic