[prev in list] [next in list] [prev in thread] [next in thread]
List: gentoo-portage-dev
Subject: [gentoo-portage-dev] [PATCH] Use cached portage.getpid() function (bug 739540)
From: Zac Medico <zmedico () gentoo ! org>
Date: 2020-08-29 21:28:13
Message-ID: 20200829212813.18453-1-zmedico () gentoo ! org
[Download RAW message or body]
Use the cached portage.getpid() function to avoid unnecessary syscalls,
and update the cache after each call to os.fork() where the fork may
invoke portage APIs.
Bug: https://bugs.gentoo.org/739540
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
bin/quickpkg | 2 +-
lib/_emerge/EbuildBinpkg.py | 4 +++-
lib/_emerge/actions.py | 2 +-
lib/portage/_emirrordist/FetchTask.py | 4 ++--
lib/portage/cache/metadata.py | 4 +++-
lib/portage/elog/mod_mail_summary.py | 5 ++---
lib/portage/elog/mod_save_summary.py | 2 +-
lib/portage/locks.py | 2 +-
lib/portage/package/ebuild/doebuild.py | 2 +-
lib/portage/process.py | 16 +++++++++++-----
lib/portage/tests/locks/test_lock_nonblock.py | 1 +
.../futures/asyncio/test_wakeup_fd_sigchld.py | 2 +-
lib/portage/util/__init__.py | 4 ++--
lib/portage/util/_eventloop/EventLoop.py | 6 +++---
.../util/_eventloop/asyncio_event_loop.py | 4 ++--
lib/portage/util/_eventloop/global_event_loop.py | 7 +++----
lib/portage/util/locale.py | 1 +
lib/portage/util/movefile.py | 2 +-
lib/portage/util/socks5.py | 2 +-
lib/portage/xpak.py | 2 +-
20 files changed, 42 insertions(+), 32 deletions(-)
diff --git a/bin/quickpkg b/bin/quickpkg
index be7d1d7af..a171b3bd5 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout):
vardb.aux_update(cpv, update_metadata)
xpdata = xpak.xpak(dblnk.dbdir)
binpkg_tmpfile = os.path.join(bintree.pkgdir,
- cpv + ".tbz2." + str(os.getpid()))
+ cpv + ".tbz2." + str(portage.getpid()))
ensure_dirs(os.path.dirname(binpkg_tmpfile))
binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
try:
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index 6e098eb8a..879b3a9aa 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -3,6 +3,8 @@
from _emerge.CompositeTask import CompositeTask
from _emerge.EbuildPhase import EbuildPhase
+
+import portage
from portage import os
class EbuildBinpkg(CompositeTask):
@@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask):
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
binpkg_tmpfile = os.path.join(bintree.pkgdir,
- pkg.cpv + ".tbz2." + str(os.getpid()))
+ pkg.cpv + ".tbz2." + str(portage.getpid()))
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
self._binpkg_tmpfile = binpkg_tmpfile
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 063f5d4a0..a4ecfe43d 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -2623,7 +2623,7 @@ def ionice(settings):
if not ionice_cmd:
return
- variables = {"PID" : str(os.getpid())}
+ variables = {"PID" : str(portage.getpid())}
cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
try:
diff --git a/lib/portage/_emirrordist/FetchTask.py \
b/lib/portage/_emirrordist/FetchTask.py index 457ca2ac6..41f96b962 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -415,7 +415,7 @@ class FetchTask(CompositeTask):
self._fetch_tmp_dir_info = 'distfiles'
distdir = self.config.options.distfiles
- tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
+ tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % portage.getpid()
variables = {
"DISTDIR": distdir,
@@ -622,7 +622,7 @@ class FetchTask(CompositeTask):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
- (tail, os.getpid()))
+ (tail, portage.getpid()))
try:
try:
diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py
index db81b8ba1..bd1b70fa0 100644
--- a/lib/portage/cache/metadata.py
+++ b/lib/portage/cache/metadata.py
@@ -6,6 +6,8 @@ import errno
import re
import stat
from operator import attrgetter
+
+import portage
from portage import os
from portage import _encodings
from portage import _unicode_encode
@@ -122,7 +124,7 @@ class database(flat_hash.database):
s = cpv.rfind("/")
fp = os.path.join(self.location,cpv[:s],
- ".update.%i.%s" % (os.getpid(), cpv[s+1:]))
+ ".update.%i.%s" % (portage.getpid(), cpv[s+1:]))
try:
myf = open(_unicode_encode(fp,
encoding=_encodings['fs'], errors='strict'), 'wb')
diff --git a/lib/portage/elog/mod_mail_summary.py \
b/lib/portage/elog/mod_mail_summary.py index ac260880e..789f55f4d 100644
--- a/lib/portage/elog/mod_mail_summary.py
+++ b/lib/portage/elog/mod_mail_summary.py
@@ -6,7 +6,6 @@ import portage
from portage.exception import AlarmSignal, PortageException
from portage.localization import _
from portage.util import writemsg
-from portage import os
from portage import _encodings
from portage import _unicode_decode
@@ -22,7 +21,7 @@ def process(mysettings, key, logentries, fulltext):
time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
encoding=_encodings['content'], errors='replace')
header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on \
%(time)s:\n\n") % \
- {"pkg": key, "pid": os.getpid(), "time": time_str}
+ {"pkg": key, "pid": portage.getpid(), "time": time_str}
config_root = mysettings["PORTAGE_CONFIGROOT"]
# Copy needed variables from the config instance,
@@ -66,7 +65,7 @@ def _finalize(mysettings, items):
mysubject = mysubject.replace("${HOST}", socket.getfqdn())
mybody = _("elog messages for the following packages generated by "
- "process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": \
socket.getfqdn()} + "process %(pid)d on host %(host)s:\n") % {"pid": \
portage.getpid(), "host": socket.getfqdn()} for key in items:
mybody += "- %s\n" % key
diff --git a/lib/portage/elog/mod_save_summary.py \
b/lib/portage/elog/mod_save_summary.py index 946a1ad4c..6e24b608f 100644
--- a/lib/portage/elog/mod_save_summary.py
+++ b/lib/portage/elog/mod_save_summary.py
@@ -79,7 +79,7 @@ def process(mysettings, key, logentries, fulltext):
encoding=_encodings['content'], errors='replace')
elogfile.write(_(">>> Messages generated by process "
"%(pid)d on %(time)s for package %(pkg)s:\n\n") %
- {"pid": os.getpid(), "time": time_str, "pkg": key})
+ {"pid": portage.getpid(), "time": time_str, "pkg": key})
elogfile.write(_unicode_decode(fulltext))
elogfile.write("\n")
elogfile.close()
diff --git a/lib/portage/locks.py b/lib/portage/locks.py
index 701093024..1073343be 100644
--- a/lib/portage/locks.py
+++ b/lib/portage/locks.py
@@ -501,7 +501,7 @@ def unlockfile(mytuple):
def hardlock_name(path):
base, tail = os.path.split(path)
return os.path.join(base, ".%s.hardlock-%s-%s" %
- (tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
+ (tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid()))
def hardlink_is_mine(link, lock):
try:
diff --git a/lib/portage/package/ebuild/doebuild.py \
b/lib/portage/package/ebuild/doebuild.py index 7bb942966..3b1991b28 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1178,7 +1178,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, \
settings=None, debug=0, bintree = portage.db[mysettings['EROOT']]['bintree']
mysettings["PORTAGE_BINPKG_TMPFILE"] = \
bintree.getname(mysettings.mycpv) + \
- ".%s" % (os.getpid(),)
+ ".%s" % (portage.getpid(),)
bintree._ensure_dir(os.path.dirname(
mysettings["PORTAGE_BINPKG_TMPFILE"]))
else:
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 8d4cf164e..48548bacc 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -79,11 +79,11 @@ if _fd_dir is not None:
raise
return range(max_fd_limit)
-elif os.path.isdir("/proc/%s/fd" % os.getpid()):
+elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
# In order for this function to work in forked subprocesses,
# os.getpid() must be called from inside the function.
def get_open_fds():
- return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid())
+ return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
if fd.isdigit())
else:
@@ -363,12 +363,13 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, \
returnpid=False, # fork, so that the result is cached in the main process.
bool(groups)
- parent_pid = os.getpid()
+ parent_pid = portage.getpid()
pid = None
try:
pid = os.fork()
if pid == 0:
+ portage._ForkWatcher.hook(portage._ForkWatcher)
try:
_exec(binary, mycommand, opt_name, fd_pipes,
env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
@@ -386,7 +387,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, \
returnpid=False, sys.stderr.flush()
finally:
- if pid == 0 or (pid is None and os.getpid() != parent_pid):
+ # Don't used portage.getpid() here, due to a race with the above
+ # portage._ForkWatcher cache update.
+ if pid == 0 or (pid is None and _os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
@@ -603,7 +606,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
# it is done before we start forking children
if cgroup:
with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
- f.write('%d\n' % os.getpid())
+ f.write('%d\n' % portage.getpid())
# Unshare (while still uid==0)
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
@@ -640,6 +643,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
if unshare_pid:
main_child_pid = os.fork()
if main_child_pid == 0:
+ # The portage.getpid() cache may need to be updated here,
+ # in case the pre_exec function invokes portage APIs.
+ portage._ForkWatcher.hook(portage._ForkWatcher)
# pid namespace requires us to become init
binary, myargs = portage._python_interpreter, [
portage._python_interpreter,
diff --git a/lib/portage/tests/locks/test_lock_nonblock.py \
b/lib/portage/tests/locks/test_lock_nonblock.py index 02ba16ad9..3448b84f6 100644
--- a/lib/portage/tests/locks/test_lock_nonblock.py
+++ b/lib/portage/tests/locks/test_lock_nonblock.py
@@ -19,6 +19,7 @@ class LockNonblockTestCase(TestCase):
lock1 = portage.locks.lockfile(path)
pid = os.fork()
if pid == 0:
+ portage._ForkWatcher.hook(portage._ForkWatcher)
portage.locks._close_fds()
# Disable close_fds since we don't exec
# (see _setup_pipes docstring).
diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py \
b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py index \
e5b104e0f..c37a6338b 100644
--- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
+++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
@@ -39,7 +39,7 @@ proc = \
loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0', loop \
loop.run_until_complete(proc.wait())
for i in range(8192):
- os.kill(os.getpid(), signal.SIGCHLD)
+ os.kill(portage.getpid(), signal.SIGCHLD)
# Verify that the child watcher still works correctly
# (this will hang if it doesn't).
diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py
index c14c15fe8..0412b2b59 100644
--- a/lib/portage/util/__init__.py
+++ b/lib/portage/util/__init__.py
@@ -1266,7 +1266,7 @@ class atomic_ofstream(ObjectProxy):
if follow_links:
canonical_path = os.path.realpath(filename)
object.__setattr__(self, '_real_name', canonical_path)
- tmp_name = "%s.%i" % (canonical_path, os.getpid())
+ tmp_name = "%s.%i" % (canonical_path, portage.getpid())
try:
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
@@ -1281,7 +1281,7 @@ class atomic_ofstream(ObjectProxy):
# new error if necessary.
object.__setattr__(self, '_real_name', filename)
- tmp_name = "%s.%i" % (filename, os.getpid())
+ tmp_name = "%s.%i" % (filename, portage.getpid())
object.__setattr__(self, '_file',
open_func(_unicode_encode(tmp_name,
encoding=_encodings['fs'], errors='strict'),
diff --git a/lib/portage/util/_eventloop/EventLoop.py \
b/lib/portage/util/_eventloop/EventLoop.py index 94e637853..ff2b73255 100644
--- a/lib/portage/util/_eventloop/EventLoop.py
+++ b/lib/portage/util/_eventloop/EventLoop.py
@@ -188,7 +188,7 @@ class EventLoop:
self._sigchld_read = None
self._sigchld_write = None
self._sigchld_src_id = None
- self._pid = os.getpid()
+ self._pid = portage.getpid()
self._asyncio_wrapper = _PortageEventLoop(loop=self)
self._asyncio_child_watcher = _PortageChildWatcher(self)
@@ -431,7 +431,7 @@ class EventLoop:
# If this signal handler was not installed by the
# current process then the signal doesn't belong to
# this EventLoop instance.
- if os.getpid() == self._pid:
+ if portage.getpid() == self._pid:
os.write(self._sigchld_write, b'\0')
def _sigchld_io_cb(self, fd, events):
@@ -1026,7 +1026,7 @@ class EventLoop:
log_lines.append('{}: {}'.format(key, value))
logging.error('\n'.join(log_lines), exc_info=exc_info)
- os.kill(os.getpid(), signal.SIGTERM)
+ os.kill(portage.getpid(), signal.SIGTERM)
def call_exception_handler(self, context):
"""
diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py \
b/lib/portage/util/_eventloop/asyncio_event_loop.py index 605e7243b..836f1c30a 100644
--- a/lib/portage/util/_eventloop/asyncio_event_loop.py
+++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
@@ -69,7 +69,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
# in order to ensure that emerge exits immediately (though
# uncleanly).
signal.signal(signal.SIGTERM, signal.SIG_DFL)
- os.kill(os.getpid(), signal.SIGTERM)
+ os.kill(portage.getpid(), signal.SIGTERM)
def _create_future(self):
"""
@@ -117,7 +117,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
self._wakeup_fd = -1
# Account for any signals that may have arrived between
# set_wakeup_fd calls.
- os.kill(os.getpid(), signal.SIGCHLD)
+ os.kill(portage.getpid(), signal.SIGCHLD)
try:
return self._loop.run_until_complete(future)
finally:
diff --git a/lib/portage/util/_eventloop/global_event_loop.py \
b/lib/portage/util/_eventloop/global_event_loop.py index 1db958d2e..21a1d1970 100644
--- a/lib/portage/util/_eventloop/global_event_loop.py
+++ b/lib/portage/util/_eventloop/global_event_loop.py
@@ -1,13 +1,12 @@
# Copyright 2012-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import os
-
+import portage
from .EventLoop import EventLoop
from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
-_MAIN_PID = os.getpid()
+_MAIN_PID = portage.getpid()
_instances = {}
@@ -17,7 +16,7 @@ def global_event_loop():
belongs exclusively to the current process.
"""
- pid = os.getpid()
+ pid = portage.getpid()
instance = _instances.get(pid)
if instance is not None:
return instance
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index 99c8f7ae7..58c687139 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -102,6 +102,7 @@ def check_locale(silent=False, env=None):
pid = os.fork()
if pid == 0:
+ portage._ForkWatcher.hook(portage._ForkWatcher)
try:
if env is not None:
try:
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index 4f8054f29..a251d369d 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -209,7 +209,7 @@ def movefile(src, dest, newmtime=None, sstat=None, \
mysettings=None, if hardlink_candidates:
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
- (tail, os.getpid()))
+ (tail, portage.getpid()))
try:
os.unlink(hardlink_tmp)
except OSError as e:
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
index 9f22c1dbe..a76d1a741 100644
--- a/lib/portage/util/socks5.py
+++ b/lib/portage/util/socks5.py
@@ -42,7 +42,7 @@ class ProxyManager:
portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs)
self.socket_path = os.path.join(tmpdir,
- '.portage.%d.net.sock' % os.getpid())
+ '.portage.%d.net.sock' % portage.getpid())
server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
spawn_kwargs = {}
# The portage_uid check solves EPERM failures in Travis CI.
diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py
index 2a4bcda21..9063c4c56 100644
--- a/lib/portage/xpak.py
+++ b/lib/portage/xpak.py
@@ -325,7 +325,7 @@ class tbz2:
self.scan() # Don't care about condition... We'll rewrite the data anyway.
if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
- tmp_fname = "%s.%d" % (self.file, os.getpid())
+ tmp_fname = "%s.%d" % (self.file, portage.getpid())
copyfile(self.file, tmp_fname)
try:
portage.util.apply_stat_permissions(self.file, self.filestat)
--
2.25.3
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic