[prev in list] [next in list] [prev in thread] [next in thread] 

List:       gentoo-portage-dev
Subject:    [gentoo-portage-dev] [PATCH] Add FEATURES=pkgdir-index-trusted
From:       Zac Medico <zmedico () gentoo ! org>
Date:       2019-08-20 1:54:56
Message-ID: 20190820015456.5093-1-zmedico () gentoo ! org
[Download RAW message or body]

Trust that the PKGDIR index file is valid, meaning that no packages
have been manually added or removed since the last call to emaint --fix
binhost. This feature eliminates overhead involved with detection of
packages that have been manually added or removed, which significantly
improves performance in some cases, such as when PKGDIR resides on
a high-latency network file system.

Bug: https://bugs.gentoo.org/688902
Signed-off-by: Zac Medico <zmedico@gentoo.org>
---
 lib/portage/const.py         |  1 +
 lib/portage/dbapi/bintree.py | 36 ++++++++++++++++++++++++++----------
 man/make.conf.5              |  8 ++++++++
 3 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/lib/portage/const.py b/lib/portage/const.py
index 0ed64a742..36b33af92 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -176,6 +176,7 @@ SUPPORTED_FEATURES       = frozenset([
 	"parallel-fetch",
 	"parallel-install",
 	"pid-sandbox",
+	"pkgdir-index-trusted",
 	"prelink-checksums",
 	"preserve-libs",
 	"protect-owned",
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 9c2d877e7..ba21e6d23 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -537,7 +537,8 @@ class binarytree(object):
 
 		self._populating = True
 		try:
-			update_pkgindex = self._populate_local()
+			update_pkgindex = self._populate_local(
+				reindex='pkgdir-index-trusted' not in self.settings.features)
 
 			if update_pkgindex and self.dbapi.writable:
 				# If the Packages file needs to be updated, then _populate_local
@@ -568,7 +569,14 @@ class binarytree(object):
 
 		self.populated = True
 
-	def _populate_local(self):
+	def _populate_local(self, reindex=True):
+		"""
+		Populates the binarytree with local package metadata.
+
+		@param reindex: detect added / modified / removed packages and
+			regenerate the index file if necessary
+		@type reindex: bool
+		"""
 		self.dbapi.clear()
 		_instance_key = self.dbapi._instance_key
 		# In order to minimize disk I/O, we never compute digests here.
@@ -580,9 +588,10 @@ class binarytree(object):
 			pkg_paths = {}
 			self._pkg_paths = pkg_paths
 			dir_files = {}
-			for parent, dir_names, file_names in os.walk(self.pkgdir):
-				relative_parent = parent[len(self.pkgdir)+1:]
-				dir_files[relative_parent] = file_names
+			if reindex:
+				for parent, dir_names, file_names in os.walk(self.pkgdir):
+					relative_parent = parent[len(self.pkgdir)+1:]
+					dir_files[relative_parent] = file_names
 
 			pkgindex = self._load_pkgindex()
 			if not self._pkgindex_version_supported(pkgindex):
@@ -597,8 +606,14 @@ class binarytree(object):
 				path = d.get("PATH")
 				if not path:
 					path = cpv + ".tbz2"
-				basename = os.path.basename(path)
-				basename_index.setdefault(basename, []).append(d)
+
+				if reindex:
+					basename = os.path.basename(path)
+					basename_index.setdefault(basename, []).append(d)
+				else:
+					instance_key = _instance_key(cpv)
+					pkg_paths[instance_key] = path
+					self.dbapi.cpv_inject(cpv)
 
 			update_pkgindex = False
 			for mydir, file_names in dir_files.items():
@@ -798,9 +813,10 @@ class binarytree(object):
 						d.pop("PATH", None)
 					metadata[_instance_key(mycpv)] = d
 
-			for instance_key in list(metadata):
-				if instance_key not in pkg_paths:
-					del metadata[instance_key]
+			if reindex:
+				for instance_key in list(metadata):
+					if instance_key not in pkg_paths:
+						del metadata[instance_key]
 
 			if update_pkgindex:
 				del pkgindex.packages[:]
diff --git a/man/make.conf.5 b/man/make.conf.5
index cc4e1eba8..9dcd35b2c 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -561,6 +561,14 @@ Supported only on Linux. Requires PID and mount namespace support
 in kernel. /proc is remounted inside the mount namespace to account
 for new PID namespace.
 .TP
+.B pkgdir\-index\-trusted
+Trust that the \fBPKGDIR\fR index file is valid, meaning that no packages
+have been manually added or removed since the last call to
+\fBemaint \-\-fix binhost\fR. This feature eliminates overhead involved
+with detection of packages that have been manually added or removed,
+which significantly improves performance in some cases, such as when
+\fBPKGDIR\fR resides on a high\-latency network file system.
+.TP
 .B prelink\-checksums
 If \fBprelink\fR(8) is installed then use it to undo any prelinks on files
 before computing checksums for merge and unmerge. This feature is
-- 
2.21.0


[prev in list] [next in list] [prev in thread] [next in thread]