[prev in list] [next in list] [prev in thread] [next in thread]
List: gentoo-commits
Subject: [gentoo-commits] proj/catalyst:master commit in: etc/, doc/, catalyst/base/, catalyst/
From: "Matt Turner" <mattst88 () gentoo ! org>
Date: 2020-04-30 22:56:22
Message-ID: 1587666035.4e908d9934f988c01782b7d4fa8cd9bfa8853efa.mattst88 () gentoo
[Download RAW message or body]
commit: 4e908d9934f988c01782b7d4fa8cd9bfa8853efa
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 9 03:37:16 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Apr 23 18:20:35 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4e908d99
catalyst: Remove snapcache support
Among the problems with snapcache were broken locking, and still needing
to pack and unpack snapshot tarballs. In a few commits, I'm going to
replace it with something significantly better.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
catalyst/base/stagebase.py | 103 +++++++++------------------------------------
catalyst/defaults.py | 3 --
doc/catalyst-config.5.txt | 10 +----
etc/catalyst.conf | 5 +--
4 files changed, 24 insertions(+), 97 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 1c0d98c6..275c07eb 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -152,7 +152,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.set_snapshot_path()
self.set_root_path()
self.set_source_path()
- self.set_snapcache_path()
self.set_chroot_path()
self.set_autoresume_path()
self.set_dest_path()
@@ -192,20 +191,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Setup our mount points.
self.mount = MOUNT_DEFAULTS.copy()
- self.mount['portdir']['source'] = self.settings['portdir']
- self.mount['portdir']['target'] = normpath(self.settings["repo_basedir"]
- + "/" +
- self.settings["repo_name"])
- self.mount['distdir']['source'] = self.settings['distdir']
- self.mount["distdir"]['target'] = self.settings["target_distdir"]
+ # Always unpack snapshot tarball
+ self.mount['portdir']['enable'] = False
- if "snapcache" not in self.settings["options"]:
- self.mount['portdir']['enable'] = False
- else:
- self.mount['portdir']['source'] = normpath("/".join([
- self.settings['snapshot_cache_path'],
- self.settings['repo_name']
- ]))
+ self.mount['distdir']['source'] = self.settings['distdir']
+ self.mount["distdir"]['target'] = self.settings['target_distdir']
# Configure any user specified options (either in catalyst.conf or on
# the command line).
@@ -461,19 +451,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["snapshot_path_hash"] = \
self.generate_hash(self.settings["snapshot_path"], "sha1")
- def set_snapcache_path(self):
- self.settings["snapshot_cache_path"] = \
- normpath(pjoin(self.settings["snapshot_cache"],
- self.settings["snapshot"]))
- if "snapcache" in self.settings["options"]:
- self.settings["snapshot_cache_path"] = \
- normpath(pjoin(self.settings["snapshot_cache"],
- self.settings["snapshot"]))
- self.snapcache_lock = \
- LockDir(self.settings["snapshot_cache_path"])
- log.info('Setting snapshot cache to %s',
- self.settings['snapshot_cache_path'])
-
def set_chroot_path(self):
"""
NOTE: the trailing slash has been removed
@@ -841,7 +818,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
unpack_info = self.decompressor.create_infodict(
source=self.settings["snapshot_path"],
- destination=self.settings["snapshot_cache_path"],
arch=self.settings["compressor_arch"],
other_options=self.settings["compressor_options"],
)
@@ -850,41 +826,22 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["repo_basedir"] + "/" + self.settings["repo_name"])
log.info('%s', self.settings['chroot_path'])
log.info('unpack_snapshot(), target_portdir = %s', target_portdir)
- if "snapcache" in self.settings["options"]:
- snapshot_cache_hash_path = pjoin(
- self.settings['snapshot_cache_path'], 'catalyst-hash')
- snapshot_cache_hash = fileutils.readfile(
- snapshot_cache_hash_path, True)
- unpack_info['mode'] = self.decompressor.determine_mode(
- unpack_info['source'])
-
- cleanup_msg = "Cleaning up invalid snapshot cache at \n\t" + \
- self.settings["snapshot_cache_path"] + \
- " (this can take a long time)..."
-
- if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
- log.info(
- 'Valid snapshot cache, skipping unpack of portage tree...')
- unpack = False
- else:
- cleanup_msg = \
- 'Cleaning up existing portage tree (this can take a long time)...'
- unpack_info['destination'] = normpath(
- self.settings["chroot_path"] + self.settings["repo_basedir"])
- unpack_info['mode'] = self.decompressor.determine_mode(
- unpack_info['source'])
-
- if "autoresume" in self.settings["options"] \
- and os.path.exists(target_portdir) \
- and self.resume.is_enabled("unpack_repo") \
- and self.settings["snapshot_path_hash"] == snapshot_hash:
- log.notice(
- 'Valid Resume point detected, skipping unpack of portage tree...')
- unpack = False
+ cleanup_msg = \
+ 'Cleaning up existing portage tree (this can take a long time)...'
+ unpack_info['destination'] = normpath(
+ self.settings["chroot_path"] + self.settings["repo_basedir"])
+ unpack_info['mode'] = self.decompressor.determine_mode(
+ unpack_info['source'])
+
+ if "autoresume" in self.settings["options"] \
+ and os.path.exists(target_portdir) \
+ and self.resume.is_enabled("unpack_repo") \
+ and self.settings["snapshot_path_hash"] == snapshot_hash:
+ log.notice(
+ 'Valid Resume point detected, skipping unpack of portage tree...')
+ unpack = False
if unpack:
- if "snapcache" in self.settings["options"]:
- self.snapcache_lock.write_lock()
if os.path.exists(target_portdir):
log.info('%s', cleanup_msg)
clear_dir(target_portdir)
@@ -893,16 +850,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
if not self.decompressor.extract(unpack_info):
log.error('%s', unpack_errmsg % unpack_info)
- if "snapcache" in self.settings["options"]:
- with open(snapshot_cache_hash_path, 'w') as myf:
- myf.write(self.settings["snapshot_path_hash"])
- else:
- log.info('Setting snapshot autoresume point')
- self.resume.enable("unpack_repo",
- data=self.settings["snapshot_path_hash"])
-
- if "snapcache" in self.settings["options"]:
- self.snapcache_lock.unlock()
+ log.info('Setting snapshot autoresume point')
+ self.resume.enable("unpack_repo",
+ data=self.settings["snapshot_path_hash"])
def config_profile_link(self):
log.info('Configuring profile link...')
@@ -967,9 +917,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
src = self.mount[x]['source']
log.debug('bind(); src = %s', src)
- if "snapcache" in self.settings["options"] and x == "portdir":
- self.snapcache_lock.read_lock()
- _cmd = None
if src == "maybe_tmpfs":
if "var_tmpfs_portage" in self.settings:
_cmd = ['mount', '-t', 'tmpfs',
@@ -1018,14 +965,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
ouch = 1
log.warning("Couldn't umount bind mount: %s", target)
- if "snapcache" in self.settings["options"] and x == "/var/db/repos/gentoo":
- try:
- # It's possible the snapshot lock object isn't created yet.
- # This is because mount safety check calls unbind before the
- # target is fully initialized
- self.snapcache_lock.unlock()
- except Exception:
- pass
if ouch:
# if any bind mounts really failed, then we need to raise
# this to potentially prevent an upcoming bash stage cleanup script
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 77133cb1..787a13cc 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -45,7 +45,6 @@ valid_config_file_values.extend([
"distcc",
"envscript",
"options",
- "snapshot_cache",
"VERBOSE",
])
@@ -70,7 +69,6 @@ confdefaults = {
"repo_basedir": "/var/db/repos",
"repo_name": "gentoo",
"sharedir": "/usr/share/catalyst",
- "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
"snapshot_name": "%(repo_name)s-",
"shdir": "%(sharedir)s/targets",
"source_matching": "strict",
@@ -162,5 +160,4 @@ option_messages = {
"pkgcache": "Package cache support enabled.",
"purge": "Purge support enabled.",
"seedcache": "Seed cache support enabled.",
- "snapcache": "Snapshot cache support enabled.",
}
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index a0cd0490..44c905d7 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -27,7 +27,7 @@ and empty lines are interpreted as comments. For example:
digests="auto"
envscript="/etc/catalyst/catalystrc"
-options="autoresume bindist kerncache pkgcache seedcache snapcache"
+options="autoresume bindist kerncache pkgcache seedcache"
# source repo settings
portdir="/var/db/repos/gentoo"
@@ -56,7 +56,7 @@ export FOO="bar"
**options*::
Set different build-time options (example: `autoresume bindist
-kerncache pkgcache seedcache snapcache`). Supported values:
+kerncache pkgcache seedcache`). Supported values:
+
--
autoresume::
@@ -95,12 +95,6 @@ seedcache::
Use the build output of a previous target if it exists to speed up the
creation of a new stage. This avoids unpacking the seed tarball.
-snapcache::
-Cache the snapshot so that it can be bind-mounted into the chroot.
-WARNING: moving parts of the portage tree from within fsscript *will*
-break your cache. The cache is unlinked before any empty or rm
-processing.
-
versioned_cache::
Name the cache directories (packagecache, kerncache) based on the version of a
spec file.
diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index 6d530a39..6a499038 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -42,9 +42,6 @@ envscript="/etc/catalyst/catalystrc"
# (used on installcd-stage2 and stage4 targets)
# seedcache = use the build output of a previous target if it exists to speed up
# the copy
-# snapcache = cache the snapshot so that it can be bind-mounted into the chroot.
-# WARNING: moving parts of the portage tree from within fsscript *will* break
-# your cache. The cache is unlinked before any empty or rm processing, though.
# sticky-config = enables the code that will keep any internal 'catalyst_use' flags
# added to the USE= for building the stage. These are usually added for legal
# or specific needs in building the the early stage. Mostly it is the
@@ -53,7 +50,7 @@ envscript="/etc/catalyst/catalystrc"
# files or directories.
#
# (These options can be used together)
-options="autoresume bindist kerncache pkgcache seedcache snapcache"
+options="autoresume bindist kerncache pkgcache seedcache"
# Source portdir specifies the source portage tree used by the snapshot target.
portdir="/var/db/repos/gentoo"
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic