[prev in list] [next in list] [prev in thread] [next in thread] 

List:       gentoo-portage-dev
Subject:    [gentoo-portage-dev] [PATCH] Add check of herds in metadata.xml (Revision: 15471)
From:       Sebastian Pipping <sping () gentoo ! org>
Date:       2010-02-27 2:52:04
Message-ID: 4B8888D4.6070503 () gentoo ! org
[Download RAW message or body]

Hello!


Motivated by an earlier patch by idl0r I've been working on an
etree-based herd check for repoman.  Right now it only checks for herds.
 As this is my first non-trivial contribution to repoman, I felt like
asking for review before extending it further.

In case you would do only minor modifications you can apply right away
by me.



Sebastian

["0001-Prepare-further-checks-on-metadata.xml.patch" (text/plain)]

>From 8bc0762fbae86e0a6eb21d381ee909a795895787 Mon Sep 17 00:00:00 2001
From: Sebastian Pipping <sebastian@pipping.org>
Date: Fri, 26 Feb 2010 22:10:43 +0100
Subject: [PATCH 1/2] Prepare further checks on metadata.xml

---
 bin/repoman              |   15 ++++++++++++---
 pym/repoman/utilities.py |    4 ++--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/bin/repoman b/bin/repoman
index 57e61f0..f6f8b05 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -1293,16 +1293,25 @@ for x in scanlist:
 	else:
 		metadata_bad = False
 
-		# load USE flags from metadata.xml
+		# read metadata.xml into memory
 		try:
 			f = open(os.path.join(checkdir, "metadata.xml"))
-			utilities.parse_metadata_use(f, muselist)
+			_metadata_xml = f.read()
 			f.close()
-		except (EnvironmentError, ParseError) as e:
+		except (EnvironmentError, ) as e:
 			metadata_bad = True
 			stats["metadata.bad"] += 1
 			fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
 			del e
+		else:
+			# load USE flags from metadata.xml
+			try:
+				utilities.parse_metadata_use(_metadata_xml, muselist)
+			except (EnvironmentError, ParseError) as e:
+				metadata_bad = True
+				stats["metadata.bad"] += 1
+				fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
+				del e
 
 		#Only carry out if in package directory or check forced
 		if xmllint_capable and not metadata_bad:
diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index 0f67e20..f82a377 100644
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -112,14 +112,14 @@ def have_profile_dir(path, maxdepth=3, filename="profiles.desc"):
 		path = normalize_path(path + "/..")
 		maxdepth -= 1
 
-def parse_metadata_use(mylines, uselist=None):
+def parse_metadata_use(metadata_xml_content, uselist=None):
 	"""
 	Records are wrapped in XML as per GLEP 56
 	returns a dict of the form a list of flags"""
 	if uselist is None:
 		uselist = []
 	try:
-		metadatadom = minidom.parse(mylines)
+		metadatadom = minidom.parseString(metadata_xml_content)
 	except ExpatError as e:
 		raise exception.ParseError("metadata.xml: %s" % (e,))
 
-- 
1.6.6


["0002-Add-check-of-herds-in-metadata.xml.patch" (text/plain)]

>From bb45dd228c145fb4855623e90cf4edaa88203f57 Mon Sep 17 00:00:00 2001
From: Sebastian Pipping <sebastian@pipping.org>
Date: Sat, 27 Feb 2010 03:37:45 +0100
Subject: [PATCH 2/2] Add check of herds in metadata.xml

---
 bin/repoman              |   16 ++++++++
 pym/repoman/herdbase.py  |   87 ++++++++++++++++++++++++++++++++++++++++++++++
 pym/repoman/utilities.py |   33 +++++++++++++++++-
 3 files changed, 135 insertions(+), 1 deletions(-)
 create mode 100644 pym/repoman/herdbase.py

diff --git a/bin/repoman b/bin/repoman
index f6f8b05..f6c9ce4 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -53,11 +53,13 @@ from portage import StringIO
 try:
 	from repoman.checks import run_checks
 	from repoman import utilities
+	from repoman.herdbase import make_herd_base
 except ImportError:
 	from os import path as osp
 	sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
 	from repoman.checks import run_checks
 	from repoman import utilities
+	from repoman.herdbase import make_herd_base
 
 from _emerge.Package import Package
 from _emerge.RootConfig import RootConfig
@@ -975,6 +977,11 @@ check_ebuild_notadded = not \
 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
 thirdpartymirrors = portage.flatten(list(repoman_settings.thirdpartymirrors().values()))
 
+try:
+	herd_base = make_herd_base(os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml"))
+except (EnvironmentError, ParseError) as e:
+	err(str(e))
+
 for x in scanlist:
 	#ebuilds and digests added to cvs respectively.
 	logging.info("checking package %s" % x)
@@ -1313,6 +1320,15 @@ for x in scanlist:
 				fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
 				del e
 
+			# Run other metadata.xml checkers
+			try:
+				utilities.check_metadata(_metadata_xml, herd_base)
+			except (EnvironmentError, ParseError, utilities.UnknownHerdsError) as e:
+				metadata_bad = True
+				stats["metadata.bad"] += 1
+				fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
+				del e
+
 		#Only carry out if in package directory or check forced
 		if xmllint_capable and not metadata_bad:
 			# xmlint can produce garbage output even on success, so only dump
diff --git a/pym/repoman/herdbase.py b/pym/repoman/herdbase.py
new file mode 100644
index 0000000..6c3cf64
--- /dev/null
+++ b/pym/repoman/herdbase.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+# repoman: Herd database analysis
+# Copyright 2010 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id: $
+
+import xml.etree.ElementTree as ET
+from xml.parsers.expat import ExpatError
+from portage.exception import ParseError
+
+__all__ = [
+	"make_herd_base"
+]
+
+_SPECIAL_HERDS = set(('no-herd', 'maintainer-needed', 'maintainer-wanted'))
+
+def _make_email(nick_name):
+	if not nick_name.endswith('@gentoo.org'):
+		nick_name = nick_name + '@gentoo.org'
+	return nick_name
+
+
+class HerdBase(object):
+	def __init__(self, herd_to_emails, all_emails):
+		self.herd_to_emails = herd_to_emails
+		self.all_emails = all_emails
+
+	def known_herd(self, herd_name):
+		if herd_name in _SPECIAL_HERDS:
+			return True
+		return herd_name in self.herd_to_emails
+
+	def known_maintainer(self, nick_name):
+		return _make_email(nick_name) in self.all_emails
+
+	def maintainer_in_herd(self, nick_name, herd_name):
+		return _make_email(nick_name) in self.herd_to_emails[herd_name]
+
+
+def make_herd_base(filename):
+	herd_to_emails = dict()
+	all_emails = set()
+
+	try:
+		xml_tree = ET.parse(filename)
+	except ExpatError as e:
+		raise ParseError("metadata.xml: " + str(e))
+	
+	herds = xml_tree.findall('herd')
+	for h in herds:
+		_herd_name = h.find('name')
+		if _herd_name is None:
+			continue
+		herd_name = _herd_name.text.strip()
+		del _herd_name
+
+		maintainers = h.findall('maintainer')
+		herd_emails = set()
+		for m in maintainers:
+			_m_email = m.find('email')
+			if _m_email is None:
+				continue
+			m_email = _m_email.text.strip()
+
+			herd_emails.add(m_email)
+			all_emails.add(m_email)
+
+		herd_to_emails[herd_name] = herd_emails
+
+	return HerdBase(herd_to_emails, all_emails)
+
+
+if __name__ == '__main__':
+	h = make_herd_base('/usr/portage/metadata/herds.xml')
+
+	assert(h.known_herd('sound'))
+	assert(not h.known_herd('media-sound'))
+
+	assert(h.known_maintainer('sping'))
+	assert(h.known_maintainer('sping@gentoo.org'))
+	assert(not h.known_maintainer('portage'))
+
+	assert(h.maintainer_in_herd('zmedico@gentoo.org', 'tools-portage'))
+	assert(not h.maintainer_in_herd('pva@gentoo.org', 'tools-portage'))
+
+	import pprint
+	pprint.pprint(h.herd_to_emails)
diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py
index f82a377..fa4ccab 100644
--- a/pym/repoman/utilities.py
+++ b/pym/repoman/utilities.py
@@ -15,7 +15,9 @@ __all__ = [
 	"get_commit_message_with_editor",
 	"get_commit_message_with_stdin",
 	"have_profile_dir",
-	"parse_metadata_use"
+	"parse_metadata_use",
+	"UnknownHerdsError",
+	"check_metadata"
 ]
 
 import codecs
@@ -30,6 +32,7 @@ except ImportError:
 from xml.dom import minidom
 from xml.dom import NotFoundErr
 from xml.parsers.expat import ExpatError
+import xml.etree.ElementTree as ET
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode
@@ -149,6 +152,34 @@ def parse_metadata_use(metadata_xml_content, uselist=None):
 	finally:
 		metadatadom.unlink()
 
+
+class UnknownHerdsError(ValueError):
+	def __init__(self, herd_names):
+		_plural = len(herd_names) != 1
+		super(UnknownHerdsError, self).__init__(
+			'Unknown %s %s' % (_plural and 'herds' or 'herd',
+			','.join('"%s"' % e for e in herd_names)))
+
+
+def check_metadata_herds(xml_tree, herd_base):
+	herd_nodes = xml_tree.findall('herd')
+	unknown_herds = [name for name in
+			(e.text.strip() for e in herd_nodes)
+			if not herd_base.known_herd(name)]
+
+	if unknown_herds:
+		raise UnknownHerdsError(unknown_herds)
+
+
+def check_metadata(metadata_xml_content, herd_base):
+	try:
+		xml_tree = ET.fromstring(metadata_xml_content)
+	except (ExpatError, ) as e:
+		raise exception.ParseError("metadata.xml: " + str(e))
+
+	check_metadata_herds(xml_tree, herd_base)
+
+
 def FindPackagesToScan(settings, startdir, reposplit):
 	""" Try to find packages that need to be scanned
 	
-- 
1.6.6



[prev in list] [next in list] [prev in thread] [next in thread]