[csw-devel] SF.net SVN: gar:[12113] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Wed Dec 29 01:10:52 CET 2010
Revision: 12113
http://gar.svn.sourceforge.net/gar/?rev=12113&view=rev
Author: wahwah
Date: 2010-12-29 00:10:51 +0000 (Wed, 29 Dec 2010)
Log Message:
-----------
checkpkg: Moving functions into a class
Functions that process NeedFile() data belong to the CheckpkgManager2 class.
Plus new unit tests, written while looking for a bug.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/checkpkg_lib.py
csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py
csw/mgar/gar/v2/lib/python/dependency_checks.py
csw/mgar/gar/v2/lib/python/dependency_checks_test.py
Modified: csw/mgar/gar/v2/lib/python/checkpkg_lib.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2010-12-29 00:10:15 UTC (rev 12112)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2010-12-29 00:10:51 UTC (rev 12113)
@@ -25,6 +25,7 @@
import cPickle
import dependency_checks
from sqlobject import sqlbuilder
+import re
class Error(Exception):
@@ -499,7 +500,7 @@
req_pkgs_reasons_by_pkgname.setdefault(pkgname, [])
(missing_deps_reasons_by_pkg,
surplus_deps,
- missing_dep_groups) = dependency_checks.ReportMissingDependencies(
+ missing_dep_groups) = self._ReportMissingDependencies(
checkpkg_interface, pkgname, declared_deps,
req_pkgs_reasons_by_pkgname[pkgname])
namespace = {
@@ -527,7 +528,129 @@
if alternatives:
messenger.SuggestGarLine("# (end of the list of alternative dependencies)")
+ def _ReportMissingDependencies(self,
+ error_mgr,
+ pkgname,
+ declared_deps,
+ req_pkgs_reasons):
+ """Processes data structures with dependency data and reports errors.
+ Processes data specific to a single package.
+
+ Args:
+ error_mgr: SetCheckInterface
+ pkgname: pkgname, a string
+ declared_deps: An iterable with declared dependencies
+ req_pkgs_reasons: Groups of reasons
+
+ data structure:
+ [
+ [
+ ("CSWfoo1", "reason"),
+ ("CSWfoo2", "reason"),
+ ],
+ [
+ ( ... ),
+ ]
+ ]
+ """
+ missing_reasons_by_pkg = {}
+ for reason_group in req_pkgs_reasons:
+ for pkg, reason in reason_group:
+ missing_reasons_by_pkg.setdefault(pkg, [])
+ if len(missing_reasons_by_pkg[pkg]) < 4:
+ missing_reasons_by_pkg[pkg].append(reason)
+ elif len(missing_reasons_by_pkg[pkg]) == 4:
+ missing_reasons_by_pkg[pkg].append("...and more.")
+ missing_dep_groups = self._MissingDepsFromReasonGroups(
+ req_pkgs_reasons, declared_deps)
+ missing_dep_groups = self._RemovePkgsFromMissing(pkgname, missing_dep_groups)
+ potential_req_pkgs = set(
+ (x for x, y in reduce(operator.add, req_pkgs_reasons, [])))
+ surplus_deps = self._GetSurplusDeps(pkgname, potential_req_pkgs, declared_deps)
+ # Using an index to avoid duplicated reasons.
+ missing_deps_reasons_by_pkg = []
+ missing_deps_idx = set()
+ for missing_deps in missing_dep_groups:
+ error_mgr.ReportErrorForPkgname(
+ pkgname, "missing-dependency", " or ".join(sorted(missing_deps)))
+ for missing_dep in missing_deps:
+ item = (missing_dep, tuple(missing_reasons_by_pkg[missing_dep]))
+ if item not in missing_deps_idx:
+ missing_deps_reasons_by_pkg.append(item)
+ missing_deps_idx.add(item)
+ for surplus_dep in surplus_deps:
+ error_mgr.ReportErrorForPkgname(pkgname, "surplus-dependency", surplus_dep)
+ return missing_deps_reasons_by_pkg, surplus_deps, missing_dep_groups
+
+ def _MissingDepsFromReasonGroups(self, reason_groups, declared_deps_set):
+ missing_dep_groups = []
+ for reason_group in reason_groups:
+ dependency_fulfilled = False
+ pkgnames = [x for x, y in reason_group]
+ for pkgname in pkgnames:
+ if pkgname in declared_deps_set:
+ dependency_fulfilled = True
+ break
+ if not dependency_fulfilled:
+ missing_dep_groups.append(pkgnames)
+ return missing_dep_groups
+
+ def _GetSurplusDeps(self, pkgname, potential_req_pkgs, declared_deps):
+ logging.debug("GetSurplusDeps(%s, potential_req_pkgs=%s, declared_deps=%s)",
+ pkgname, declared_deps, potential_req_pkgs)
+ # Surplus dependencies
+ # In actual use, there should always be some potential dependencies.
+ # assert potential_req_pkgs, "There should be some potential deps!"
+ surplus_deps = declared_deps.difference(potential_req_pkgs)
+ no_report_surplus = set()
+ for sp_regex in common_constants.DO_NOT_REPORT_SURPLUS:
+ for maybe_surplus in surplus_deps:
+ if re.match(sp_regex, maybe_surplus):
+ logging.debug(
+ "GetSurplusDeps(): Not reporting %s as surplus because it matches %s.",
+ maybe_surplus, sp_regex)
+ no_report_surplus.add(maybe_surplus)
+ surplus_deps = surplus_deps.difference(no_report_surplus)
+ # For some packages (such as dev packages) we don't report surplus deps at
+ # all.
+ if surplus_deps:
+ for regex_str in common_constants.DO_NOT_REPORT_SURPLUS_FOR:
+ if re.match(regex_str, pkgname):
+ logging.debug(
+ "GetSurplusDeps(): Not reporting any surplus because "
+ "it matches %s", regex_str)
+ surplus_deps = frozenset()
+ break
+ return surplus_deps
+
+ def _RemovePkgsFromMissing(self, pkgname, missing_dep_groups):
+ "Removes packages from the list of missing deps."
+ pkgs_to_remove = set()
+ missing_deps_flat = set(reduce(operator.add, missing_dep_groups, []))
+ for regex_str in common_constants.DO_NOT_REPORT_MISSING_RE:
+ regex = re.compile(regex_str)
+ for dep_pkgname in missing_deps_flat:
+ if re.match(regex, dep_pkgname):
+ pkgs_to_remove.add(dep_pkgname)
+
+ # Some packages might have suggestions to depend on themselves, e.g.
+ # CSWpython contains .py files, and checkpkg would suggest that it should
+ # depend on itself, if not for the following two lines of code.
+ if pkgname in missing_deps_flat:
+ pkgs_to_remove.add(pkgname)
+
+ logging.debug("Removing %s from the list of missing pkgs.", pkgs_to_remove)
+ new_missing_dep_groups = set()
+ for missing_deps_group in missing_dep_groups:
+ new_missing_deps_group = set()
+ for dep in missing_deps_group:
+ if dep not in pkgs_to_remove:
+ new_missing_deps_group.add(dep)
+ if new_missing_deps_group:
+ new_missing_dep_groups.add(frozenset(new_missing_deps_group))
+ return new_missing_dep_groups
+
def GetAllTags(self, stats_obj_list):
errors = {}
catalog = Catalog()
Modified: csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py 2010-12-29 00:10:15 UTC (rev 12112)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py 2010-12-29 00:10:51 UTC (rev 12113)
@@ -13,6 +13,7 @@
import mox
import test_base
import cPickle
+from testdata import stubs
from testdata.neon_stats import pkgstats as neon_stats
@@ -126,7 +127,178 @@
'# (end of the list of alternative dependencies)']
self.assertEquals(expected_gar_lines, gar_lines)
+ def test_ReportDependencies(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ checkpkg_interface_mock = self.mox.CreateMock(
+ checkpkg_lib.IndividualCheckInterface)
+ needed_files = [
+ ("CSWfoo", "/opt/csw/bin/needed_file", "reason1"),
+ ]
+ needed_pkgs = []
+ messenger_stub = stubs.MessengerStub()
+ declared_deps_by_pkgname = {
+ "CSWfoo": frozenset([
+ "CSWbar-1",
+ "CSWbar-2",
+ ]),
+ }
+ checkpkg_interface_mock.GetPkgByPath('/opt/csw/bin/needed_file').AndReturn(
+ ["CSWfoo-one", "CSWfoo-two"]
+ )
+ checkpkg_interface_mock.ReportErrorForPkgname(
+ 'CSWfoo', 'missing-dependency', 'CSWfoo-one or CSWfoo-two')
+ checkpkg_interface_mock.ReportErrorForPkgname(
+ 'CSWfoo', 'surplus-dependency', 'CSWbar-2')
+ checkpkg_interface_mock.ReportErrorForPkgname(
+ 'CSWfoo', 'surplus-dependency', 'CSWbar-1')
+ self.mox.ReplayAll()
+ m._ReportDependencies(checkpkg_interface_mock,
+ needed_files,
+ needed_pkgs,
+ messenger_stub,
+ declared_deps_by_pkgname)
+ def testSurplusDeps(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ potential_req_pkgs = set([u"CSWbar"])
+ declared_deps = set([u"CSWbar", u"CSWsurplus"])
+ expected = set(["CSWsurplus"])
+ self.assertEquals(
+ expected,
+ m._GetSurplusDeps("CSWfoo", potential_req_pkgs, declared_deps))
+
+ def testMissingDepsFromReasonGroups(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ reason_groups = [
+ [(u"CSWfoo1", ""),
+ (u"CSWfoo2", "")],
+ [(u"CSWbar", "")],
+ ]
+ declared_deps = set([u"CSWfoo2"])
+ expected = [[u"CSWbar"]]
+ result = m._MissingDepsFromReasonGroups(
+ reason_groups, declared_deps)
+ self.assertEqual(expected, result)
+
+ def testMissingDepsFromReasonGroupsTwo(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ reason_groups = [
+ [(u"CSWfoo1", "reason 1"),
+ (u"CSWfoo2", "reason 1")],
+ [(u"CSWbar", "reason 2")],
+ ]
+ declared_deps = set([])
+ expected = [[u'CSWfoo1', u'CSWfoo2'], [u'CSWbar']]
+ result = m._MissingDepsFromReasonGroups(
+ reason_groups, declared_deps)
+ self.assertEqual(result, expected)
+
+ def test_RemovePkgsFromMissing(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ missing_dep_groups = [['CSWfoo-one', 'CSWfoo']]
+ expected = set(
+ [
+ frozenset(['CSWfoo', 'CSWfoo-one']),
+ ]
+ )
+ result = m._RemovePkgsFromMissing("CSWbaz", missing_dep_groups)
+ self.assertEqual(expected, result)
+
+ def testReportMissingDependenciesOne(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ error_mgr_mock = self.mox.CreateMock(checkpkg_lib.IndividualCheckInterface)
+ declared_deps = frozenset([u"CSWfoo"])
+ req_pkgs_reasons = [
+ [
+ (u"CSWfoo", "reason 1"),
+ (u"CSWfoo-2", "reason 2"),
+ ],
+ [
+ ("CSWbar", "reason 3"),
+ ],
+ ]
+ error_mgr_mock.ReportErrorForPkgname(
+ 'CSWexamined', 'missing-dependency', 'CSWbar')
+ self.mox.ReplayAll()
+ m._ReportMissingDependencies(
+ error_mgr_mock, "CSWexamined", declared_deps, req_pkgs_reasons)
+
+ def testReportMissingDependenciesTwo(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ error_mgr_mock = self.mox.CreateMock(checkpkg_lib.IndividualCheckInterface)
+ declared_deps = frozenset([])
+ req_pkgs_reasons = [
+ [
+ (u"CSWfoo-1", "reason 1"),
+ (u"CSWfoo-2", "reason 1"),
+ ],
+ ]
+ error_mgr_mock.ReportErrorForPkgname(
+ 'CSWexamined', 'missing-dependency', u'CSWfoo-1 or CSWfoo-2')
+ self.mox.ReplayAll()
+ m._ReportMissingDependencies(
+ error_mgr_mock, "CSWexamined", declared_deps, req_pkgs_reasons)
+
+ def testReportMissingDependenciesIntegration(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ catalog_mock = self.mox.CreateMock(checkpkg_lib.Catalog)
+ checkpkg_interface = checkpkg_lib.IndividualCheckInterface(
+ "CSWfoo", "AlienOS5.2", "sparkle", "calcified", catalog_mock)
+ declared_deps_by_pkgname = {
+ "CSWfoo": frozenset(),
+ }
+ declared_deps = frozenset([])
+ pkgs_providing_path = ["CSWproviding-%02d" % x for x in range(20)]
+ catalog_mock.GetPkgByPath(
+ '/opt/csw/sbin',
+ 'AlienOS5.2',
+ 'sparkle',
+ 'calcified').AndReturn(pkgs_providing_path)
+ self.mox.ReplayAll()
+ checkpkg_interface.NeedFile("/opt/csw/sbin", "reason 1")
+ needed_files = checkpkg_interface.needed_files
+ needed_pkgs = checkpkg_interface.needed_pkgs
+ messenger_stub = stubs.MessengerStub()
+ m._ReportDependencies(
+ checkpkg_interface,
+ needed_files,
+ needed_pkgs,
+ messenger_stub,
+ declared_deps_by_pkgname)
+ self.assertEqual(1, len(checkpkg_interface.errors))
+ self.assertEqual(
+ " or ".join(sorted(pkgs_providing_path)),
+ checkpkg_interface.errors[0].tag_info)
+
+ def testReportMissingDependenciesSurplus(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ error_mgr_mock = self.mox.CreateMock(checkpkg_lib.IndividualCheckInterface)
+ declared_deps = frozenset([u"CSWfoo", u"CSWbar", u"CSWsurplus"])
+ req_pkgs_reasons = [
+ [
+ (u"CSWfoo", "reason 1"),
+ (u"CSWfoo-2", "reason 2"),
+ ],
+ [
+ ("CSWbar", "reason 3"),
+ ],
+ ]
+ error_mgr_mock.ReportErrorForPkgname(
+ 'CSWexamined', 'surplus-dependency', u'CSWsurplus')
+ self.mox.ReplayAll()
+ m._ReportMissingDependencies(
+ error_mgr_mock, "CSWexamined", declared_deps, req_pkgs_reasons)
+
+
class CheckpkgManager2DatabaseIntegrationTest(
test_base.SqlObjectTestMixin, unittest.TestCase):
Modified: csw/mgar/gar/v2/lib/python/dependency_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/dependency_checks.py 2010-12-29 00:10:15 UTC (rev 12112)
+++ csw/mgar/gar/v2/lib/python/dependency_checks.py 2010-12-29 00:10:51 UTC (rev 12113)
@@ -244,19 +244,6 @@
# logger.warning("New paths: %s" % pprint.pformat(pkg_by_path))
return pkg_by_path
-def MissingDepsFromReasonGroups(reason_groups, declared_deps_set):
- missing_dep_groups = []
- for reason_group in reason_groups:
- dependency_fulfilled = False
- pkgnames = [x for x, y in reason_group]
- for pkgname in pkgnames:
- if pkgname in declared_deps_set:
- dependency_fulfilled = True
- break
- if not dependency_fulfilled:
- missing_dep_groups.append(pkgnames)
- return missing_dep_groups
-
def SuggestLibraryPackage(error_mgr, messenger,
pkgname, catalogname,
description,
@@ -290,99 +277,4 @@
"# The end of %s definition" % pkgname)
-def GetSurplusDeps(pkgname, potential_req_pkgs, declared_deps):
- logging.debug("GetSurplusDeps(%s, potential_req_pkgs=%s, declared_deps=%s)",
- pkgname, declared_deps, potential_req_pkgs)
- # Surplus dependencies
- # In actual use, there should always be some potential dependencies.
- # assert potential_req_pkgs, "There should be some potential deps!"
- surplus_deps = declared_deps.difference(potential_req_pkgs)
- no_report_surplus = set()
- for sp_regex in common_constants.DO_NOT_REPORT_SURPLUS:
- for maybe_surplus in surplus_deps:
- if re.match(sp_regex, maybe_surplus):
- logging.debug(
- "GetSurplusDeps(): Not reporting %s as surplus because it matches %s.",
- maybe_surplus, sp_regex)
- no_report_surplus.add(maybe_surplus)
- surplus_deps = surplus_deps.difference(no_report_surplus)
- # For some packages (such as dev packages) we don't report surplus deps at
- # all.
- if surplus_deps:
- for regex_str in common_constants.DO_NOT_REPORT_SURPLUS_FOR:
- if re.match(regex_str, pkgname):
- logging.debug(
- "GetSurplusDeps(): Not reporting any surplus because "
- "it matches %s", regex_str)
- surplus_deps = frozenset()
- break
- return surplus_deps
-
-def ReportMissingDependencies(error_mgr, pkgname, declared_deps, req_pkgs_reasons):
- """Processes data structures with dependency data and reports errors.
-
- Processes data specific to a single package.
-
- Args:
- error_mgr: SetCheckInterface
- pkgname: pkgname, a string
- declared_deps: An iterable with declared dependencies
- req_pkgs_reasons: Groups of reasons
-
- data structure:
- [
- [
- ("CSWfoo1", "reason"),
- ("CSWfoo2", "reason"),
- ],
- [
- ( ... ),
- ]
- ]
- """
- missing_reasons_by_pkg = {}
- for reason_group in req_pkgs_reasons:
- for pkg, reason in reason_group:
- missing_reasons_by_pkg.setdefault(pkg, [])
- if len(missing_reasons_by_pkg[pkg]) < 4:
- missing_reasons_by_pkg[pkg].append(reason)
- elif len(missing_reasons_by_pkg[pkg]) == 4:
- missing_reasons_by_pkg[pkg].append("...and more.")
- missing_dep_groups = MissingDepsFromReasonGroups(
- req_pkgs_reasons, declared_deps)
- pkgs_to_remove = set()
- for regex_str in common_constants.DO_NOT_REPORT_MISSING_RE:
- regex = re.compile(regex_str)
- for dep_pkgname in reduce(operator.add, missing_dep_groups, []):
- if re.match(regex, dep_pkgname):
- pkgs_to_remove.add(dep_pkgname)
- if pkgname in reduce(operator.add, missing_dep_groups, []):
- pkgs_to_remove.add(pkgname)
- logging.debug("Removing %s from the list of missing pkgs.", pkgs_to_remove)
- new_missing_dep_groups = set()
- for missing_deps in missing_dep_groups:
- new_missing_deps = set()
- for dep in missing_deps:
- if dep not in pkgs_to_remove:
- new_missing_deps.add(dep)
- if new_missing_deps:
- new_missing_dep_groups.add(tuple(new_missing_deps))
- potential_req_pkgs = set(
- (x for x, y in reduce(operator.add, req_pkgs_reasons, [])))
- missing_dep_groups = new_missing_dep_groups
- surplus_deps = GetSurplusDeps(pkgname, potential_req_pkgs, declared_deps)
- # Using an index to avoid duplicated reasons.
- missing_deps_reasons_by_pkg = []
- missing_deps_idx = set()
- for missing_deps in missing_dep_groups:
- error_mgr.ReportErrorForPkgname(
- pkgname, "missing-dependency", " or ".join(missing_deps))
- for missing_dep in missing_deps:
- item = (missing_dep, tuple(missing_reasons_by_pkg[missing_dep]))
- if item not in missing_deps_idx:
- missing_deps_reasons_by_pkg.append(item)
- missing_deps_idx.add(item)
- for surplus_dep in surplus_deps:
- error_mgr.ReportErrorForPkgname(pkgname, "surplus-dependency", surplus_dep)
- return missing_deps_reasons_by_pkg, surplus_deps, missing_dep_groups
Modified: csw/mgar/gar/v2/lib/python/dependency_checks_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/dependency_checks_test.py 2010-12-29 00:10:15 UTC (rev 12112)
+++ csw/mgar/gar/v2/lib/python/dependency_checks_test.py 2010-12-29 00:10:51 UTC (rev 12113)
@@ -201,21 +201,6 @@
self.assertEquals(expected, result)
-class TestMissingDepsFromReasonGroups(unittest.TestCase):
-
- def testOne(self):
- reason_groups = [
- [(u"CSWfoo1", ""),
- (u"CSWfoo2", "")],
- [(u"CSWbar", "")],
- ]
- declared_deps = set([u"CSWfoo2"])
- expected = [[u"CSWbar"]]
- result = dependency_checks.MissingDepsFromReasonGroups(
- reason_groups, declared_deps)
- self.assertEqual(result, expected)
-
-
class TestLibraries(mox.MoxTestBase):
def setUp(self):
@@ -327,57 +312,5 @@
base_pkgname)
-class TestReportMissingDependencies(mox.MoxTestBase):
-
- def testReportOneError(self):
- error_mgr_mock = self.mox.CreateMock(checkpkg_lib.IndividualCheckInterface)
- declared_deps = frozenset([u"CSWfoo"])
- req_pkgs_reasons = [
- [
- (u"CSWfoo", "reason 1"),
- (u"CSWfoo-2", "reason 2"),
- ],
- [
- ("CSWbar", "reason 3"),
- ],
- ]
- error_mgr_mock.ReportErrorForPkgname(
- 'CSWexamined', 'missing-dependency', 'CSWbar')
- self.mox.ReplayAll()
- dependency_checks.ReportMissingDependencies(
- error_mgr_mock, "CSWexamined", declared_deps, req_pkgs_reasons)
-
- def testReportSurplus(self):
- error_mgr_mock = self.mox.CreateMock(checkpkg_lib.IndividualCheckInterface)
- declared_deps = frozenset([u"CSWfoo", u"CSWbar", u"CSWsurplus"])
- req_pkgs_reasons = [
- [
- (u"CSWfoo", "reason 1"),
- (u"CSWfoo-2", "reason 2"),
- ],
- [
- ("CSWbar", "reason 3"),
- ],
- ]
- error_mgr_mock.ReportErrorForPkgname(
- 'CSWexamined', 'surplus-dependency', u'CSWsurplus')
- self.mox.ReplayAll()
- dependency_checks.ReportMissingDependencies(
- error_mgr_mock, "CSWexamined", declared_deps, req_pkgs_reasons)
-
-
-class TestReportMissingDependencies(mox.MoxTestBase):
-
- def testSurplusDeps(self):
- potential_req_pkgs = set([u"CSWbar"])
- declared_deps = set([u"CSWbar", u"CSWsurplus"])
- expected = set(["CSWsurplus"])
- self.assertEquals(
- expected,
- dependency_checks.GetSurplusDeps("CSWfoo",
- potential_req_pkgs,
- declared_deps))
-
-
if __name__ == '__main__':
unittest.main()
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list