[csw-devel] SF.net SVN: gar:[9143] csw/mgar/gar/v2-git
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sat Mar 13 20:00:51 CET 2010
Revision: 9143
http://gar.svn.sourceforge.net/gar/?rev=9143&view=rev
Author: wahwah
Date: 2010-03-13 19:00:51 +0000 (Sat, 13 Mar 2010)
Log Message:
-----------
mGAR v2-git, more merge from v2.
Modified Paths:
--------------
csw/mgar/gar/v2-git/lib/python/checkpkg.py
csw/mgar/gar/v2-git/lib/python/checkpkg_test.py
csw/mgar/gar/v2-git/lib/python/package_checks.py
Removed Paths:
-------------
csw/mgar/gar/v2-git/lib/checkpkg.d/checkpkg-libs.py
Property Changed:
----------------
csw/mgar/gar/v2-git/
csw/mgar/gar/v2-git/pkglib/csw/depend
Property changes on: csw/mgar/gar/v2-git
___________________________________________________________________
Modified: svn:mergeinfo
- /csw/mgar/gar/v2:4936-6678,6915-9127
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-checkpkg-stats:8454-8649
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-dirpackage:8125-8180
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-skayser:6087-6132
+ /csw/mgar/gar/v2:4936-6678,6915-9142
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-checkpkg-stats:8454-8649
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-dirpackage:8125-8180
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-skayser:6087-6132
Deleted: csw/mgar/gar/v2-git/lib/checkpkg.d/checkpkg-libs.py
===================================================================
--- csw/mgar/gar/v2-git/lib/checkpkg.d/checkpkg-libs.py 2010-03-13 18:13:13 UTC (rev 9142)
+++ csw/mgar/gar/v2-git/lib/checkpkg.d/checkpkg-libs.py 2010-03-13 19:00:51 UTC (rev 9143)
@@ -1,185 +0,0 @@
-#!/opt/csw/bin/python2.6
-#
-# $Id$
-
-"""A check for dependencies between shared libraries.
-This is currently more of a prototype than a mature program, but it has some
-unit tests and it appears to be working. The main problem is that it's not
-divided into smaller testable sections.
-"""
-
-import os
-import os.path
-import copy
-import re
-import subprocess
-import logging
-import sys
-import textwrap
-from Cheetah import Template
-
-CHECKPKG_MODULE_NAME = "shared library linking consistency"
-
-# The following bit of code sets the correct path to Python libraries
-# distributed with GAR.
-path_list = [os.path.dirname(__file__),
- "..", "..", "lib", "python"]
-sys.path.append(os.path.join(*path_list))
-import checkpkg
-import opencsw
-
-def CheckSharedLibraryConsistency(pkgs_data, debug):
- ws_re = re.compile(r"\s+")
- result_ok = True
- errors = []
- binaries = []
- binaries_by_pkgname = {}
- sonames_by_pkgname = {}
- pkg_by_any_filename = {}
- needed_sonames_by_binary = {}
- filenames_by_soname = {}
- for pkg_data in pkgs_data:
- binaries_base = [os.path.basename(x) for x in pkg_data["binaries"]]
- pkgname = pkg_data["basic_stats"]["pkgname"]
- binaries_by_pkgname[pkgname] = binaries_base
- binaries.extend(pkg_data["binaries"])
- for filename in pkg_data["all_filenames"]:
- pkg_by_any_filename[filename] = pkgname
- for binary_data in pkg_data["binaries_dump_info"]:
- binary_base_name = os.path.basename(binary_data["base_name"])
- needed_sonames_by_binary[binary_base_name] = binary_data
- filenames_by_soname[binary_data[checkpkg.SONAME]] = binary_base_name
-
- # Making the binaries unique
- binaries = set(binaries)
- isalist = pkg_data["isalist"]
-
- # Building indexes by soname to simplify further processing
- # These are indexes "by soname".
- (needed_sonames,
- binaries_by_soname,
- runpath_by_needed_soname) = checkpkg.BuildIndexesBySoname(
- needed_sonames_by_binary)
-
- pkgmap = checkpkg.SystemPkgmap()
- logging.debug("Determining the soname-package relationships.")
- # lines by soname is an equivalent of $EXTRACTDIR/shortcatalog
- runpath_data_by_soname = {}
- for soname in needed_sonames:
- runpath_data_by_soname[soname] = pkgmap.GetPkgmapLineByBasename(soname)
- lines_by_soname = checkpkg.GetLinesBySoname(
- runpath_data_by_soname, needed_sonames, runpath_by_needed_soname, isalist)
-
- # Creating a map from files to packages.
- pkgs_by_filename = {}
- for soname, line in lines_by_soname.iteritems():
- # TODO: Find all the packages, not just the last field.
- fields = re.split(ws_re, line.strip())
- # For now, we'll assume that the last field is the package.
- pkgname = fields[-1]
- pkgs_by_filename[soname] = pkgname
-
- # A shared object dependency/provisioning report, plus checking.
- #
- # This section is somewhat overlapping with checkpkg.AnalyzeDependencies(),
- # it has a different purpose: it reports the relationships between shared
- # libraries, binaries using them and packages providing them. Ideally, the
- # same bit of code would do both checking and reporting.
- #
- # TODO: Rewrite this using cheetah templates
- if debug and needed_sonames:
- print "Analysis of sonames needed by the package set:"
- binaries_with_missing_sonames = set([])
- for soname in needed_sonames:
- logging.debug("Analyzing: %s", soname)
- if soname in filenames_by_soname:
- print "%s is provided by the package itself" % soname
- elif soname in lines_by_soname:
- print ("%s is provided by %s and required by:"
- % (soname,
- pkgs_by_filename[soname]))
- filename_lines = " ".join(sorted(binaries_by_soname[soname]))
- for line in textwrap.wrap(filename_lines, 70):
- print " ", line
- else:
- print ("%s is required by %s, but we don't know what provides it."
- % (soname, binaries_by_soname[soname]))
- for binary in binaries_by_soname[soname]:
- binaries_with_missing_sonames.add(binary)
- if soname in checkpkg.ALLOWED_ORPHAN_SONAMES:
- print "However, it's a whitelisted soname."
- else:
- pass
- # The error checking needs to be unified: done in one place only.
- # errors.append(
- # checkpkg.CheckpkgTag(
- # "%s is required by %s, but "
- # "we don't know what provides it."
- # % (soname, binaries_by_soname[soname])))
- if binaries_with_missing_sonames:
- print "The following are binaries with missing sonames:"
- binary_lines = " ".join(sorted(binaries_with_missing_sonames))
- for line in textwrap.wrap(binary_lines, 70):
- print " ", line
- print
-
- dependent_pkgs = {}
- for checker in pkgs_data:
- pkgname = checker["basic_stats"]["pkgname"]
- declared_dependencies = dict(checker["depends"])
- missing_deps, surplus_deps, orphan_sonames = checkpkg.AnalyzeDependencies(
- pkgname,
- declared_dependencies,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_filename,
- filenames_by_soname,
- pkg_by_any_filename)
- namespace = {
- "pkgname": pkgname,
- "missing_deps": missing_deps,
- "surplus_deps": surplus_deps,
- "orphan_sonames": orphan_sonames,
- }
- t = Template.Template(checkpkg.REPORT_TMPL, searchList=[namespace])
- print unicode(t)
-
- for soname in orphan_sonames:
- errors.append(
- checkpkg.CheckpkgTag(
- pkgname,
- "orphan-soname",
- soname))
- for missing_dep in missing_deps:
- errors.append(
- checkpkg.CheckpkgTag(
- pkgname,
- "missing-dependency",
- missing_dep))
- return errors
-
-
-def main():
- options, args = checkpkg.GetOptions()
- md5sums = args
- # CheckpkgManager class abstracts away things such as the collection of
- # results.
- check_manager = checkpkg.CheckpkgManager(CHECKPKG_MODULE_NAME,
- options.stats_basedir,
- md5sums,
- options.debug)
-
- check_manager.RegisterSetCheck(CheckSharedLibraryConsistency)
-
- exit_code, screen_report, tags_report = check_manager.Run()
- f = open(options.output, "w")
- f.write(tags_report)
- f.close()
- print screen_report.strip()
- sys.exit(exit_code)
-
-
-if __name__ == '__main__':
- main()
-
-# vim:set sw=2 ts=2 sts=2 expandtab:
Modified: csw/mgar/gar/v2-git/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2-git/lib/python/checkpkg.py 2010-03-13 18:13:13 UTC (rev 9142)
+++ csw/mgar/gar/v2-git/lib/python/checkpkg.py 2010-03-13 19:00:51 UTC (rev 9143)
@@ -31,7 +31,7 @@
DO_NOT_REPORT_MISSING = set([])
DO_NOT_REPORT_MISSING_RE = [r"SUNW.*", r"\*SUNW.*"]
DUMP_BIN = "/usr/ccs/bin/dump"
-PSTAMP_RE = r"(?P<username>\w)+@(?P<hostname>[\w\.]+)-(?P<timestamp>\d+)"
+PSTAMP_RE = r"(?P<username>\w+)@(?P<hostname>[\w\.-]+)-(?P<timestamp>\d+)"
DESCRIPTION_RE = r"^([\S]+) - (.*)$"
SYSTEM_SYMLINKS = (
@@ -816,7 +816,11 @@
if not self.system_pkgmap:
self.system_pkgmap = SystemPkgmap()
+ def GetPkgmapLineByBasename(self, basename):
+ """Proxies calls to self.system_pkgmap."""
+ return self.system_pkgmap.GetPkgmapLineByBasename(basename)
+
class IndividualCheckInterface(CheckInterfaceBase):
"""To be passed to the checking functions.
@@ -833,9 +837,10 @@
self.errors.append(tag)
-def SetCheckInterface(object):
+class SetCheckInterface(CheckInterfaceBase):
"""To be passed to set checking functions."""
- def __init__(self, system_pkgmap):
+
+ def __init__(self, system_pkgmap=None):
super(SetCheckInterface, self).__init__(system_pkgmap)
self.errors = []
@@ -872,19 +877,21 @@
self._RegisterIndividualCheck(member)
elif member_name.startswith("SetCheck"):
logging.debug("Registering set check %s", repr(member_name))
- self._RegisterIndividualCheck(member)
+ self._RegisterSetCheck(member)
def GetAllTags(self, packages_data):
errors = {}
+ # TODO: Configure the logger with the logging level.
logging_level = logging.INFO
if self.debug:
logging_level = logging.DEBUG
+ pkgmap = SystemPkgmap()
# Individual checks
for pkg_data in packages_data:
for function in self.individual_checks:
all_stats = pkg_data.GetAllStats()
pkgname = all_stats["basic_stats"]["pkgname"]
- check_interface = IndividualCheckInterface(pkgname)
+ check_interface = IndividualCheckInterface(pkgname, pkgmap)
logger = logging.getLogger("%s-%s" % (pkgname, function.__name__))
logger.debug("Calling %s", function.__name__)
function(all_stats, check_interface, logger=logger)
@@ -894,7 +901,7 @@
for function in self.set_checks:
pkgs_data = [x.GetAllStats() for x in packages_data]
logger = logging.getLogger("SetCheck-%s" % (function.__name__,))
- check_interface = SetCheckInterface()
+ check_interface = SetCheckInterface(pkgmap)
logger.debug("Calling %s", function.__name__)
function(pkgs_data, check_interface, logger)
if check_interface.errors:
Modified: csw/mgar/gar/v2-git/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2-git/lib/python/checkpkg_test.py 2010-03-13 18:13:13 UTC (rev 9142)
+++ csw/mgar/gar/v2-git/lib/python/checkpkg_test.py 2010-03-13 19:00:51 UTC (rev 9143)
@@ -1,6 +1,7 @@
#!/opt/csw/bin/python2.6
# $Id$
+import re
import unittest
import mox
import difflib
@@ -827,6 +828,15 @@
result = "Maciej Blizinski"
self.assertEqual(result, checkpkg.ExtractMaintainerName(data))
+ def testPstampRegex(self):
+ pstamp = "hson at solaris9s-csw-20100313144445"
+ expected = {
+ 'username': 'hson',
+ 'timestamp': '20100313144445',
+ 'hostname': 'solaris9s-csw'
+ }
+ self.assertEqual(expected, re.match(checkpkg.PSTAMP_RE, pstamp).groupdict())
+
if __name__ == '__main__':
unittest.main()
Modified: csw/mgar/gar/v2-git/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2-git/lib/python/package_checks.py 2010-03-13 18:13:13 UTC (rev 9142)
+++ csw/mgar/gar/v2-git/lib/python/package_checks.py 2010-03-13 19:00:51 UTC (rev 9143)
@@ -12,7 +12,9 @@
# error_mgr.ReportError("something-is-wrong")
import re
+import os
import checkpkg
+from Cheetah import Template
PATHS_ALLOWED_ONLY_IN = {
"CSWcommon": ["/opt",
@@ -100,6 +102,125 @@
"%s class=%s" % (entry["path"], entry["class"]))
+def SetCheckSharedLibraryConsistency(pkgs_data, error_mgr, logger):
+ ws_re = re.compile(r"\s+")
+ result_ok = True
+ binaries = []
+ binaries_by_pkgname = {}
+ sonames_by_pkgname = {}
+ pkg_by_any_filename = {}
+ needed_sonames_by_binary = {}
+ filenames_by_soname = {}
+ for pkg_data in pkgs_data:
+ binaries_base = [os.path.basename(x) for x in pkg_data["binaries"]]
+ pkgname = pkg_data["basic_stats"]["pkgname"]
+ binaries_by_pkgname[pkgname] = binaries_base
+ binaries.extend(pkg_data["binaries"])
+ for filename in pkg_data["all_filenames"]:
+ pkg_by_any_filename[filename] = pkgname
+ for binary_data in pkg_data["binaries_dump_info"]:
+ binary_base_name = os.path.basename(binary_data["base_name"])
+ needed_sonames_by_binary[binary_base_name] = binary_data
+ filenames_by_soname[binary_data[checkpkg.SONAME]] = binary_base_name
+
+ # Making the binaries unique
+ binaries = set(binaries)
+ isalist = pkg_data["isalist"]
+
+ # Building indexes by soname to simplify further processing
+ # These are indexes "by soname".
+ (needed_sonames,
+ binaries_by_soname,
+ runpath_by_needed_soname) = checkpkg.BuildIndexesBySoname(
+ needed_sonames_by_binary)
+
+ logger.debug("Determining the soname-package relationships.")
+ # lines by soname is an equivalent of $EXTRACTDIR/shortcatalog
+ runpath_data_by_soname = {}
+ for soname in needed_sonames:
+ runpath_data_by_soname[soname] = error_mgr.GetPkgmapLineByBasename(soname)
+ lines_by_soname = checkpkg.GetLinesBySoname(
+ runpath_data_by_soname, needed_sonames, runpath_by_needed_soname, isalist)
+
+ # Creating a map from files to packages.
+ pkgs_by_filename = {}
+ for soname, line in lines_by_soname.iteritems():
+ # TODO: Find all the packages, not just the last field.
+ fields = re.split(ws_re, line.strip())
+ # For now, we'll assume that the last field is the package.
+ pkgname = fields[-1]
+ pkgs_by_filename[soname] = pkgname
+
+ # A shared object dependency/provisioning report, plus checking.
+ #
+ # This section is somewhat overlapping with checkpkg.AnalyzeDependencies(),
+ # it has a different purpose: it reports the relationships between shared
+ # libraries, binaries using them and packages providing them. Ideally, the
+ # same bit of code would do both checking and reporting.
+ #
+ # TODO: Rewrite this using cheetah templates
+ if False and needed_sonames:
+ print "Analysis of sonames needed by the package set:"
+ binaries_with_missing_sonames = set([])
+ for soname in needed_sonames:
+ logger.debug("Analyzing: %s", soname)
+ if soname in filenames_by_soname:
+ print "%s is provided by the package itself" % soname
+ elif soname in lines_by_soname:
+ print ("%s is provided by %s and required by:"
+ % (soname,
+ pkgs_by_filename[soname]))
+ filename_lines = " ".join(sorted(binaries_by_soname[soname]))
+ for line in textwrap.wrap(filename_lines, 70):
+ print " ", line
+ else:
+ print ("%s is required by %s, but we don't know what provides it."
+ % (soname, binaries_by_soname[soname]))
+ for binary in binaries_by_soname[soname]:
+ binaries_with_missing_sonames.add(binary)
+ if soname in checkpkg.ALLOWED_ORPHAN_SONAMES:
+ print "However, it's a whitelisted soname."
+ else:
+ pass
+ # The error checking needs to be unified: done in one place only.
+ # errors.append(
+ # checkpkg.CheckpkgTag(
+ # "%s is required by %s, but "
+ # "we don't know what provides it."
+ # % (soname, binaries_by_soname[soname])))
+ if binaries_with_missing_sonames:
+ print "The following are binaries with missing sonames:"
+ binary_lines = " ".join(sorted(binaries_with_missing_sonames))
+ for line in textwrap.wrap(binary_lines, 70):
+ print " ", line
+ print
+
+ dependent_pkgs = {}
+ for checker in pkgs_data:
+ pkgname = checker["basic_stats"]["pkgname"]
+ declared_dependencies = dict(checker["depends"])
+ missing_deps, surplus_deps, orphan_sonames = checkpkg.AnalyzeDependencies(
+ pkgname,
+ declared_dependencies,
+ binaries_by_pkgname,
+ needed_sonames_by_binary,
+ pkgs_by_filename,
+ filenames_by_soname,
+ pkg_by_any_filename)
+ namespace = {
+ "pkgname": pkgname,
+ "missing_deps": missing_deps,
+ "surplus_deps": surplus_deps,
+ "orphan_sonames": orphan_sonames,
+ }
+ t = Template.Template(checkpkg.REPORT_TMPL, searchList=[namespace])
+ print unicode(t)
+ for soname in orphan_sonames:
+ error_mgr.ReportError(pkgname, "orphan-soname", soname)
+ for missing_dep in missing_deps:
+ error_mgr.ReportError(pkgname, "missing-dependency", missing_dep)
+
+
def SetCheckDependencies(pkgs_data, error_mgr, logger):
"""Dependencies must be either installed in the system, or in the set."""
pass
Property changes on: csw/mgar/gar/v2-git/pkglib/csw/depend
___________________________________________________________________
Modified: svn:mergeinfo
- /csw/mgar/gar/v2/pkglib/csw/depend:7201-9127
/csw/mgar/gar/v2-checkpkg/pkglib/csw/depend:7722-7855
/csw/mgar/gar/v2-checkpkg-stats/pkglib/csw/depend:8454-8649
/csw/mgar/gar/v2-dirpackage/pkglib/csw/depend:8125-8180
+ /csw/mgar/gar/v2/pkglib/csw/depend:7201-9142
/csw/mgar/gar/v2-checkpkg/pkglib/csw/depend:7722-7855
/csw/mgar/gar/v2-checkpkg-stats/pkglib/csw/depend:8454-8649
/csw/mgar/gar/v2-dirpackage/pkglib/csw/depend:8125-8180
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list