[csw-devel] SF.net SVN: gar:[10460] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Tue Jul 6 19:43:26 CEST 2010
Revision: 10460
http://gar.svn.sourceforge.net/gar/?rev=10460&view=rev
Author: wahwah
Date: 2010-07-06 17:43:26 +0000 (Tue, 06 Jul 2010)
Log Message:
-----------
mGAR v2: checkpkg, optimizing stats analysis, allowing to skip applying of the overrides.
Modified Paths:
--------------
csw/mgar/gar/v2/bin/checkpkg
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/checkpkg_test.py
csw/mgar/gar/v2/lib/python/dependency_checks.py
Modified: csw/mgar/gar/v2/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg 2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/bin/checkpkg 2010-07-06 17:43:26 UTC (rev 10460)
@@ -107,8 +107,9 @@
SKIP_STATS_COLLECTION=0
MD5_SUMS_CATALOG_FILE=""
INSTALL_CONTENTS_FILES="/var/sadm/install/contents"
+ANALYZE=1
-while getopts hsdNM:o:c: opt; do
+while getopts hsdNM:o:c:A opt; do
case "${opt}" in
c)
INSTALL_CONTENTS_FILES="${INSTALL_CONTENTS_FILES} ${OPTARG}"
@@ -131,6 +132,9 @@
M)
MD5_SUMS_CATALOG_FILE="${OPTARG}"
;;
+ A)
+ ANALYZE=0
+ ;;
*)
echo "Unknown option '${opt}'"
;;
@@ -143,6 +147,7 @@
readonly SAVED_TAGS_FILE
readonly SAVE_TAGS
readonly SKIP_STATS_COLLECTION
+readonly ANALYZE
if [[ "${display_help}" -eq 1 ]] ; then
print 'Usage: checkpkg [options] pkg1 [pkg2 ....]'
@@ -153,6 +158,7 @@
print ' -d display debug messages'
print ' -N skip statistics collection'
print ' -M <file> use package md5sums from a catalog file'
+ print ' -A Do not analyze the results.'
exit 0
fi
@@ -359,15 +365,20 @@
fi
done
+if [[ "${ANALYZE}" -eq 1 ]]; then
# Collecting errors and applying the overrides.
# This has to use the original files.
-${command_basedir}/analyze_module_results.py \
- -e "${EXTRACTDIR}" \
- "$@"
-if [[ "$?" -ne 0 ]]; then
- errmsg "${RED}Modular checks are reporting errors.${COLOR_RESET}"
+ echo "Applying the overrides and analyzing the results."
+ ${command_basedir}/analyze_module_results.py \
+ -e "${EXTRACTDIR}" \
+ "$@"
+ if [[ "$?" -ne 0 ]]; then
+ errmsg "${RED}Modular checks are reporting errors.${COLOR_RESET}"
+ else
+ print "${GREEN}All modular tests were successful.${COLOR_RESET}"
+ fi
else
- print "${GREEN}All modular tests were successful.${COLOR_RESET}"
+ echo "Skipping result analysis."
fi
print ""
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-06 17:43:26 UTC (rev 10460)
@@ -553,85 +553,107 @@
return dict([[str(x.pkgname), str(x.pkg_desc)] for x in res])
-def ExpandRunpath(runpath, isalist):
- # Emulating $ISALIST expansion
- if '$ISALIST' in runpath:
- expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
- else:
- expanded_list = [runpath]
- return expanded_list
+class LddEmulator(object):
+ """A class to emulate ldd(1)
-def ExpandSymlink(symlink, target, input_path):
- # A lot of time is spent here, e.g. 13841985 calls, 206s.
- # TODO: Optimize this. Make it a class and add a cache?
- symlink_re = re.compile(r"%s(/|$)" % symlink)
- if re.search(symlink_re, input_path):
- result = input_path.replace(symlink, target)
- else:
- result = input_path
- return result
+ Used primarily to resolve SONAMEs and detect package dependencies.
+ """
+ def __init__(self):
+ self.runpath_expand_cache = {}
+ self.symlink_expand_cache = {}
+ self.symlink64_cache = {}
+ self.runpath_sanitize_cache = {}
-def Emulate64BitSymlinks(runpath_list):
- """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
+ def ExpandRunpath(self, runpath, isalist):
+ key = (runpath, tuple(isalist))
+ if key not in self.runpath_expand_cache:
+ # Emulating $ISALIST expansion
+ if '$ISALIST' in runpath:
+ expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
+ else:
+ expanded_list = [runpath]
+ self.runpath_expand_cache[key] = expanded_list
+ return self.runpath_expand_cache[key]
- Since we don't know the architecture, we'll adding both amd64 and sparcv9.
- It should be safe.
- """
- symlinked_list = []
- for runpath in runpath_list:
- for symlink, expansion_list in SYSTEM_SYMLINKS:
- for target in expansion_list:
- expanded = ExpandSymlink(symlink, target, runpath)
- if expanded not in symlinked_list:
- symlinked_list.append(expanded)
- return symlinked_list
+ def ExpandSymlink(self, symlink, target, input_path):
+ key = (symlink, target, input_path)
+ if key not in self.symlink_expand_cache:
+ # A lot of time is spent here, e.g. 13841985 calls, 206s.
+ # TODO: Optimize this. Make it a class and add a cache?
+ symlink_re = re.compile(r"%s(/|$)" % symlink)
+ if re.search(symlink_re, input_path):
+ result = input_path.replace(symlink, target)
+ else:
+ result = input_path
+ self.symlink_expand_cache[key] = result
+ return self.symlink_expand_cache[key]
+ def Emulate64BitSymlinks(self, runpath_list):
+ """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
-def SanitizeRunpath(runpath):
- while True:
- if runpath.endswith("/"):
- runpath = runpath[:-1]
- elif "//" in runpath:
- runpath = runpath.replace("//", "/")
- else:
- break
- return runpath
+ Since we don't know the architecture, we'll adding both amd64 and sparcv9.
+ It should be safe.
+ """
+ key = tuple(runpath_list)
+ if key not in self.symlink64_cache:
+ symlinked_list = []
+ for runpath in runpath_list:
+ for symlink, expansion_list in SYSTEM_SYMLINKS:
+ for target in expansion_list:
+ expanded = self.ExpandSymlink(symlink, target, runpath)
+ if expanded not in symlinked_list:
+ symlinked_list.append(expanded)
+ self.symlink64_cache[key] = symlinked_list
+ return self.symlink64_cache[key]
+ def SanitizeRunpath(self, runpath):
+ if runpath not in self.runpath_sanitize_cache:
+ new_runpath = runpath
+ while True:
+ if new_runpath.endswith("/"):
+ new_runpath = new_runpath[:-1]
+ elif "//" in new_runpath:
+ new_runpath = new_runpath.replace("//", "/")
+ else:
+ break
+ self.runpath_sanitize_cache[runpath] = new_runpath
+ return self.runpath_sanitize_cache[runpath]
-def ResolveSoname(runpath, soname, isalist, path_list):
- """Emulates ldd behavior, minimal implementation.
- runpath: e.g. ["/opt/csw/lib/$ISALIST", "/usr/lib"]
- soname: e.g. "libfoo.so.1"
- isalist: e.g. ["sparcv9", "sparcv8"]
- path_list: A list of paths where the soname is present, e.g.
- ["/opt/csw/lib", "/opt/csw/lib/sparcv9"]
+ def ResolveSoname(self, runpath, soname, isalist, path_list):
+ """Emulates ldd behavior, minimal implementation.
- The function returns the one path.
- """
- runpath = SanitizeRunpath(runpath)
- runpath_list = ExpandRunpath(runpath, isalist)
- runpath_list = Emulate64BitSymlinks(runpath_list)
- # Emulating the install time symlinks, for instance, if the prototype contains
- # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
- # the shared library ends up in /opt/csw/lib/foo.so.0 and should be
- # findable even when RPATH does not contain $ISALIST.
- original_paths_by_expanded_paths = {}
- for p in path_list:
- expanded_p_list = Emulate64BitSymlinks([p])
- # We can't just expand and return; we need to return one of the paths given
- # in the path_list.
- for expanded_p in expanded_p_list:
- original_paths_by_expanded_paths[expanded_p] = p
- logging.debug("%s: looking for %s in %s",
- soname, runpath_list, original_paths_by_expanded_paths.keys())
- for runpath_expanded in runpath_list:
- if runpath_expanded in original_paths_by_expanded_paths:
- logging.debug("Found %s",
- original_paths_by_expanded_paths[runpath_expanded])
- return original_paths_by_expanded_paths[runpath_expanded]
+ runpath: e.g. ["/opt/csw/lib/$ISALIST", "/usr/lib"]
+ soname: e.g. "libfoo.so.1"
+ isalist: e.g. ["sparcv9", "sparcv8"]
+ path_list: A list of paths where the soname is present, e.g.
+ ["/opt/csw/lib", "/opt/csw/lib/sparcv9"]
+ The function returns the one path.
+ """
+ runpath = self.SanitizeRunpath(runpath)
+ runpath_list = self.ExpandRunpath(runpath, isalist)
+ runpath_list = self.Emulate64BitSymlinks(runpath_list)
+ # Emulating the install time symlinks, for instance, if the prototype contains
+ # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
+ # the shared library ends up in /opt/csw/lib/foo.so.0 and should be
+ # findable even when RPATH does not contain $ISALIST.
+ original_paths_by_expanded_paths = {}
+ for p in path_list:
+ expanded_p_list = self.Emulate64BitSymlinks([p])
+ # We can't just expand and return; we need to return one of the paths given
+ # in the path_list.
+ for expanded_p in expanded_p_list:
+ original_paths_by_expanded_paths[expanded_p] = p
+ logging.debug("%s: looking for %s in %s",
+ soname, runpath_list, original_paths_by_expanded_paths.keys())
+ for runpath_expanded in runpath_list:
+ if runpath_expanded in original_paths_by_expanded_paths:
+ logging.debug("Found %s",
+ original_paths_by_expanded_paths[runpath_expanded])
+ return original_paths_by_expanded_paths[runpath_expanded]
+
def ParseDumpOutput(dump_output):
binary_data = {RUNPATH: [],
NEEDED_SONAMES: []}
@@ -653,6 +675,10 @@
binary_data[RUNPATH].extend(runpath)
elif rpath:
binary_data[RUNPATH].extend(rpath)
+
+ # Converting runpath to a tuple, which is a hashable data type and can act as
+ # a key in a dict.
+ binary_data[RUNPATH] = tuple(binary_data[RUNPATH])
binary_data["RUNPATH RPATH the same"] = (runpath == rpath)
binary_data["RPATH set"] = bool(rpath)
binary_data["RUNPATH set"] = bool(runpath)
@@ -880,7 +906,7 @@
count = itertools.count()
bar = progressbar.ProgressBar()
bar.maxval = len(pkgs_data) * len(self.individual_checks)
- logging.info("Running checks.")
+ logging.info("Running individual checks.")
bar.start()
for pkg_data in pkgs_data:
pkgname = pkg_data["basic_stats"]["pkgname"]
@@ -917,7 +943,7 @@
if ret:
logging.error("Calling isalist has failed.")
isalist = re.split(r"\s+", stdout.strip())
- return isalist
+ return tuple(isalist)
class PackageStats(DatabaseClient):
Modified: csw/mgar/gar/v2/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-07-06 17:43:26 UTC (rev 10460)
@@ -39,55 +39,56 @@
def setUp(self):
self.pkgmap_mocker = mox.Mox()
+ self.e = checkpkg.LddEmulator()
def testExpandRunpath_1(self):
isalist = ["foo", "bar"]
runpath = "/opt/csw/lib/$ISALIST"
expected = ["/opt/csw/lib/foo", "/opt/csw/lib/bar"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
+ self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist))
def testExpandRunpath_2(self):
isalist = ["foo", "bar"]
runpath = "/opt/csw/mysql5/lib/$ISALIST/mysql"
expected = ["/opt/csw/mysql5/lib/foo/mysql",
"/opt/csw/mysql5/lib/bar/mysql"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
+ self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist))
def testEmulate64BitSymlinks_1(self):
runpath_list = ["/opt/csw/mysql5/lib/foo/mysql/64"]
expected = "/opt/csw/mysql5/lib/foo/mysql/amd64"
- self.assertTrue(expected in checkpkg.Emulate64BitSymlinks(runpath_list))
+ self.assertTrue(expected in self.e.Emulate64BitSymlinks(runpath_list))
def testEmulate64BitSymlinks_2(self):
runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
expected = "/opt/csw/mysql5/lib/amd64/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertTrue(expected in result, "%s not in %s" % (expected, result))
def testEmulate64BitSymlinks_3(self):
runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
expected = "/opt/csw/mysql5/lib/sparcv9/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertTrue(expected in result, "%s not in %s" % (expected, result))
def testEmulate64BitSymlinks_4(self):
"""No repeated paths because of symlink expansion"""
runpath_list = ["/opt/csw/lib"]
expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertEquals(1, len(result), "len(%s) != %s" % (result, 1))
def testEmulateSymlinks_3(self):
runpath_list = ["/opt/csw/bdb4"]
expected = "/opt/csw/bdb42"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertTrue(expected in result, "%s not in %s" % (expected, result))
def testEmulateSymlinks_4(self):
runpath_list = ["/opt/csw/bdb42"]
expected = "/opt/csw/bdb42"
not_expected = "/opt/csw/bdb422"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertTrue(expected in result,
"%s not in %s" % (expected, result))
self.assertFalse(not_expected in result,
@@ -97,7 +98,7 @@
"""Install time symlink expansion."""
runpath_list = ["/opt/csw/lib/i386"]
expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
+ result = self.e.Emulate64BitSymlinks(runpath_list)
self.assertTrue(expected in result, "%s not in %s" % (expected, result))
def testEmulateSymlinks_6(self):
@@ -105,7 +106,7 @@
runpath_list = ["/opt/csw/lib/i386"]
expected = "/opt/csw/lib"
not_expected = "/opt/csw/lib/i386"
- result = checkpkg.ExpandSymlink("/opt/csw/lib/i386",
+ result = self.e.ExpandSymlink("/opt/csw/lib/i386",
"/opt/csw/lib",
"/opt/csw/lib/i386")
self.assertTrue(expected in result, "%s not in %s" % (expected, result))
@@ -114,11 +115,11 @@
def testSanitizeRunpath_1(self):
self.assertEqual("/opt/csw/lib",
- checkpkg.SanitizeRunpath("/opt/csw/lib/"))
+ self.e.SanitizeRunpath("/opt/csw/lib/"))
def testSanitizeRunpath_2(self):
self.assertEqual("/opt/csw/lib",
- checkpkg.SanitizeRunpath("/opt//csw////lib/"))
+ self.e.SanitizeRunpath("/opt//csw////lib/"))
@@ -137,18 +138,18 @@
'libnsl.so.1',
'libm.so.1',
'libz.so.1'],
- 'runpath': ['/opt/csw/lib/$ISALIST',
+ 'runpath': ('/opt/csw/lib/$ISALIST',
'/opt/csw/lib',
'/opt/csw/mysql5/lib/$ISALIST',
'/opt/csw/mysql5/lib',
- '/opt/csw/mysql5/lib/$ISALIST/mysql'],
+ '/opt/csw/mysql5/lib/$ISALIST/mysql'),
'soname': 'libmysqlclient.so.15',
}
self.assertEqual(expected,
checkpkg.ParseDumpOutput(dump_1.DATA_DUMP_OUTPUT))
def testEmpty(self):
- expected_runpath = []
+ expected_runpath = ()
self.assertEqual(
expected_runpath,
checkpkg.ParseDumpOutput(dump_2.DATA_DUMP_OUTPUT)["runpath"])
@@ -166,11 +167,11 @@
'libnsl.so.1',
'libm.so.1',
'libz.so.1'],
- 'runpath': ['/opt/csw/lib/$ISALIST',
+ 'runpath': ('/opt/csw/lib/$ISALIST',
'/opt/csw/lib',
'/opt/csw/mysql5/lib/$ISALIST',
'/opt/csw/mysql5/lib',
- '/opt/csw/mysql5/lib/$ISALIST/mysql'],
+ '/opt/csw/mysql5/lib/$ISALIST/mysql'),
'soname': 'libmysqlclient.so.15',
}
self.assertEqual(
Modified: csw/mgar/gar/v2/lib/python/dependency_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/dependency_checks.py 2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/dependency_checks.py 2010-07-06 17:43:26 UTC (rev 10460)
@@ -21,6 +21,7 @@
orphan_sonames = []
required_deps = []
isalist = pkg_data["isalist"]
+ ldd_emulator = checkpkg.LddEmulator()
for binary_info in pkg_data["binaries_dump_info"]:
for soname in binary_info["needed sonames"]:
resolved = False
@@ -32,10 +33,10 @@
path_list)
runpath_list = binary_info["runpath"] + checkpkg.SYS_DEFAULT_RUNPATH
for runpath in runpath_list:
- resolved_path = checkpkg.ResolveSoname(runpath,
- soname,
- isalist,
- path_list)
+ resolved_path = ldd_emulator.ResolveSoname(runpath,
+ soname,
+ isalist,
+ path_list)
if resolved_path:
logger.debug("%s needed by %s:",
soname, binary_info["path"])
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list