[csw-devel] SF.net SVN: gar:[8752] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Tue Feb 23 01:10:13 CET 2010
Revision: 8752
http://gar.svn.sourceforge.net/gar/?rev=8752&view=rev
Author: wahwah
Date: 2010-02-23 00:10:13 +0000 (Tue, 23 Feb 2010)
Log Message:
-----------
mGAR v2: checkpkg, using nm to collect symbols and anlayze them. Slow.
Modified Paths:
--------------
csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
csw/mgar/gar/v2/lib/checkpkg.d/checkpkg-missing-symbols.py
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/checkpkg_test.py
csw/mgar/gar/v2/lib/python/package_checks.py
csw/mgar/gar/v2/lib/python/package_checks_test.py
Property Changed:
----------------
csw/mgar/gar/v2/lib/python/package_checks_test.py
Modified: csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -36,7 +36,7 @@
packages = [opencsw.CswSrv4File(x, options.debug) for x in args]
stats_list = [checkpkg.PackageStats(pkg) for pkg in packages]
for pkg_stats in stats_list:
- pkg_stats.CollectStats()
+ pkg_stats.CollectStats()
if __name__ == '__main__':
- main()
+ main()
Modified: csw/mgar/gar/v2/lib/checkpkg.d/checkpkg-missing-symbols.py
===================================================================
--- csw/mgar/gar/v2/lib/checkpkg.d/checkpkg-missing-symbols.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/lib/checkpkg.d/checkpkg-missing-symbols.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -19,28 +19,10 @@
"..", "..", "lib", "python"]
sys.path.append(os.path.join(*path_list))
import checkpkg
+import package_checks
# Defining checking functions.
-def CheckForMissingSymbols(pkg_data, debug):
- """Looks for "symbol not found" in ldd -r output."""
- errors = []
- binaries = pkg_data["binaries"]
- symbol_re = re.compile(r"symbol not found:")
- for binary in binaries:
- lines = pkg_data["ldd_dash_r"][binary]
- missing_symbols = False
- for line in lines:
- if re.search(symbol_re, line):
- missing_symbols = True
- binary_base = os.path.basename(binary)
- if missing_symbols:
- errors.append(checkpkg.CheckpkgTag(
- pkg_data["basic_stats"]["pkgname"],
- "symbol-not-found", binary_base))
- return errors
-
-
def main():
options, args = checkpkg.GetOptions()
md5sums = args
@@ -51,7 +33,7 @@
md5sums,
options.debug)
# Registering functions defined above.
- check_manager.RegisterIndividualCheck(CheckForMissingSymbols)
+ check_manager.RegisterSetCheck(package_checks.CheckForMissingSymbols)
# Running the checks, reporting and exiting.
exit_code, screen_report, tags_report = check_manager.Run()
f = open(options.output, "w")
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -111,6 +111,10 @@
pass
+class StdoutSyntaxError(Error):
+ pass
+
+
def GetOptions():
parser = optparse.OptionParser()
parser.add_option("-b", dest="stats_basedir",
@@ -811,7 +815,7 @@
def ApplyOverrides(error_tags, overrides):
"""Filters out all the error tags that overrides apply to.
-
+
O(N * M), but N and M are always small.
"""
tags_after_overrides = []
@@ -845,6 +849,7 @@
"basic_stats",
"binaries",
"binaries_dump_info",
+ "defined_symbols",
"depends",
"isalist",
"ldd_dash_r",
@@ -982,8 +987,10 @@
retcode = ldd_proc.wait()
if retcode:
logging.error("%s returned an error: %s", args, stderr)
- lines = stdout.splitlines()
- ldd_output[binary] = lines
+ ldd_info = []
+ for line in stdout.splitlines():
+ ldd_info.append(self._ParseLddDashRline(line))
+ ldd_output[binary] = ldd_info
return ldd_output
def GetDefinedSymbols(self):
@@ -1036,7 +1043,14 @@
sym = { 'address': fields[0], 'type': fields[1], 'name': fields[2] }
return sym
- def CollectStats(self):
+ def CollectStats(self, force=False):
+ if not self.StatsDirExists() or force:
+ self._CollectStats()
+
+ def _CollectStats(self):
+ """The list of variables needs to be synchronized with the one
+ at the top of this class.
+ """
stats_path = self.GetStatsPath()
self.MakeStatsDir()
dir_pkg = self.GetDirFormatPkg()
@@ -1109,7 +1123,8 @@
def _ParseLddDashRline(self, line):
found_re = r"^\t(?P<soname>\S+)\s+=>\s+(?P<path_found>\S+)"
symbol_not_found_re = r"^\tsymbol not found:\s(?P<symbol>\S+)\s+\((?P<path_not_found>\S+)\)"
- common_re = r"(%s|%s)" % (found_re, symbol_not_found_re)
+ only_so = r"^\t(?P<path_only>\S+)$"
+ common_re = r"(%s|%s|%s)" % (found_re, symbol_not_found_re, only_so)
m = re.match(common_re, line)
response = {}
if m:
@@ -1120,11 +1135,16 @@
response["soname"] = d["soname"]
response["path"] = d["path_found"]
response["symbol"] = None
- elif "symbol" in d:
+ elif "symbol" in d and d["symbol"]:
response["state"] = "symbol-not-found"
response["soname"] = None
response["path"] = d["path_not_found"]
response["symbol"] = d["symbol"]
+ elif d["path_only"]:
+ response["state"] = "OK"
+ response["soname"] = None
+ response["path"] = d["path_only"]
+ response["symbol"] = None
else:
raise StdoutSyntaxError("Could not parse %s" % repr(line))
else:
Modified: csw/mgar/gar/v2/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -39,6 +39,7 @@
\tsymbol not found: LocalToUtf (/opt/csw/lib/postgresql/8.4/utf8_and_gbk.so)
\tsymbol not found: UtfToLocal (/opt/csw/lib/postgresql/8.4/utf8_and_gbk.so)
\tlibm.so.2 => /lib/libm.so.2
+\t/usr/lib/secure/s8_preload.so.1
"""
class DependenciesUnitTest_1(unittest.TestCase):
@@ -799,6 +800,20 @@
}
self.assertEqual(expected, self.pkgstats._ParseLddDashRline(line))
+ def test_ParseLddDashRlineFound(self):
+ line = '\t/usr/lib/secure/s8_preload.so.1'
+ expected = {
+ 'state': 'OK',
+ 'soname': None,
+ 'path': '/usr/lib/secure/s8_preload.so.1',
+ 'symbol': None,
+ }
+ self.assertEqual(expected, self.pkgstats._ParseLddDashRline(line))
+ def test_ParseLddDashRlineManyLines(self):
+ for line in LDD_R_OUTPUT_1.splitlines():
+ parsed = self.pkgstats._ParseLddDashRline(line)
+
+
if __name__ == '__main__':
unittest.main()
Modified: csw/mgar/gar/v2/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/lib/python/package_checks.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -137,3 +137,41 @@
print ("However, be aware that there might be other reasons "
"to keep it architecture-specific.")
return errors
+
+
+def CheckForMissingSymbols(pkgs_data, debug):
+ """Analyzes missing symbols reported by ldd -r.
+
+ 1. Collect triplets: pkgname, binary, missing symbol
+ 2. If there are any missing symbols, collect all the symbols that are provided
+ by the set of packages.
+ 3. From the list of missing symbols, remove all symbols that are provided
+ by the set of packages.
+ 4. Report any remaining symbols as errors.
+
+ What indexes do we need?
+
+ symbol -> (pkgname, binary)
+ set(allsymbols)
+ """
+ errors = []
+ missing_symbols = []
+ all_symbols = set()
+ for pkg_data in pkgs_data:
+ pkgname = pkg_data["basic_stats"]["pkgname"]
+ binaries = pkg_data["binaries"]
+ for binary in binaries:
+ for ldd_elem in pkg_data["ldd_dash_r"][binary]:
+ if ldd_elem["state"] == "symbol-not-found":
+ missing_symbols.append((pkgname,
+ binary,
+ ldd_elem["symbol"]))
+ for symbol in pkg_data["defined_symbols"][binary]:
+ all_symbols.add(symbol)
+ # Remove symbols defined elsewhere.
+ while missing_symbols:
+ ms_pkgname, ms_binary, ms_symbol = missing_symbols.pop()
+ if ms_symbol not in all_symbols:
+ errors.append(checkpkg.CheckpkgTag(
+ ms_pkgname, "symbol-not-found", "%s %s" % (ms_binary, ms_symbol)))
+ return errors
Modified: csw/mgar/gar/v2/lib/python/package_checks_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-02-22 20:21:12 UTC (rev 8751)
+++ csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-02-23 00:10:13 UTC (rev 8752)
@@ -67,8 +67,50 @@
self.pkg_data_2["binaries"] = []
self.pkg_data_2["pkginfo"] = self.LoadData("example-1-pkginfo")
errors = pc.CheckArchitectureVsContents(self.pkg_data_2, False)
- print errors
+ self.failIf(errors)
+ def testCheckForMissingSymbols(self):
+ ldd_dash_r_yml = """opt/csw/lib/postgresql/8.4/_int.so:
+- {path: /usr/lib/libc.so.1, soname: libc.so.1, state: OK, symbol: null}
+- {path: /usr/lib/libdl.so.1, soname: libdl.so.1, state: OK, symbol: null}
+- {path: /tmp/pkg_W8UcnK/CSWlibpq-84/root/opt/csw/lib/postgresql/8.4/_int.so, soname: null,
+ state: symbol-not-found, symbol: CurrentMemoryContext}
+- {path: /tmp/pkg_W8UcnK/CSWlibpq-84/root/opt/csw/lib/postgresql/8.4/_int.so, soname: null,
+ state: symbol-not-found, symbol: MemoryContextAlloc}
+- {path: /tmp/pkg_W8UcnK/CSWlibpq-84/root/opt/csw/lib/postgresql/8.4/_int.so, soname: null,
+ state: symbol-not-found, symbol: errstart}
+- {path: /tmp/pkg_W8UcnK/CSWlibpq-84/root/opt/csw/lib/postgresql/8.4/_int.so, soname: null,
+ state: symbol-not-found, symbol: errcode}
+opt/csw/lib/postgresql/8.4/_int2.so:
+- {path: /usr/lib/libdl.so.1, soname: libdl.so.1, state: OK, symbol: null}"""
+ defined_symbols_yml = """opt/csw/lib/postgresql/8.4/_int.so: [Pg_magic_func, _fini, _init, _int_contained,
+ _int_contains, _int_different, _int_inter, _int_overlap, _int_same, _int_union,
+ _int_unique, _intbig_in, _intbig_out, boolop, bqarr_in, bqarr_out, compASC, compDESC,
+ copy_intArrayType, execconsistent, g_int_compress, g_int_consistent, g_int_decompress,
+ g_int_penalty, g_int_picksplit, g_int_same, g_int_union, g_intbig_compress, g_intbig_consistent,
+ g_intbig_decompress, g_intbig_penalty, g_intbig_picksplit, g_intbig_same, g_intbig_union,
+ gensign, ginconsistent, ginint4_consistent, ginint4_queryextract, icount, idx, inner_int_contains,
+ inner_int_inter, inner_int_overlap, inner_int_union, int_to_intset, intarray_add_elem,
+ intarray_concat_arrays, intarray_del_elem, intarray_match_first, intarray_push_array,
+ intarray_push_elem, internal_size, intset, intset_subtract, intset_union_elem, isort,
+ new_intArrayType, pg_finfo__int_contained, pg_finfo__int_contains, pg_finfo__int_different,
+ pg_finfo__int_inter, pg_finfo__int_overlap, pg_finfo__int_same, pg_finfo__int_union,
+ pg_finfo__intbig_in, pg_finfo__intbig_out, pg_finfo_boolop, pg_finfo_bqarr_in, pg_finfo_bqarr_out,
+ pg_finfo_g_int_compress, pg_finfo_g_int_consistent, pg_finfo_g_int_decompress, pg_finfo_g_int_penalty,
+ pg_finfo_g_int_picksplit, pg_finfo_g_int_same, pg_finfo_g_int_union, pg_finfo_g_intbig_compress,
+ pg_finfo_g_intbig_consistent, pg_finfo_g_intbig_decompress, pg_finfo_g_intbig_penalty,
+ pg_finfo_g_intbig_picksplit, pg_finfo_g_intbig_same, pg_finfosc, subarray, uniq]
+opt/csw/lib/postgresql/8.4/adminpack.so: [Pg_magic_func, _fini, _init, pg_file_rename,
+ pg_file_unlink, pg_file_write, pg_finfo_pg_file_rename, pg_finfo_pg_file_unlink,
+ pg_finfo_pg_file_write, pg_finfo_pg_logdir_ls, pg_logdir_ls]"""
+ self.pkg_data_2["ldd_dash_r"] = yaml.safe_load(ldd_dash_r_yml)
+ self.pkg_data_2["defined_symbols"] = yaml.safe_load(defined_symbols_yml)
+ self.pkg_data_2["binaries"] = ["opt/csw/lib/postgresql/8.4/_int.so",
+ "opt/csw/lib/postgresql/8.4/_int2.so"]
+ errors = pc.CheckForMissingSymbols([self.pkg_data_2], False)
+ self.failUnless(errors)
+
+
if __name__ == '__main__':
unittest.main()
Property changes on: csw/mgar/gar/v2/lib/python/package_checks_test.py
___________________________________________________________________
Added: svn:keywords
+ Id
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list