[csw-devel] SF.net SVN: gar:[10460] csw/mgar/gar/v2

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Tue Jul 6 19:43:26 CEST 2010


Revision: 10460
          http://gar.svn.sourceforge.net/gar/?rev=10460&view=rev
Author:   wahwah
Date:     2010-07-06 17:43:26 +0000 (Tue, 06 Jul 2010)

Log Message:
-----------
mGAR v2: checkpkg, optimizing stats analysis, allowing to skip applying of the overrides.

Modified Paths:
--------------
    csw/mgar/gar/v2/bin/checkpkg
    csw/mgar/gar/v2/lib/python/checkpkg.py
    csw/mgar/gar/v2/lib/python/checkpkg_test.py
    csw/mgar/gar/v2/lib/python/dependency_checks.py

Modified: csw/mgar/gar/v2/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg	2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/bin/checkpkg	2010-07-06 17:43:26 UTC (rev 10460)
@@ -107,8 +107,9 @@
 SKIP_STATS_COLLECTION=0
 MD5_SUMS_CATALOG_FILE=""
 INSTALL_CONTENTS_FILES="/var/sadm/install/contents"
+ANALYZE=1
 
-while getopts hsdNM:o:c: opt; do
+while getopts hsdNM:o:c:A opt; do
 	case "${opt}" in
 	  c)
 	    INSTALL_CONTENTS_FILES="${INSTALL_CONTENTS_FILES} ${OPTARG}"
@@ -131,6 +132,9 @@
     M)
       MD5_SUMS_CATALOG_FILE="${OPTARG}"
       ;;
+    A)
+      ANALYZE=0
+      ;;
     *)
       echo "Unknown option '${opt}'"
       ;;
@@ -143,6 +147,7 @@
 readonly SAVED_TAGS_FILE
 readonly SAVE_TAGS
 readonly SKIP_STATS_COLLECTION
+readonly ANALYZE
 
 if [[ "${display_help}" -eq 1 ]] ; then
   print 'Usage: checkpkg [options] pkg1 [pkg2 ....]'
@@ -153,6 +158,7 @@
   print '   -d         display debug messages'
   print '   -N         skip statistics collection'
   print '   -M <file>  use package md5sums from a catalog file'
+  print '   -A         Do not analyze the results.'
   exit 0
 fi
 
@@ -359,15 +365,20 @@
   fi
 done
 
+if [[ "${ANALYZE}" -eq 1 ]]; then
 # Collecting errors and applying the overrides.
 # This has to use the original files.
-${command_basedir}/analyze_module_results.py \
-    -e "${EXTRACTDIR}" \
-    "$@"
-if [[ "$?" -ne 0 ]]; then
-	errmsg "${RED}Modular checks are reporting errors.${COLOR_RESET}"
+  echo "Applying the overrides and analyzing the results."
+  ${command_basedir}/analyze_module_results.py \
+      -e "${EXTRACTDIR}" \
+      "$@"
+  if [[ "$?" -ne 0 ]]; then
+    errmsg "${RED}Modular checks are reporting errors.${COLOR_RESET}"
+  else
+    print "${GREEN}All modular tests were successful.${COLOR_RESET}"
+  fi
 else
-	print "${GREEN}All modular tests were successful.${COLOR_RESET}"
+	echo "Skipping result analysis."
 fi
 
 print ""

Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py	2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py	2010-07-06 17:43:26 UTC (rev 10460)
@@ -553,85 +553,107 @@
     return dict([[str(x.pkgname), str(x.pkg_desc)] for x in res])
 
 
-def ExpandRunpath(runpath, isalist):
-  # Emulating $ISALIST expansion
-  if '$ISALIST' in runpath:
-    expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
-  else:
-    expanded_list = [runpath]
-  return expanded_list
+class LddEmulator(object):
+  """A class to emulate ldd(1)
 
-def ExpandSymlink(symlink, target, input_path):
-  # A lot of time is spent here, e.g. 13841985 calls, 206s.
-  # TODO: Optimize this.  Make it a class and add a cache?
-  symlink_re = re.compile(r"%s(/|$)" % symlink)
-  if re.search(symlink_re, input_path):
-    result = input_path.replace(symlink, target)
-  else:
-    result = input_path
-  return result
+  Used primarily to resolve SONAMEs and detect package dependencies.
+  """
+  def __init__(self):
+    self.runpath_expand_cache = {}
+    self.symlink_expand_cache = {}
+    self.symlink64_cache = {}
+    self.runpath_sanitize_cache = {}
 
-def Emulate64BitSymlinks(runpath_list):
-  """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
+  def ExpandRunpath(self, runpath, isalist):
+    key = (runpath, tuple(isalist))
+    if key not in self.runpath_expand_cache:
+      # Emulating $ISALIST expansion
+      if '$ISALIST' in runpath:
+        expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
+      else:
+        expanded_list = [runpath]
+      self.runpath_expand_cache[key] = expanded_list
+    return self.runpath_expand_cache[key]
 
-  Since we don't know the architecture, we'll adding both amd64 and sparcv9.
-  It should be safe.
-  """
-  symlinked_list = []
-  for runpath in runpath_list:
-    for symlink, expansion_list in SYSTEM_SYMLINKS:
-      for target in expansion_list:
-        expanded = ExpandSymlink(symlink, target, runpath)
-        if expanded not in symlinked_list:
-          symlinked_list.append(expanded)
-  return symlinked_list
+  def ExpandSymlink(self, symlink, target, input_path):
+    key = (symlink, target, input_path)
+    if key not in self.symlink_expand_cache:
+      # A lot of time is spent here, e.g. 13841985 calls, 206s.
+      # TODO: Optimize this.  Make it a class and add a cache?
+      symlink_re = re.compile(r"%s(/|$)" % symlink)
+      if re.search(symlink_re, input_path):
+        result = input_path.replace(symlink, target)
+      else:
+        result = input_path
+      self.symlink_expand_cache[key] = result
+    return self.symlink_expand_cache[key]
 
+  def Emulate64BitSymlinks(self, runpath_list):
+    """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
 
-def SanitizeRunpath(runpath):
-  while True:
-    if runpath.endswith("/"):
-      runpath = runpath[:-1]
-    elif "//" in runpath:
-      runpath = runpath.replace("//", "/")
-    else:
-      break
-  return runpath
+    Since we don't know the architecture, we'll adding both amd64 and sparcv9.
+    It should be safe.
+    """
+    key = tuple(runpath_list)
+    if key not in self.symlink64_cache:
+      symlinked_list = []
+      for runpath in runpath_list:
+        for symlink, expansion_list in SYSTEM_SYMLINKS:
+          for target in expansion_list:
+            expanded = self.ExpandSymlink(symlink, target, runpath)
+            if expanded not in symlinked_list:
+              symlinked_list.append(expanded)
+      self.symlink64_cache[key] = symlinked_list
+    return self.symlink64_cache[key]
 
+  def SanitizeRunpath(self, runpath):
+    if runpath not in self.runpath_sanitize_cache:
+      new_runpath = runpath
+      while True:
+        if new_runpath.endswith("/"):
+          new_runpath = new_runpath[:-1]
+        elif "//" in new_runpath:
+          new_runpath = new_runpath.replace("//", "/")
+        else:
+          break
+      self.runpath_sanitize_cache[runpath] = new_runpath
+    return self.runpath_sanitize_cache[runpath]
 
-def ResolveSoname(runpath, soname, isalist, path_list):
-  """Emulates ldd behavior, minimal implementation.
 
-  runpath: e.g. ["/opt/csw/lib/$ISALIST", "/usr/lib"]
-  soname: e.g. "libfoo.so.1"
-  isalist: e.g. ["sparcv9", "sparcv8"]
-  path_list: A list of paths where the soname is present, e.g.
-             ["/opt/csw/lib", "/opt/csw/lib/sparcv9"]
+  def ResolveSoname(self, runpath, soname, isalist, path_list):
+    """Emulates ldd behavior, minimal implementation.
 
-  The function returns the one path.
-  """
-  runpath = SanitizeRunpath(runpath)
-  runpath_list = ExpandRunpath(runpath, isalist)
-  runpath_list = Emulate64BitSymlinks(runpath_list)
-  # Emulating the install time symlinks, for instance, if the prototype contains
-  # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
-  # the shared library ends up in /opt/csw/lib/foo.so.0 and should be
-  # findable even when RPATH does not contain $ISALIST.
-  original_paths_by_expanded_paths = {}
-  for p in path_list:
-    expanded_p_list = Emulate64BitSymlinks([p])
-    # We can't just expand and return; we need to return one of the paths given
-    # in the path_list.
-    for expanded_p in expanded_p_list:
-      original_paths_by_expanded_paths[expanded_p] = p
-  logging.debug("%s: looking for %s in %s",
-      soname, runpath_list, original_paths_by_expanded_paths.keys())
-  for runpath_expanded in runpath_list:
-    if runpath_expanded in original_paths_by_expanded_paths:
-      logging.debug("Found %s",
-                    original_paths_by_expanded_paths[runpath_expanded])
-      return original_paths_by_expanded_paths[runpath_expanded]
+    runpath: e.g. ["/opt/csw/lib/$ISALIST", "/usr/lib"]
+    soname: e.g. "libfoo.so.1"
+    isalist: e.g. ["sparcv9", "sparcv8"]
+    path_list: A list of paths where the soname is present, e.g.
+               ["/opt/csw/lib", "/opt/csw/lib/sparcv9"]
 
+    The function returns the one path.
+    """
+    runpath = self.SanitizeRunpath(runpath)
+    runpath_list = self.ExpandRunpath(runpath, isalist)
+    runpath_list = self.Emulate64BitSymlinks(runpath_list)
+    # Emulating the install time symlinks, for instance, if the prototype contains
+    # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
+    # the shared library ends up in /opt/csw/lib/foo.so.0 and should be
+    # findable even when RPATH does not contain $ISALIST.
+    original_paths_by_expanded_paths = {}
+    for p in path_list:
+      expanded_p_list = self.Emulate64BitSymlinks([p])
+      # We can't just expand and return; we need to return one of the paths given
+      # in the path_list.
+      for expanded_p in expanded_p_list:
+        original_paths_by_expanded_paths[expanded_p] = p
+    logging.debug("%s: looking for %s in %s",
+        soname, runpath_list, original_paths_by_expanded_paths.keys())
+    for runpath_expanded in runpath_list:
+      if runpath_expanded in original_paths_by_expanded_paths:
+        logging.debug("Found %s",
+                      original_paths_by_expanded_paths[runpath_expanded])
+        return original_paths_by_expanded_paths[runpath_expanded]
 
+
 def ParseDumpOutput(dump_output):
   binary_data = {RUNPATH: [],
                  NEEDED_SONAMES: []}
@@ -653,6 +675,10 @@
     binary_data[RUNPATH].extend(runpath)
   elif rpath:
     binary_data[RUNPATH].extend(rpath)
+
+  # Converting runpath to a tuple, which is a hashable data type and can act as
+  # a key in a dict.
+  binary_data[RUNPATH] = tuple(binary_data[RUNPATH])
   binary_data["RUNPATH RPATH the same"] = (runpath == rpath)
   binary_data["RPATH set"] = bool(rpath)
   binary_data["RUNPATH set"] = bool(runpath)
@@ -880,7 +906,7 @@
     count = itertools.count()
     bar = progressbar.ProgressBar()
     bar.maxval = len(pkgs_data) * len(self.individual_checks)
-    logging.info("Running checks.")
+    logging.info("Running individual checks.")
     bar.start()
     for pkg_data in pkgs_data:
       pkgname = pkg_data["basic_stats"]["pkgname"]
@@ -917,7 +943,7 @@
   if ret:
     logging.error("Calling isalist has failed.")
   isalist = re.split(r"\s+", stdout.strip())
-  return isalist
+  return tuple(isalist)
 
 
 class PackageStats(DatabaseClient):

Modified: csw/mgar/gar/v2/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_test.py	2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/checkpkg_test.py	2010-07-06 17:43:26 UTC (rev 10460)
@@ -39,55 +39,56 @@
 
   def setUp(self):
     self.pkgmap_mocker = mox.Mox()
+    self.e = checkpkg.LddEmulator()
 
   def testExpandRunpath_1(self):
     isalist = ["foo", "bar"]
     runpath = "/opt/csw/lib/$ISALIST"
     expected = ["/opt/csw/lib/foo", "/opt/csw/lib/bar"]
-    self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
+    self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist))
 
   def testExpandRunpath_2(self):
     isalist = ["foo", "bar"]
     runpath = "/opt/csw/mysql5/lib/$ISALIST/mysql"
     expected = ["/opt/csw/mysql5/lib/foo/mysql",
                 "/opt/csw/mysql5/lib/bar/mysql"]
-    self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
+    self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist))
 
   def testEmulate64BitSymlinks_1(self):
     runpath_list = ["/opt/csw/mysql5/lib/foo/mysql/64"]
     expected = "/opt/csw/mysql5/lib/foo/mysql/amd64"
-    self.assertTrue(expected in checkpkg.Emulate64BitSymlinks(runpath_list))
+    self.assertTrue(expected in self.e.Emulate64BitSymlinks(runpath_list))
 
   def testEmulate64BitSymlinks_2(self):
     runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
     expected = "/opt/csw/mysql5/lib/amd64/mysql/foo"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertTrue(expected in result, "%s not in %s" % (expected, result))
 
   def testEmulate64BitSymlinks_3(self):
     runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
     expected = "/opt/csw/mysql5/lib/sparcv9/mysql/foo"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertTrue(expected in result, "%s not in %s" % (expected, result))
 
   def testEmulate64BitSymlinks_4(self):
     """No repeated paths because of symlink expansion"""
     runpath_list = ["/opt/csw/lib"]
     expected = "/opt/csw/lib"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertEquals(1, len(result), "len(%s) != %s" % (result, 1))
 
   def testEmulateSymlinks_3(self):
     runpath_list = ["/opt/csw/bdb4"]
     expected = "/opt/csw/bdb42"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertTrue(expected in result, "%s not in %s" % (expected, result))
 
   def testEmulateSymlinks_4(self):
     runpath_list = ["/opt/csw/bdb42"]
     expected = "/opt/csw/bdb42"
     not_expected = "/opt/csw/bdb422"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertTrue(expected in result,
                     "%s not in %s" % (expected, result))
     self.assertFalse(not_expected in result,
@@ -97,7 +98,7 @@
     """Install time symlink expansion."""
     runpath_list = ["/opt/csw/lib/i386"]
     expected = "/opt/csw/lib"
-    result = checkpkg.Emulate64BitSymlinks(runpath_list)
+    result = self.e.Emulate64BitSymlinks(runpath_list)
     self.assertTrue(expected in result, "%s not in %s" % (expected, result))
 
   def testEmulateSymlinks_6(self):
@@ -105,7 +106,7 @@
     runpath_list = ["/opt/csw/lib/i386"]
     expected = "/opt/csw/lib"
     not_expected = "/opt/csw/lib/i386"
-    result = checkpkg.ExpandSymlink("/opt/csw/lib/i386",
+    result = self.e.ExpandSymlink("/opt/csw/lib/i386",
                                     "/opt/csw/lib",
                                     "/opt/csw/lib/i386")
     self.assertTrue(expected in result, "%s not in %s" % (expected, result))
@@ -114,11 +115,11 @@
 
   def testSanitizeRunpath_1(self):
     self.assertEqual("/opt/csw/lib",
-                     checkpkg.SanitizeRunpath("/opt/csw/lib/"))
+                     self.e.SanitizeRunpath("/opt/csw/lib/"))
 
   def testSanitizeRunpath_2(self):
     self.assertEqual("/opt/csw/lib",
-                     checkpkg.SanitizeRunpath("/opt//csw////lib/"))
+                     self.e.SanitizeRunpath("/opt//csw////lib/"))
 
 
 
@@ -137,18 +138,18 @@
                            'libnsl.so.1',
                            'libm.so.1',
                            'libz.so.1'],
-        'runpath': ['/opt/csw/lib/$ISALIST',
+        'runpath': ('/opt/csw/lib/$ISALIST',
                     '/opt/csw/lib',
                     '/opt/csw/mysql5/lib/$ISALIST',
                     '/opt/csw/mysql5/lib',
-                    '/opt/csw/mysql5/lib/$ISALIST/mysql'],
+                    '/opt/csw/mysql5/lib/$ISALIST/mysql'),
         'soname': 'libmysqlclient.so.15',
     }
     self.assertEqual(expected,
                      checkpkg.ParseDumpOutput(dump_1.DATA_DUMP_OUTPUT))
 
   def testEmpty(self):
-    expected_runpath = []
+    expected_runpath = ()
     self.assertEqual(
         expected_runpath,
         checkpkg.ParseDumpOutput(dump_2.DATA_DUMP_OUTPUT)["runpath"])
@@ -166,11 +167,11 @@
                            'libnsl.so.1',
                            'libm.so.1',
                            'libz.so.1'],
-        'runpath': ['/opt/csw/lib/$ISALIST',
+        'runpath': ('/opt/csw/lib/$ISALIST',
                     '/opt/csw/lib',
                     '/opt/csw/mysql5/lib/$ISALIST',
                     '/opt/csw/mysql5/lib',
-                    '/opt/csw/mysql5/lib/$ISALIST/mysql'],
+                    '/opt/csw/mysql5/lib/$ISALIST/mysql'),
         'soname': 'libmysqlclient.so.15',
     }
     self.assertEqual(

Modified: csw/mgar/gar/v2/lib/python/dependency_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/dependency_checks.py	2010-07-06 14:59:47 UTC (rev 10459)
+++ csw/mgar/gar/v2/lib/python/dependency_checks.py	2010-07-06 17:43:26 UTC (rev 10460)
@@ -21,6 +21,7 @@
   orphan_sonames = []
   required_deps = []
   isalist = pkg_data["isalist"]
+  ldd_emulator = checkpkg.LddEmulator()
   for binary_info in pkg_data["binaries_dump_info"]:
     for soname in binary_info["needed sonames"]:
       resolved = False
@@ -32,10 +33,10 @@
                    path_list)
       runpath_list = binary_info["runpath"] + checkpkg.SYS_DEFAULT_RUNPATH
       for runpath in runpath_list:
-        resolved_path = checkpkg.ResolveSoname(runpath,
-                                               soname,
-                                               isalist,
-                                               path_list)
+        resolved_path = ldd_emulator.ResolveSoname(runpath,
+                                                   soname,
+                                                   isalist,
+                                                   path_list)
         if resolved_path:
           logger.debug("%s needed by %s:",
                  soname, binary_info["path"])


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.


More information about the devel mailing list