[csw-devel] SF.net SVN: gar:[7743] csw/mgar/gar/v2-checkpkg/bin

wahwah at users.sourceforge.net wahwah at users.sourceforge.net
Thu Dec 24 18:16:17 CET 2009


Revision: 7743
          http://gar.svn.sourceforge.net/gar/?rev=7743&view=rev
Author:   wahwah
Date:     2009-12-24 17:16:17 +0000 (Thu, 24 Dec 2009)

Log Message:
-----------
mGAR v2-checkpkg: sqlite3 backend for the system-wide pkgmap cache, much faster

Modified Paths:
--------------
    csw/mgar/gar/v2-checkpkg/bin/checkpkg
    csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py
    csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
    csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py

Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg	2009-12-24 16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg	2009-12-24 17:16:17 UTC (rev 7743)
@@ -26,7 +26,6 @@
 
 LOCAL_ARCH=`uname -p`
 
-
 # always print out a warning message. (to stderr)
 # exit script, if quit_on_warn set
 
@@ -67,6 +66,12 @@
 	exit 1
 }
 
+debugmsg() {
+	if [[ "${DEBUG}" != "" ]]; then
+		print "DEBUG: $*" > /dev/fd/2
+	fi
+}
+
 set_variables_for_individual_package_check() {
 	f=$1
 	file $f |sed 's/^.*://' |grep gzip >/dev/null
@@ -566,33 +571,38 @@
 set_variables_for_individual_package_check "$f"
 
 echo "Running the experimental plugin infrastructure."
+test_suite_ok=1
 checkpkg_scriptname=`basename $0`
 checkpkg_basedir=${0%/${checkpkg_scriptname}}
 plugindir=${checkpkg_basedir}/checkpkg.d
 if [[ "${DEBUG}" != "" ]]; then
 	extra_options="--debug"
 fi
-echo "plugindir: '$plugindir'"
+debugmsg "plugindir: '$plugindir'"
 if [[ -d "$plugindir" ]]; then
 	# echo plugin dir exists
 	for plugin in "${plugindir}"/checkpkg-*; do
 		if [[ -x "${plugin}" ]]; then
-			echo "Executing: ${plugin} $extra_options -e \"${EXTRACTDIR}\" ${pkgnames}"
+			debugmsg "Executing: ${plugin} $extra_options -e \"${EXTRACTDIR}\" ${pkgnames}"
 			${plugin} $extra_options -e "${EXTRACTDIR}" ${pkgnames}
 			if [[ "$?" -ne 0 ]]; then
-				errmsg "Plugin ${plugin} has returned an error."
+				print "TEST: ${plugin} [FAIL]"
+				test_suite_ok=0
+			else
+				print "TEST: ${plugin} [OK]"
 			fi
 		else
-			echo "'${plugin}' is not executable"
+			debugmsg "'${plugin}' is not executable"
 		fi
 	done
 else
-	echo plugin dir does not exist
+	debugmsg "plugin dir does not exist"
 fi
-# echo "Profiling."
-# python -m cProfile gar/bin/checkpkg.d/checkpkg-libs.py --debug -e "${EXTRACTDIR}"  ${pkgnames}
-# End of plugin section
 
+if [[ ${test_suite_ok} -ne 1 ]]; then
+	errmsg "One or more tests have failed."
+fi
+
 print ""
 
 # Cleaning up after all packages

Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py	2009-12-24 16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-dummy.py	2009-12-24 17:16:17 UTC (rev 7743)
@@ -3,6 +3,7 @@
 
 import checkpkg
 import os.path
+import logging
 
 def main():
   options, args = checkpkg.GetOptions()
@@ -12,8 +13,8 @@
     pkgpath = os.path.join(options.extractdir, pkgname)
     if not os.path.isdir(pkgpath):
       raise checkpkg.PackageError("The package directory doesn't exist: %s" % pkgpath)
-    print ("Dummy plugin says the package %s is extracted to %s"
-           % (pkgname, options.extractdir))
+    logging.debug("Dummy plugin says the package %s is extracted to %s",
+                  pkgname, options.extractdir)
 
 
 if __name__ == '__main__':

Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py	2009-12-24 16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py	2009-12-24 17:16:17 UTC (rev 7743)
@@ -121,18 +121,21 @@
     pkgs_by_soname[soname] = pkgname
 
   # A shared object dependency/provisioning report, plus checking.
-  for soname in needed_sonames:
-    if soname in needed_sonames_by_binary:
-      print "%s is provided by the package itself" % soname
-    elif soname in lines_by_soname:
-      print ("%s is required by %s and provided by %s" 
-             % (soname,
-                binaries_by_soname[soname],
-                repr(pkgs_by_soname[soname])))
-    else:
-      print ("%s is required by %s, but we don't know what provides it."
-             % (soname, binaries_by_soname[soname]))
-      result_ok = False
+  if needed_sonames:
+    print "Analysis of sonames needed by the package set:"
+    for soname in needed_sonames:
+      if soname in needed_sonames_by_binary:
+        print "%s is provided by the package itself" % soname
+      elif soname in lines_by_soname:
+        print ("%s is required by %s and provided by %s" 
+               % (soname,
+                  binaries_by_soname[soname],
+                  repr(pkgs_by_soname[soname])))
+      else:
+        print ("%s is required by %s, but we don't know what provides it."
+               % (soname, binaries_by_soname[soname]))
+        result_ok = False
+    print
 
   dependent_pkgs = {}
   for checker in checkers:
@@ -152,12 +155,12 @@
         logging.warn("%s not found in needed_sonames_by_binary (%s)",
                      binary, needed_sonames_by_binary.keys())
     declared_dependencies_set = set(declared_dependencies)
-    print "You can consider including the following packages in the dependencies:"
-    for dep_pkgname in sorted(so_dependencies.difference(declared_dependencies_set)):
-      print "  ", dep_pkgname,
-      if dep_pkgname.startswith("SUNW"):
-        print "(it's safe to ignore this one)",
-      print
+    missing_deps = so_dependencies.difference(declared_dependencies_set)
+    if missing_deps:
+      print "SUGGESTION: you may want to add some or all of the following as depends:"
+      print "   (Feel free to ignore SUNW or SPRO packages)"
+      for dep_pkgname in sorted(missing_deps):
+        print ">", dep_pkgname
     
     surplus_dependencies = declared_dependencies_set.difference(so_dependencies)
     surplus_dependencies = surplus_dependencies.difference(TYPICAL_DEPENDENCIES)

Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py	2009-12-24 16:36:19 UTC (rev 7742)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py	2009-12-24 17:16:17 UTC (rev 7743)
@@ -10,6 +10,7 @@
 import subprocess
 import cPickle
 import re
+import sqlite3
 
 SYSTEM_PKGMAP = "/var/sadm/install/contents"
 WS_RE = re.compile(r"\s+")
@@ -89,26 +90,38 @@
 class SystemPkgmap(object):
   """A class to hold and manipulate the /var/sadm/install/contents file."""
   
-  PICKLE_NAME = "var-sadm-install-contents.pickle"
   STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"] 
   CHECKPKG_DIR = ".checkpkg"
+  SQLITE3_DBNAME = "var-sadm-install-contents-cache"
 
   def __init__(self):
     """There is no need to re-parse it each time.
 
     Read it slowly the first time and cache it for later."""
     self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
-    self.pickle_path = os.path.join(self.checkpkg_dir, self.PICKLE_NAME)
-    if os.path.exists(self.pickle_path):
-      logging.info("Unpickling %s, this can take up to 30s.", self.pickle_path)
-      pickle_fd = open(self.pickle_path, "r")
-      self.pkmap_lines_by_basename = cPickle.load(pickle_fd)
-      pickle_fd.close()
+    self.db_path = os.path.join(self.checkpkg_dir, self.SQLITE3_DBNAME)
+    if os.path.exists(self.db_path):
+      logging.debug("Connecting to the %s database.", self.db_path)
+      self.conn = sqlite3.connect(self.db_path)
     else:
-      # The original checkpkg code to port is in the comments.
-      #
+      logging.info("Building a cache of /var/sadm/install/contents.")
+      if not os.path.exists(self.checkpkg_dir):
+        logging.debug("Creating %s", self.checkpkg_dir)
+        os.mkdir(self.checkpkg_dir)
+      self.conn = sqlite3.connect(self.db_path)
+      c = self.conn.cursor()
+      c.execute("""
+          CREATE TABLE systempkgmap (
+            id INTEGER PRIMARY KEY,
+            basename TEXT,
+            path TEXT,
+            line TEXT
+          );
+      """)
+
       # egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
       #     fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
+
       system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
 
       stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
@@ -117,29 +130,23 @@
       # soname - {<path1>: <line1>, <path2>: <line2>, ...}
       logging.debug("Building in-memory data structure for the %s file",
                     SYSTEM_PKGMAP)
-      pkmap_lines_by_basename = {}
       for line in system_pkgmap_fd:
         if stop_re.search(line):
           continue
         fields = re.split(WS_RE, line)
         pkgmap_entry_path = fields[0].split("=")[0]
         pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
-        if pkgmap_entry_base_name not in pkmap_lines_by_basename:
-          pkmap_lines_by_basename[pkgmap_entry_base_name] = {}
-        pkmap_lines_by_basename[pkgmap_entry_base_name][pkgmap_entry_dir] = line
-      logging.debug("The data structure contains %s files",
-                    len(pkmap_lines_by_basename))
-      self.pkmap_lines_by_basename = pkmap_lines_by_basename
-      if not os.path.exists(self.checkpkg_dir):
-        logging.debug("Creating %s", self.checkpkg_dir)
-        os.mkdir(self.checkpkg_dir)
-      logging.debug("Pickling to %s", self.pickle_path)
-      pickle_fd = open(self.pickle_path, "w")
-      cPickle.dump(self.pkmap_lines_by_basename, pickle_fd)
-      pickle_fd.close()
+        sql = "INSERT INTO systempkgmap (basename, path, line) VALUES (?, ?, ?);"
+        c.execute(sql, (pkgmap_entry_base_name, pkgmap_entry_dir, line))
+      logging.info("Creating an index.")
+      sql = "CREATE INDEX basename_idx ON systempkgmap(basename);"
+      self.conn.execute(sql)
 
   def GetPkgmapLineByBasename(self, filename):
-    if filename in self.pkmap_lines_by_basename:
-      return self.pkmap_lines_by_basename[filename]
-    else:
-      raise KeyError, "%s not found in self.pkmap_lines_by_basename" % filename
+    sql = "SELECT path, line FROM systempkgmap WHERE basename = ?;"
+    c = self.conn.cursor()
+    c.execute(sql, [filename])
+    lines = {}
+    for row in c:
+      lines[row[0]] = row[1]
+    return lines


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.



More information about the devel mailing list