[csw-devel] SF.net SVN: gar:[8181] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Tue Jan 26 11:43:08 CET 2010
Revision: 8181
http://gar.svn.sourceforge.net/gar/?rev=8181&view=rev
Author: wahwah
Date: 2010-01-26 10:43:08 +0000 (Tue, 26 Jan 2010)
Log Message:
-----------
mGAR v2: Merging v2-dirpackage (started at Wintercamp)
Modified Paths:
--------------
csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-libs.py
csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-obsolete-deps.py
csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-you-can-write-your-own.py
csw/mgar/gar/v2/bin/checkpkg.d/update_contents_cache.py
csw/mgar/gar/v2/gar.pkg.mk
Added Paths:
-----------
csw/mgar/gar/v2/lib/
csw/mgar/gar/v2/lib/python/
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/checkpkg_test.py
csw/mgar/gar/v2/lib/python/gartest.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/opencsw_test.py
csw/mgar/gar/v2/lib/python/testdata/
csw/mgar/gar/v2/lib/python/testdata/README
csw/mgar/gar/v2/lib/python/testdata/__init__.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWlibpq_84.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51client.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51devel.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51rt.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql5client_8x.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWpostfix.py
csw/mgar/gar/v2/lib/python/testdata/dump_output_1.py
csw/mgar/gar/v2/lib/python/testdata/dump_output_2.py
csw/mgar/gar/v2/tests/
csw/mgar/gar/v2/tests/example_test.py
csw/mgar/gar/v2/tests/run_tests.py
csw/mgar/gar/v2/tests/static/
csw/mgar/gar/v2/tests/static/example/
csw/mgar/gar/v2/tests/static/example/Makefile
csw/mgar/gar/v2/tests/static/example/checksums
csw/mgar/gar/v2/tests/static/example/gar
Removed Paths:
-------------
csw/mgar/gar/v2/bin/checkpkg.d/checkpkg.py
csw/mgar/gar/v2/bin/checkpkg.d/checkpkg_test.py
csw/mgar/gar/v2/bin/checkpkg.d/testdata/
csw/mgar/gar/v2/lib/python/
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/checkpkg_test.py
csw/mgar/gar/v2/lib/python/gartest.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/opencsw_test.py
csw/mgar/gar/v2/lib/python/testdata/
csw/mgar/gar/v2/lib/python/testdata/README
csw/mgar/gar/v2/lib/python/testdata/__init__.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWlibpq_84.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51client.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51devel.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql51rt.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWmysql5client_8x.py
csw/mgar/gar/v2/lib/python/testdata/checkpkg_test_data_CSWpostfix.py
csw/mgar/gar/v2/lib/python/testdata/dump_output_1.py
csw/mgar/gar/v2/lib/python/testdata/dump_output_2.py
csw/mgar/gar/v2/tests/example_test.py
csw/mgar/gar/v2/tests/run_tests.py
csw/mgar/gar/v2/tests/static/
csw/mgar/gar/v2/tests/static/example/
csw/mgar/gar/v2/tests/static/example/Makefile
csw/mgar/gar/v2/tests/static/example/checksums
csw/mgar/gar/v2/tests/static/example/gar
Property Changed:
----------------
csw/mgar/gar/v2/
csw/mgar/gar/v2/pkglib/csw/depend
Property changes on: csw/mgar/gar/v2
___________________________________________________________________
Modified: svn:mergeinfo
- /csw/mgar/gar/v2:4936-6678
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-skayser:6087-6132
+ /csw/mgar/gar/v2:4936-6678
/csw/mgar/gar/v2-checkpkg:7722-7855
/csw/mgar/gar/v2-collapsed-modulations:6895
/csw/mgar/gar/v2-dirpackage:8125-8180
/csw/mgar/gar/v2-migrateconf:7082-7211
/csw/mgar/gar/v2-skayser:6087-6132
Modified: csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-libs.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-libs.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-libs.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -8,7 +8,6 @@
# unit tests and it appears to be working. The main problem is that it's not
# divided into smaller testable sections.
-import checkpkg
import os
import os.path
import copy
@@ -18,6 +17,14 @@
import sys
import textwrap
+# The following bit of code sets the correct path to Python libraries
+# distributed with GAR.
+path_list = [os.getcwd(),
+ os.path.split(sys.argv[0])[0],
+ "..", "..", "lib", "python"]
+sys.path.append(os.path.join(*path_list))
+import checkpkg
+
DUMP_BIN = "/usr/ccs/bin/dump"
def GetIsalist():
Modified: csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-obsolete-deps.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-obsolete-deps.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-obsolete-deps.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -1,11 +1,18 @@
#!/opt/csw/bin/python2.6
# $Id$
-import checkpkg
import logging
import os.path
import sys
+# The following bit of code sets the correct path to Python libraries
+# distributed with GAR.
+path_list = [os.getcwd(),
+ os.path.split(sys.argv[0])[0],
+ "..", "..", "lib", "python"]
+sys.path.append(os.path.join(*path_list))
+import checkpkg
+
OBSOLETE_DEPS = {
# "CSWfoo": {
# "hint": "Do this...",
@@ -39,5 +46,6 @@
else:
sys.exit(1)
+
if __name__ == '__main__':
main()
Modified: csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-you-can-write-your-own.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-you-can-write-your-own.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/checkpkg-you-can-write-your-own.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -6,10 +6,18 @@
Copy it and modify.
"""
-import checkpkg
import logging
import os.path
+import sys
+# The following bit of code sets the correct path to Python libraries
+# distributed with GAR.
+path_list = [os.getcwd(),
+ os.path.split(sys.argv[0])[0],
+ "..", "..", "lib", "python"]
+sys.path.append(os.path.join(*path_list))
+import checkpkg
+
def main():
options, args = checkpkg.GetOptions()
if not os.path.isdir(options.extractdir):
@@ -23,4 +31,4 @@
if __name__ == '__main__':
- main()
+ main()
Deleted: csw/mgar/gar/v2/bin/checkpkg.d/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/checkpkg.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/checkpkg.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -1,569 +0,0 @@
-# $Id$
-#
-# This is the checkpkg library, common for all checkpkg tests written in
-# Python.
-
-import itertools
-import logging
-import optparse
-import os
-import os.path
-import re
-import socket
-import sqlite3
-import subprocess
-import StringIO
-from Cheetah import Template
-
-SYSTEM_PKGMAP = "/var/sadm/install/contents"
-WS_RE = re.compile(r"\s+")
-NEEDED_SONAMES = "needed sonames"
-RUNPATH = "runpath"
-SONAME = "soname"
-CONFIG_MTIME = "mtime"
-DO_NOT_REPORT_SURPLUS = set([u"CSWcommon", u"CSWcswclassutils", u"CSWisaexec"])
-DO_NOT_REPORT_MISSING = set([u"SUNWlibC", u"SUNWcsl", u"SUNWlibms",
- u"*SUNWcslr", u"*SUNWlibC", u"*SUNWlibms",
- u"SUNWcslx"])
-SYSTEM_SYMLINKS = (
- ("/opt/csw/bdb4", ["/opt/csw/bdb42"]),
- ("/64", ["/amd64", "/sparcv9"]),
- ("/opt/csw/lib/i386", ["/opt/csw/lib"]),
-)
-
-# This shared library is present on Solaris 10 on amd64, but it's missing on
-# Solaris 8 on i386. It's okay if it's missing.
-ALLOWED_ORPHAN_SONAMES = set([u"libm.so.2"])
-DEPENDENCY_FILENAME_REGEXES = (
- (r".*\.pl", u"CSWperl"),
- (r".*\.pm", u"CSWperl"),
- (r".*\.py", u"CSWpython"),
- (r".*\.rb", u"CSWruby"),
-)
-
-REPORT_TMPL = u"""$pkgname:
-#if $missing_deps
-SUGGESTION: you may want to add some or all of the following as depends:
- (Feel free to ignore SUNW or SPRO packages)
-#for $pkg in $sorted($missing_deps)
-> $pkg
-#end for
-#end if
-#if $surplus_deps
-The following packages might be unnecessary dependencies:
-#for $pkg in $sorted($surplus_deps)
-? $pkg
-#end for
-#end if
-#if $orphan_sonames
-The following sonames don't belong to any package:
-#for $soname in $sorted($orphan_sonames)
-! $soname
-#end for
-#end if
-#if not $missing_deps and not $surplus_deps and not $orphan_sonames
-+ Dependencies of $pkgname look good.
-#end if
-"""
-
-class Error(Exception):
- pass
-
-
-class ConfigurationError(Error):
- pass
-
-
-class PackageError(Error):
- pass
-
-
-def GetOptions():
- parser = optparse.OptionParser()
- parser.add_option("-e", dest="extractdir",
- help="The directory into which the package has been extracted")
- parser.add_option("-d", "--debug", dest="debug",
- default=False, action="store_true",
- help="Turn on debugging messages")
- (options, args) = parser.parse_args()
- if not options.extractdir:
- raise ConfigurationError("ERROR: -e option is missing.")
- # Using set() to make the arguments unique.
- return options, set(args)
-
-
-class CheckpkgBase(object):
- """This class has functionality overlapping with DirectoryFormatPackage
- from the opencsw.py library. The classes should be merged.
- """
-
- def __init__(self, extractdir, pkgname):
- self.extractdir = extractdir
- self.pkgname = pkgname
- self.pkgpath = os.path.join(self.extractdir, self.pkgname)
-
- def CheckPkgpathExists(self):
- if not os.path.isdir(self.pkgpath):
- raise PackageError("%s does not exist or is not a directory"
- % self.pkgpath)
-
- def ListBinaries(self):
- """Shells out to list all the binaries from a given package.
-
- Original checkpkg code:
-
- # #########################################
- # # find all executables and dynamic libs,and list their filenames.
- # listbinaries() {
- # if [ ! -d $1 ] ; then
- # print errmsg $1 not a directory
- # rm -rf $EXTRACTDIR
- # exit 1
- # fi
- #
- # find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
- # }
- """
- self.CheckPkgpathExists()
- find_tmpl = "find %s -print | xargs file | grep ELF | nawk -F: '{print $1}'"
- find_proc = subprocess.Popen(find_tmpl % self.pkgpath,
- shell=True, stdout=subprocess.PIPE)
- stdout, stderr = find_proc.communicate()
- ret = find_proc.wait()
- if ret:
- logging.error("The find command returned an error.")
- return stdout.splitlines()
-
- def GetAllFilenames(self):
- self.CheckPkgpathExists()
- file_basenames = []
- for root, dirs, files in os.walk(self.pkgpath):
- file_basenames.extend(files)
- return file_basenames
-
- def GetDependencies(self):
- fd = open(os.path.join(self.pkgpath, "install", "depend"), "r")
- depends = {}
- for line in fd:
- fields = re.split(WS_RE, line)
- if fields[0] == "P":
- depends[fields[1]] = " ".join(fields[1:])
- fd.close()
- return depends
-
- def FormatDepsReport(self, missing_deps, surplus_deps, orphan_sonames):
- """A intermediate version in which StringIO is used."""
- namespace = {
- "pkgname": self.pkgname,
- "missing_deps": missing_deps,
- "surplus_deps": surplus_deps,
- "orphan_sonames": orphan_sonames,
- }
- t = Template.Template(REPORT_TMPL, searchList=[namespace])
- return unicode(t)
-
-
-class SystemPkgmap(object):
- """A class to hold and manipulate the /var/sadm/install/contents file.
-
- TODO: Implement timestamp checking and refreshing the cache.
- """
-
- STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
- CHECKPKG_DIR = ".checkpkg"
- SQLITE3_DBNAME_TMPL = "var-sadm-install-contents-cache-%s"
-
- def __init__(self):
- """There is no need to re-parse it each time.
-
- Read it slowly the first time and cache it for later."""
- self.cache = {}
- self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
- self.fqdn = socket.getfqdn()
- self.db_path = os.path.join(self.checkpkg_dir,
- self.SQLITE3_DBNAME_TMPL % self.fqdn)
- self.file_mtime = None
- self.cache_mtime = None
- if os.path.exists(self.db_path):
- logging.debug("Connecting to the %s database.", self.db_path)
- self.conn = sqlite3.connect(self.db_path)
- if not self.IsDatabaseUpToDate():
- logging.warning("Rebuilding the package cache, can take a few minutes.")
- self.PurgeDatabase()
- self.PopulateDatabase()
- else:
- print "Building a cache of /var/sadm/install/contents."
- print "The cache will be kept in %s." % self.db_path
- if not os.path.exists(self.checkpkg_dir):
- logging.debug("Creating %s", self.checkpkg_dir)
- os.mkdir(self.checkpkg_dir)
- self.conn = sqlite3.connect(self.db_path)
- c = self.conn.cursor()
- c.execute("""
- CREATE TABLE systempkgmap (
- id INTEGER PRIMARY KEY,
- basename TEXT,
- path TEXT,
- line TEXT
- );
- """)
- logging.debug("Creating the config table.")
- c.execute("""
- CREATE TABLE config (
- key VARCHAR(255) PRIMARY KEY,
- float_value FLOAT,
- str_value VARCHAR(255)
- );
- """)
- self.PopulateDatabase()
-
- def SymlinkDuringInstallation(self, p):
- """Emulates the effect of some symlinks present during installations."""
- p = p.replace("/opt/csw/lib/i386", "/opt/csw/lib")
-
- def PopulateDatabase(self):
- """Imports data into the database.
-
- Original bit of code from checkpkg:
-
- egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
- fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
- """
-
- system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
- stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
- # Creating a data structure:
- # soname - {<path1>: <line1>, <path2>: <line2>, ...}
- logging.debug("Building sqlite3 cache db of the %s file",
- SYSTEM_PKGMAP)
- c = self.conn.cursor()
- count = itertools.count()
- for line in system_pkgmap_fd:
- i = count.next()
- if not i % 1000:
- print "\r%s" % i,
- if stop_re.search(line):
- continue
- fields = re.split(WS_RE, line)
- pkgmap_entry_path = fields[0].split("=")[0]
- pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
- sql = "INSERT INTO systempkgmap (basename, path, line) VALUES (?, ?, ?);"
- c.execute(sql, (pkgmap_entry_base_name, pkgmap_entry_dir, line.strip()))
- print
- print "Creating the main database index."
- sql = "CREATE INDEX basename_idx ON systempkgmap(basename);"
- c.execute(sql)
- self.SetDatabaseMtime()
- self.conn.commit()
-
- def SetDatabaseMtime(self):
- c = self.conn.cursor()
- sql = "DELETE FROM config WHERE key = ?;"
- c.execute(sql, [CONFIG_MTIME])
- mtime = self.GetFileMtime()
- logging.debug("Inserting the mtime (%s) into the database.", mtime)
- sql = """
- INSERT INTO config (key, float_value)
- VALUES (?, ?);
- """
- c.execute(sql, [CONFIG_MTIME, mtime])
-
- def GetPkgmapLineByBasename(self, filename):
- if filename in self.cache:
- return self.cache[filename]
- sql = "SELECT path, line FROM systempkgmap WHERE basename = ?;"
- c = self.conn.cursor()
- c.execute(sql, [filename])
- lines = {}
- for row in c:
- lines[row[0]] = row[1]
- if len(lines) == 0:
- logging.debug("Cache doesn't contain filename %s", filename)
- self.cache[filename] = lines
- return lines
-
- def GetDatabaseMtime(self):
- if not self.cache_mtime:
- sql = """
- SELECT float_value FROM config
- WHERE key = ?;
- """
- c = self.conn.cursor()
- c.execute(sql, [CONFIG_MTIME])
- row = c.fetchone()
- if not row:
- # raise ConfigurationError("Could not find the mtime setting")
- self.cache_mtime = 1
- else:
- self.cache_mtime = row[0]
- return self.cache_mtime
-
- def GetFileMtime(self):
- if not self.file_mtime:
- stat_data = os.stat(SYSTEM_PKGMAP)
- self.file_mtime = stat_data.st_mtime
- return self.file_mtime
-
- def IsDatabaseUpToDate(self):
- f_mtime = self.GetFileMtime()
- d_mtime = self.GetDatabaseMtime()
- logging.debug("f_mtime %s, d_time: %s", f_mtime, d_mtime)
- return self.GetFileMtime() <= self.GetDatabaseMtime()
-
- def PurgeDatabase(self):
- logging.info("Purging the cache database")
- c = self.conn.cursor()
- sql = "DELETE FROM config;"
- c.execute(sql)
- sql = "DELETE FROM systempkgmap;"
- c.execute(sql)
- sql = "DROP INDEX basename_idx;"
- try:
- c.execute(sql)
- except sqlite3.OperationalError, e:
- logging.warn(e)
-
-def SharedObjectDependencies(pkgname,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename):
- """This is one of the more obscure and more important pieces of code.
-
- I tried to make it simpler, but given that the operations here involve
- whole sets of packages, it's not easy.
- """
- so_dependencies = set()
- orphan_sonames = set()
- self_provided = set()
- for binary in binaries_by_pkgname[pkgname]:
- needed_sonames = needed_sonames_by_binary[binary][NEEDED_SONAMES]
- for soname in needed_sonames:
- if soname in filenames_by_soname:
- filename = filenames_by_soname[soname]
- pkg = pkg_by_any_filename[filename]
- self_provided.add(soname)
- so_dependencies.add(pkg)
- elif soname in pkgs_by_soname:
- so_dependencies.add(pkgs_by_soname[soname])
- else:
- orphan_sonames.add(soname)
- return so_dependencies, self_provided, orphan_sonames
-
-
-def GuessDepsByFilename(pkgname, pkg_by_any_filename):
- """Guesses dependencies based on filename regexes."""
- guessed_deps = set()
- for pattern, dep_pkgname in DEPENDENCY_FILENAME_REGEXES:
- # If any file name matches, add the dep, go to the next pattern/pkg
- # combination.
- pattern_re = re.compile("^%s$" % pattern)
- for filename in pkg_by_any_filename:
- if (re.match(pattern_re, filename)
- and
- pkgname == pkg_by_any_filename[filename]):
- guessed_deps.add(dep_pkgname)
- break
- return guessed_deps
-
-
-def GuessDepsByPkgname(pkgname, pkg_by_any_filename):
- # More guessed dependencies: If one package is a substring of another, it
- # might be a hint. For example, CSWmysql51test should depend on CSWmysql51.
- # However, the rt (runtime) packages should not want to depend on the main
- # package.
- guessed_deps = set()
- all_other_pkgs = set(pkg_by_any_filename.values())
- for other_pkg in all_other_pkgs:
- other_pkg = unicode(other_pkg)
- if pkgname == other_pkg:
- continue
- if pkgname.startswith(other_pkg):
- endings = ["devel", "test", "bench", "dev"]
- for ending in endings:
- if pkgname.endswith(ending):
- guessed_deps.add(other_pkg)
- return guessed_deps
-
-
-def AnalyzeDependencies(pkgname,
- declared_dependencies,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename):
- """Gathers and merges dependency results from other functions.
-
- declared_dependencies: Dependencies that the package in question claims to
- have.
-
- binaries_by_pkgname: A dictionary mapping pkgnames (CSWfoo) to binary names
- (without paths)
-
- needed_sonames_by_binary: A dictionary mapping binary file name to
- a dictionary containing: "needed sonames",
- "soname", "rpath". Based on examining the binary
- files within the packages.
-
- pkgs_by_soname: A dictionary mapping sonames to pkgnames, based on the
- contents of the system wide pkgmap
- (/var/sadm/install/contents)
-
- filenames_by_soname: A dictionary mapping shared library sonames to filenames,
- based on files within packages
-
- pkg_by_any_filename: Mapping from file names to packages names, based on the
- contents of the packages under examination.
- """
- declared_dependencies_set = set(declared_dependencies)
-
- so_dependencies, self_provided, orphan_sonames = SharedObjectDependencies(
- pkgname,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename)
- auto_dependencies = reduce(lambda x, y: x.union(y),
- [
- so_dependencies,
- GuessDepsByFilename(pkgname, pkg_by_any_filename),
- GuessDepsByPkgname(pkgname, pkg_by_any_filename),
- ])
- missing_deps = auto_dependencies.difference(declared_dependencies_set)
- # Don't report itself as a suggested dependency.
- missing_deps = missing_deps.difference(set([pkgname]))
- missing_deps = missing_deps.difference(set(DO_NOT_REPORT_MISSING))
- surplus_deps = declared_dependencies_set.difference(auto_dependencies)
- surplus_deps = surplus_deps.difference(DO_NOT_REPORT_SURPLUS)
- orphan_sonames = orphan_sonames.difference(ALLOWED_ORPHAN_SONAMES)
- return missing_deps, surplus_deps, orphan_sonames
-
-
-def ExpandRunpath(runpath, isalist):
- # Emulating $ISALIST expansion
- if '$ISALIST' in runpath:
- expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
- else:
- expanded_list = [runpath]
- return expanded_list
-
-def ExpandSymlink(symlink, target, input_path):
- symlink_re = re.compile(r"%s(/|$)" % symlink)
- if re.search(symlink_re, input_path):
- result = input_path.replace(symlink, target)
- else:
- result = input_path
- return result
-
-def Emulate64BitSymlinks(runpath_list):
- """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
-
- Since we don't know the architecture, we'll adding both amd64 and sparcv9.
- It should be safe.
- """
- symlinked_list = []
- for runpath in runpath_list:
- for symlink, expansion_list in SYSTEM_SYMLINKS:
- for target in expansion_list:
- expanded = ExpandSymlink(symlink, target, runpath)
- if expanded not in symlinked_list:
- symlinked_list.append(expanded)
- return symlinked_list
-
-
-def SanitizeRunpath(runpath):
- ok = False
- while True:
- if runpath.endswith("/"):
- runpath = runpath[:-1]
- elif "//" in runpath:
- runpath = runpath.replace("//", "/")
- else:
- break
- return runpath
-
-
-def GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist):
- """Works out which system pkgmap lines correspond to given sonames."""
- lines_by_soname = {}
- for soname in needed_sonames:
- # This is the critical part of the algorithm: it iterates over the
- # runpath and finds the first matching one.
- runpath_found = False
- for runpath in runpath_by_needed_soname[soname]:
- runpath = SanitizeRunpath(runpath)
- runpath_list = ExpandRunpath(runpath, isalist)
- runpath_list = Emulate64BitSymlinks(runpath_list)
- soname_runpath_data = pkgmap.GetPkgmapLineByBasename(soname)
- # Emulating the install time symlinks, for instance, if the prototype contains
- # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
- # the shared library ends up in /opt/csw/lib/foo.so.0 and should be findable even when
- # RPATH does not contain $ISALIST.
- new_soname_runpath_data = {}
- for p in soname_runpath_data:
- expanded_p_list = Emulate64BitSymlinks([p])
- for expanded_p in expanded_p_list:
- new_soname_runpath_data[expanded_p] = soname_runpath_data[p]
- soname_runpath_data = new_soname_runpath_data
-
- logging.debug("%s: will be looking for %s in %s" %
- (soname, runpath_list, soname_runpath_data.keys()))
- for runpath_expanded in runpath_list:
- if runpath_expanded in soname_runpath_data:
- lines_by_soname[soname] = soname_runpath_data[runpath_expanded]
- runpath_found = True
- # This break only goes out of the inner loop,
- # need another one below to finish the outer loop.
- break
- if runpath_found:
- break
- return lines_by_soname
-
-
-def BuildIndexesBySoname(needed_sonames_by_binary):
- """Builds data structures indexed by soname.
-
- Building indexes
- {"foo.so": ["/opt/csw/lib/gcc4", "/opt/csw/lib", ...],
- ...
- }
- """
- needed_sonames = set()
- binaries_by_soname = {}
- runpath_by_needed_soname = {}
- for binary_name, data in needed_sonames_by_binary.iteritems():
- for soname in data[NEEDED_SONAMES]:
- needed_sonames.add(soname)
- if soname not in runpath_by_needed_soname:
- runpath_by_needed_soname[soname] = []
- runpath_by_needed_soname[soname].extend(data[RUNPATH])
- if soname not in binaries_by_soname:
- binaries_by_soname[soname] = set()
- binaries_by_soname[soname].add(binary_name)
- return needed_sonames, binaries_by_soname, runpath_by_needed_soname
-
-
-def ParseDumpOutput(dump_output):
- binary_data = {RUNPATH: [],
- NEEDED_SONAMES: []}
- for line in dump_output.splitlines():
- fields = re.split(WS_RE, line)
- # TODO: Make it a unit test
- # logging.debug("%s says: %s", DUMP_BIN, fields)
- if len(fields) < 3:
- continue
- if fields[1] == "NEEDED":
- binary_data[NEEDED_SONAMES].append(fields[2])
- elif fields[1] == "RUNPATH":
- binary_data[RUNPATH].extend(fields[2].split(":"))
- elif fields[1] == "SONAME":
- binary_data[SONAME] = fields[2]
- # Adding the default runtime path search option.
- binary_data[RUNPATH].append("/usr/lib/$ISALIST")
- binary_data[RUNPATH].append("/usr/lib")
- binary_data[RUNPATH].append("/lib/$ISALIST")
- binary_data[RUNPATH].append("/lib")
- return binary_data
Deleted: csw/mgar/gar/v2/bin/checkpkg.d/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/checkpkg_test.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/checkpkg_test.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -1,589 +0,0 @@
-#!/opt/csw/bin/python2.6
-# $Id$
-
-import unittest
-import mox
-import difflib
-import checkpkg
-import testdata.checkpkg_test_data_CSWmysql51rt as d1
-import testdata.checkpkg_test_data_CSWmysql51client as d2
-import testdata.checkpkg_test_data_CSWmysql51 as d3
-import testdata.checkpkg_test_data_CSWmysql51devel as d4
-import testdata.checkpkg_test_data_CSWlibpq_84 as d5
-import testdata.checkpkg_test_data_CSWmysql5client_8x as d6
-import testdata.checkpkg_test_data_CSWpostfix as d7
-import testdata.dump_output_1 as dump_1
-import testdata.dump_output_2 as dump_2
-
-"""A set of unit tests for the library checking code.
-
-A bunch of lines to test in the interactive Python shell.
-
-import sys
-sys.path.append("gar/bin/checkpkg.d")
-import checkpkg
-import testdata.checkpkg_test_data_CSWmysql5client_8x as d6
-
-checkpkg.SharedObjectDependencies("CSWmysql5client",
-d6.DATA_BINARIES_BY_PKGNAME, d6.DATA_NEEDED_SONAMES_BY_BINARY,
-d6.DATA_PKGS_BY_FILENAME, d6.DATA_FILENAMES_BY_SONAME,
-d6.DATA_PKG_BY_ANY_FILENAME)
-
-sqlite3 ~/.checkpkg/var-sadm-install-contents-cache-build8x
-SELECT * FROM systempkgmap WHERE basename = 'libncursesw.so.5';
-"""
-
-class DependenciesUnitTest_1(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d1.DATA_PKGNAME,
- d1.DATA_DECLARED_DEPENDENCIES,
- d1.DATA_BINARIES_BY_PKGNAME,
- d1.DATA_NEEDED_SONAMES_BY_BINARY,
- d1.DATA_PKGS_BY_FILENAME,
- d1.DATA_FILENAMES_BY_SONAME,
- d1.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_2(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d2.DATA_PKGNAME,
- d2.DATA_DECLARED_DEPENDENCIES,
- d2.DATA_BINARIES_BY_PKGNAME,
- d2.DATA_NEEDED_SONAMES_BY_BINARY,
- d2.DATA_PKGS_BY_FILENAME,
- d2.DATA_FILENAMES_BY_SONAME,
- d2.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_3(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d3.DATA_PKGNAME,
- d3.DATA_DECLARED_DEPENDENCIES,
- d3.DATA_BINARIES_BY_PKGNAME,
- d3.DATA_NEEDED_SONAMES_BY_BINARY,
- d3.DATA_PKGS_BY_FILENAME,
- d3.DATA_FILENAMES_BY_SONAME,
- d3.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([u'CSWmysql51client']), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set(['CSWmysql51rt'])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_4(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d4.DATA_PKGNAME,
- d4.DATA_DECLARED_DEPENDENCIES,
- d4.DATA_BINARIES_BY_PKGNAME,
- d4.DATA_NEEDED_SONAMES_BY_BINARY,
- d4.DATA_PKGS_BY_FILENAME,
- d4.DATA_FILENAMES_BY_SONAME,
- d4.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_5(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d5.DATA_PKGNAME,
- d5.DATA_DECLARED_DEPENDENCIES,
- d5.DATA_BINARIES_BY_PKGNAME,
- d5.DATA_NEEDED_SONAMES_BY_BINARY,
- d5.DATA_PKGS_BY_FILENAME,
- d5.DATA_FILENAMES_BY_SONAME,
- d5.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- # This tends to report itself...
- expected = set([u'SUNWgss'])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_6(unittest.TestCase):
-
- def setUp(self):
- (self.missing_deps,
- self.surplus_deps,
- self.orphan_sonames) = checkpkg.AnalyzeDependencies(
- d6.DATA_PKGNAME,
- d6.DATA_DECLARED_DEPENDENCIES,
- d6.DATA_BINARIES_BY_PKGNAME,
- d6.DATA_NEEDED_SONAMES_BY_BINARY,
- d6.DATA_PKGS_BY_FILENAME,
- d6.DATA_FILENAMES_BY_SONAME,
- d6.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_7(unittest.TestCase):
-
- def setUp(self):
- (self.missing_deps,
- self.surplus_deps,
- self.orphan_sonames) = checkpkg.AnalyzeDependencies(
- d7.DATA_PKGNAME,
- d7.DATA_DECLARED_DEPENDENCIES,
- d7.DATA_BINARIES_BY_PKGNAME,
- d7.DATA_NEEDED_SONAMES_BY_BINARY,
- d7.DATA_PKGS_BY_FILENAME,
- d7.DATA_FILENAMES_BY_SONAME,
- d7.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class GuessDepsUnitTest(unittest.TestCase):
-
- def testGuessDepsByFilename1(self):
- expected = set([u"CSWpython"])
- pkgname = u"CSWfoo"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByFilename(pkgname, pkg_by_filename))
-
- def testGuessDepsByFilename2(self):
- expected = set([])
- pkgname = u"CSWfoo"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWbar",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByFilename(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname1(self):
- expected = set([u"CSWfoo"])
- pkgname = u"CSWfoo-devel"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/bin/barfoo": u"CSWfoobar",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByPkgname(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname2(self):
- expected = set([])
- pkgname = u"CSWzfoo-devel"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/bin/barfoo": u"CSWfoobar",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByPkgname(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname3(self):
- self.assertEqual(set([u"CSWmysql51"]),
- checkpkg.GuessDepsByPkgname(u"CSWmysql51devel",
- d4.DATA_PKG_BY_ANY_FILENAME))
-
- def testGuessDepsByPkgname4(self):
- data1 = set(['CSWmysql51', 'CSWmysql51rt', 'CSWmysql51test',
- 'CSWmysql51client', 'CSWmysql51bench', 'CSWmysql51devel'])
- data2 = dict(((x, x) for x in data1))
- self.assertEqual(set([u"CSWmysql51"]), checkpkg.GuessDepsByPkgname(u"CSWmysql51devel", data2))
-
- def testGuessDepsByPkgname4(self):
- data1 = set(['CSWmysql51', 'CSWmysql51rt', 'CSWmysql51test',
- 'CSWmysql51client', 'CSWmysql51bench', 'CSWmysql51devel'])
- data2 = dict(((x, x) for x in data1))
- self.assertEqual(set([]), checkpkg.GuessDepsByPkgname(u"CSWmysql51rt", data2))
-
-
-class GetLinesBySonameUnitTest(unittest.TestCase):
-
- class PkgmapStub(object):
-
- def __init__(self, cache):
- self.cache = cache
-
- def GetPkgmapLineByBasename(self, soname):
- return self.cache[soname]
-
- def setUp(self):
- self.pkgmap_mocker = mox.Mox()
-
- def testExpandRunpath_1(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/lib/$ISALIST"
- expected = ["/opt/csw/lib/foo", "/opt/csw/lib/bar"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
-
- def testExpandRunpath_2(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/mysql5/lib/$ISALIST/mysql"
- expected = ["/opt/csw/mysql5/lib/foo/mysql", "/opt/csw/mysql5/lib/bar/mysql"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
-
- def testEmulate64BitSymlinks_1(self):
- runpath_list = ["/opt/csw/mysql5/lib/foo/mysql/64"]
- expected = "/opt/csw/mysql5/lib/foo/mysql/amd64"
- self.assertTrue(expected in checkpkg.Emulate64BitSymlinks(runpath_list))
-
- def testEmulate64BitSymlinks_2(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/amd64/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_3(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/sparcv9/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_4(self):
- """No repeated paths because of symlink expansion"""
- runpath_list = ["/opt/csw/lib"]
- expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertEquals(1, len(result), "len(%s) != %s" % (result, 1))
-
- def testEmulateSymlinks_3(self):
- runpath_list = ["/opt/csw/bdb4"]
- expected = "/opt/csw/bdb42"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_4(self):
- runpath_list = ["/opt/csw/bdb42"]
- expected = "/opt/csw/bdb42"
- not_expected = "/opt/csw/bdb422"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result, "%s is in %s" % (not_expected, result))
-
- def testEmulateSymlinks_5(self):
- """Install time symlink expansion."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_6(self):
- """ExpandSymlink for /opt/csw/lib/i386."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- not_expected = "/opt/csw/lib/i386"
- result = checkpkg.ExpandSymlink("/opt/csw/lib/i386", "/opt/csw/lib", "/opt/csw/lib/i386")
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result, "%s is in %s" % (not_expected, result))
-
- def testGetLinesBySoname(self):
- expected = {'foo.so.1': '/opt/csw/lib/isa-value-1/foo.so.1 foo'}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo")
- lines1 = {"/opt/csw/lib/isa-value-1": "/opt/csw/lib/isa-value-1/foo.so.1 foo",
- "/usr/lib": "/usr/lib/foo.so.1 foo"}
- # pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- pkgmap.GetPkgmapLineByBasename("foo")
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {"foo.so.1": ["/opt/csw/lib/$ISALIST", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_3(self):
- expected = {'foo.so.1': '/opt/csw/lib/isa-value-1/foo.so.1 foo'}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo")
- lines1 = {
- "/opt/csw/lib/isa-value-1": "/opt/csw/lib/isa-value-1/foo.so.1 foo",
- "/opt/csw/lib": "/opt/csw/lib/foo.so.1 foo",
- "/usr/lib": "/usr/lib/foo.so.1 foo"}
- # pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- pkgmap.GetPkgmapLineByBasename("foo")
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {
- "foo.so.1": ["/opt/csw/lib/$ISALIST", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(
- pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_4(self):
- """A more complex test, four ISAs."""
- expected = {'foo.so.1': '/opt/csw/lib/isa-value-1/foo.so.1 foo'}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo")
- lines1 = {
- "/opt/csw/lib/isa-value-1":
- "/opt/csw/lib/isa-value-1/foo.so.1 foo",
- "/opt/csw/mysql5/lib/isa-value-2":
- "/opt/csw/mysql5/lib/isa-value-2/foo.so.1 foo",
- "/opt/csw/mysql5/lib/isa-value-1":
- "/opt/csw/mysql5/lib/isa-value-1/foo.so.1 foo",
- "/opt/csw/lib":
- "/opt/csw/lib/foo.so.1 foo",
- "/usr/lib":
- "/usr/lib/foo.so.1 foo"}
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- pkgmap.GetPkgmapLineByBasename("foo")
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {
- "foo.so.1": ["/opt/csw/mysql5/lib/$ISALIST/mysql",
- "/opt/csw/lib/$ISALIST",
- "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(
- pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_5(self):
- """Based on CSWmysql5client on build8x."""
- soname = u'libm.so.1'
- expected = {u'libm.so.1': u'/usr/lib/libm.so.1 f none 0755 root bin '
- u'99844 3884 1050525375 SUNWlibms\n'}
-
- pkgmap_stub = self.PkgmapStub(d6.DATA_PKGMAP_CACHE)
- (needed_sonames,
- binaries_by_soname,
- runpath_by_needed_soname) = checkpkg.BuildIndexesBySoname(
- d6.DATA_NEEDED_SONAMES_BY_BINARY)
- result = checkpkg.GetLinesBySoname(
- pkgmap_stub,
- set([soname]),
- runpath_by_needed_soname,
- d6.DATA_ISALIST)
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_6(self):
- """Based on CSWmysql5client on build8x."""
- soname = u'libz.so.1'
- expected = {u'libz.so.1': u'/opt/csw/lib/pentium_pro+mmx/libz.so.1=libz.so.1.2.3 '
- u's none CSWzlib\n'}
- pkgmap_stub = self.PkgmapStub(d6.DATA_PKGMAP_CACHE)
- (needed_sonames,
- binaries_by_soname,
- runpath_by_needed_soname) = checkpkg.BuildIndexesBySoname(
- d6.DATA_NEEDED_SONAMES_BY_BINARY)
- result = checkpkg.GetLinesBySoname(
- pkgmap_stub,
- set([soname]),
- runpath_by_needed_soname,
- d6.DATA_ISALIST)
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_7(self):
- """A test for 64-bit symlink expansion."""
- soname = u'libncursesw.so.5'
- # To test the 64-bit symlink expansion
- expected = {
- u'libncursesw.so.5':
- u'/opt/csw/lib/amd64/libncursesw.so.5=libncursesw.so.5.7 '
- u's none CSWncurses\n'}
- pkgmap_stub = self.PkgmapStub(d6.DATA_PKGMAP_CACHE)
- (needed_sonames,
- binaries_by_soname,
- runpath_by_needed_soname) = checkpkg.BuildIndexesBySoname(
- d6.DATA_NEEDED_SONAMES_BY_BINARY)
- # The original data did not have amd64 in the isalist.
- isalist = ['amd64', 'pentium_pro+mmx', 'pentium_pro', 'pentium+mmx',
- 'pentium', 'i486', 'i386', 'i86']
- result = checkpkg.GetLinesBySoname(
- pkgmap_stub,
- set([soname]),
- runpath_by_needed_soname,
- isalist)
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_8(self):
- expected = {'foo.so.1': '/opt/csw/postgresql/lib/foo.so.1 foo'}
- lines1 = {"/opt/csw/postgresql/lib": "/opt/csw/postgresql/lib/foo.so.1 foo"}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {"foo.so.1": ["/opt/csw/postgresql/lib/", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_9(self):
- """Emulation of binaries installed into /opt/csw/lib/i386.
-
- The problem is that /opt/csw/lib/i386 is a symlink and the binaries
- end up in /opt/csw/lib instead.
- """
- expected = {'foo.so.0': '/opt/csw/lib/i386/foo.so.0 foo'}
- lines1 = {"/opt/csw/lib/i386": "/opt/csw/lib/i386/foo.so.0 foo"}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo.so.0").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- needed_sonames = set(["foo.so.0"])
- runpath_by_needed_soname = {"foo.so.0": ["/opt/csw/lib", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testSanitizeRunpath_1(self):
- self.assertEqual("/opt/csw/lib", checkpkg.SanitizeRunpath("/opt/csw/lib/"))
-
- def testSanitizeRunpath_2(self):
- self.assertEqual("/opt/csw/lib", checkpkg.SanitizeRunpath("/opt//csw////lib/"))
-
-
-
-class ParseDumpOutputUnitTest(unittest.TestCase):
-
- def test_1(self):
- expected = {
- 'soname': 'libmysqlclient.so.15',
- 'runpath': ['/opt/csw/lib/$ISALIST',
- '/opt/csw/lib',
- '/opt/csw/mysql5/lib/$ISALIST',
- '/opt/csw/mysql5/lib',
- '/opt/csw/mysql5/lib/$ISALIST/mysql',
- # These four are artificially appended
- '/usr/lib/$ISALIST',
- '/usr/lib',
- '/lib/$ISALIST',
- '/lib'],
- 'needed sonames': ['librt.so.1',
- 'libresolv.so.2',
- 'libc.so.1',
- 'libgen.so.1',
- 'libsocket.so.1',
- 'libnsl.so.1',
- 'libm.so.1',
- 'libz.so.1']}
- self.assertEqual(expected,
- checkpkg.ParseDumpOutput(dump_1.DATA_DUMP_OUTPUT))
-
- def test_2(self):
- expected_runpath = ['/usr/lib/$ISALIST', '/usr/lib', '/lib/$ISALIST', '/lib']
- self.assertEqual(
- expected_runpath,
- checkpkg.ParseDumpOutput(dump_2.DATA_DUMP_OUTPUT)["runpath"])
-
-
-class FormatDepsReportUnitTest(unittest.TestCase):
-
- def AssertTextEqual(self, text1, text2):
- difference = "\n".join(difflib.context_diff(text2.splitlines(), text1.splitlines()))
- self.assertEqual(text1, text2, difference)
-
- def testAll(self):
- missing_deps = set([u'SUNWgss', u'*SUNWlxsl'])
- surplus_deps = set(['CSWsudo', 'CSWlibxslt'])
- orphan_sonames = set([u'libm.so.2'])
- testdata = (missing_deps, surplus_deps, orphan_sonames)
- checker = checkpkg.CheckpkgBase("/tmp/nonexistent", "CSWfoo")
- expected = u"""CSWfoo:
-SUGGESTION: you may want to add some or all of the following as depends:
- (Feel free to ignore SUNW or SPRO packages)
-> *SUNWlxsl
-> SUNWgss
-The following packages might be unnecessary dependencies:
-? CSWlibxslt
-? CSWsudo
-The following sonames don't belong to any package:
-! libm.so.2
-"""
- result = checker.FormatDepsReport(*testdata)
- self.AssertTextEqual(result, expected)
-
- def testNone(self):
- missing_deps = set([])
- surplus_deps = set([])
- orphan_sonames = set([])
- testdata = (missing_deps, surplus_deps, orphan_sonames)
- checker = checkpkg.CheckpkgBase("/tmp/nonexistent", "CSWfoo")
- expected = u"""CSWfoo:
-+ Dependencies of CSWfoo look good.
-"""
- result = checker.FormatDepsReport(*testdata)
- self.AssertTextEqual(result, expected)
-
-
-if __name__ == '__main__':
- unittest.main()
Modified: csw/mgar/gar/v2/bin/checkpkg.d/update_contents_cache.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg.d/update_contents_cache.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/bin/checkpkg.d/update_contents_cache.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -5,9 +5,18 @@
# This file only creates an instance of SystemPkgmap in order to update the
# package cache (if necessary), and display the information about the update.
-import checkpkg
+import os
+import os.path
+import sys
import logging
+# The following bit of code sets the correct path to Python libraries
+# distributed with GAR.
+path_list = [os.getcwd(),
+ os.path.split(sys.argv[0])[0],
+ "..", "..", "lib", "python"]
+sys.path.append(os.path.join(*path_list))
+import checkpkg
def main():
print "Checking if the package cache is up to date."
Modified: csw/mgar/gar/v2/gar.pkg.mk
===================================================================
--- csw/mgar/gar/v2/gar.pkg.mk 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/gar.pkg.mk 2010-01-26 10:43:08 UTC (rev 8181)
@@ -693,7 +693,7 @@
# We depend on extract as the additional package files (like .gspec) must be
# unpacked to global/ for packaging. E. g. 'merge' depends only on the specific
# modulations and does not fill global/.
-ENABLE_CHECK ?= 1
+_package: ENABLE_CHECK ?= 1
_package: validateplatform extract-global merge $(SPKG_DESTDIRS) pre-package $(PACKAGE_TARGETS) post-package $(if $(ENABLE_CHECK),pkgcheck)
@$(MAKECOOKIE)
@@ -705,6 +705,18 @@
@echo
@$(DONADA)
+dirpackage: _DIRPACKAGE=1
+dirpackage: ENABLE_CHECK=
+dirpackage: _package
+ @echo "The following packages have been built:"
+ @echo
+ @$(MAKE) -s PLATFORM=$(PLATFORM) _dirpkgshow
+ @echo
+ @$(DONADA)
+
+_dirpkgshow:
+ @$(foreach SPEC,$(_PKG_SPECS),echo " $(SPKG_SPOOLDIR)/$(SPEC)";)
+
_pkgshow:
@$(foreach SPEC,$(_PKG_SPECS),printf " %-20s %s\n" $(SPEC) $(SPKG_EXPORT)/$(shell $(call _PKG_ENV,$(SPEC)) $(GARBIN)/mkpackage -qs $(WORKDIR)/$(SPEC).gspec -D pkgfile).gz;)
@@ -720,7 +732,7 @@
--pkgbase $(SPKG_PKGBASE) \
--pkgroot $(SPKG_PKGROOT) \
-v WORKDIR_FIRSTMOD=../build-$(firstword $(MODULATIONS)) \
- --compress \
+ $(if $(_DIRPACKAGE),--notransfer --nocompress,--compress) \
$(MKPACKAGE_ARGS) ) || exit 2
@$(MAKECOOKIE)
@@ -754,6 +766,8 @@
repackage: pkgreset package
+redirpackage: pkgreset dirpackage
+
# This rule automatically logs into every host where a package for this software should
# be built. It is especially suited for automated build bots.
platforms:
Deleted: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2-dirpackage/lib/python/checkpkg.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -1,569 +0,0 @@
-# $Id$
-#
-# This is the checkpkg library, common for all checkpkg tests written in
-# Python.
-
-import itertools
-import logging
-import optparse
-import os
-import os.path
-import re
-import socket
-import sqlite3
-import subprocess
-import StringIO
-from Cheetah import Template
-
-SYSTEM_PKGMAP = "/var/sadm/install/contents"
-WS_RE = re.compile(r"\s+")
-NEEDED_SONAMES = "needed sonames"
-RUNPATH = "runpath"
-SONAME = "soname"
-CONFIG_MTIME = "mtime"
-DO_NOT_REPORT_SURPLUS = set([u"CSWcommon", u"CSWcswclassutils", u"CSWisaexec"])
-DO_NOT_REPORT_MISSING = set([u"SUNWlibC", u"SUNWcsl", u"SUNWlibms",
- u"*SUNWcslr", u"*SUNWlibC", u"*SUNWlibms",
- u"SUNWcslx"])
-SYSTEM_SYMLINKS = (
- ("/opt/csw/bdb4", ["/opt/csw/bdb42"]),
- ("/64", ["/amd64", "/sparcv9"]),
- ("/opt/csw/lib/i386", ["/opt/csw/lib"]),
-)
-
-# This shared library is present on Solaris 10 on amd64, but it's missing on
-# Solaris 8 on i386. It's okay if it's missing.
-ALLOWED_ORPHAN_SONAMES = set([u"libm.so.2"])
-DEPENDENCY_FILENAME_REGEXES = (
- (r".*\.pl", u"CSWperl"),
- (r".*\.pm", u"CSWperl"),
- (r".*\.py", u"CSWpython"),
- (r".*\.rb", u"CSWruby"),
-)
-
-REPORT_TMPL = u"""$pkgname:
-#if $missing_deps
-SUGGESTION: you may want to add some or all of the following as depends:
- (Feel free to ignore SUNW or SPRO packages)
-#for $pkg in $sorted($missing_deps)
-> $pkg
-#end for
-#end if
-#if $surplus_deps
-The following packages might be unnecessary dependencies:
-#for $pkg in $sorted($surplus_deps)
-? $pkg
-#end for
-#end if
-#if $orphan_sonames
-The following sonames don't belong to any package:
-#for $soname in $sorted($orphan_sonames)
-! $soname
-#end for
-#end if
-#if not $missing_deps and not $surplus_deps and not $orphan_sonames
-+ Dependencies of $pkgname look good.
-#end if
-"""
-
-class Error(Exception):
- pass
-
-
-class ConfigurationError(Error):
- pass
-
-
-class PackageError(Error):
- pass
-
-
-def GetOptions():
- parser = optparse.OptionParser()
- parser.add_option("-e", dest="extractdir",
- help="The directory into which the package has been extracted")
- parser.add_option("-d", "--debug", dest="debug",
- default=False, action="store_true",
- help="Turn on debugging messages")
- (options, args) = parser.parse_args()
- if not options.extractdir:
- raise ConfigurationError("ERROR: -e option is missing.")
- # Using set() to make the arguments unique.
- return options, set(args)
-
-
-class CheckpkgBase(object):
- """This class has functionality overlapping with DirectoryFormatPackage
- from the opencsw.py library. The classes should be merged.
- """
-
- def __init__(self, extractdir, pkgname):
- self.extractdir = extractdir
- self.pkgname = pkgname
- self.pkgpath = os.path.join(self.extractdir, self.pkgname)
-
- def CheckPkgpathExists(self):
- if not os.path.isdir(self.pkgpath):
- raise PackageError("%s does not exist or is not a directory"
- % self.pkgpath)
-
- def ListBinaries(self):
- """Shells out to list all the binaries from a given package.
-
- Original checkpkg code:
-
- # #########################################
- # # find all executables and dynamic libs,and list their filenames.
- # listbinaries() {
- # if [ ! -d $1 ] ; then
- # print errmsg $1 not a directory
- # rm -rf $EXTRACTDIR
- # exit 1
- # fi
- #
- # find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
- # }
- """
- self.CheckPkgpathExists()
- find_tmpl = "find %s -print | xargs file | grep ELF | nawk -F: '{print $1}'"
- find_proc = subprocess.Popen(find_tmpl % self.pkgpath,
- shell=True, stdout=subprocess.PIPE)
- stdout, stderr = find_proc.communicate()
- ret = find_proc.wait()
- if ret:
- logging.error("The find command returned an error.")
- return stdout.splitlines()
-
- def GetAllFilenames(self):
- self.CheckPkgpathExists()
- file_basenames = []
- for root, dirs, files in os.walk(self.pkgpath):
- file_basenames.extend(files)
- return file_basenames
-
- def GetDependencies(self):
- fd = open(os.path.join(self.pkgpath, "install", "depend"), "r")
- depends = {}
- for line in fd:
- fields = re.split(WS_RE, line)
- if fields[0] == "P":
- depends[fields[1]] = " ".join(fields[1:])
- fd.close()
- return depends
-
- def FormatDepsReport(self, missing_deps, surplus_deps, orphan_sonames):
- """A intermediate version in which StringIO is used."""
- namespace = {
- "pkgname": self.pkgname,
- "missing_deps": missing_deps,
- "surplus_deps": surplus_deps,
- "orphan_sonames": orphan_sonames,
- }
- t = Template.Template(REPORT_TMPL, searchList=[namespace])
- return unicode(t)
-
-
-class SystemPkgmap(object):
- """A class to hold and manipulate the /var/sadm/install/contents file.
-
- TODO: Implement timestamp checking and refreshing the cache.
- """
-
- STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
- CHECKPKG_DIR = ".checkpkg"
- SQLITE3_DBNAME_TMPL = "var-sadm-install-contents-cache-%s"
-
- def __init__(self):
- """There is no need to re-parse it each time.
-
- Read it slowly the first time and cache it for later."""
- self.cache = {}
- self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
- self.fqdn = socket.getfqdn()
- self.db_path = os.path.join(self.checkpkg_dir,
- self.SQLITE3_DBNAME_TMPL % self.fqdn)
- self.file_mtime = None
- self.cache_mtime = None
- if os.path.exists(self.db_path):
- logging.debug("Connecting to the %s database.", self.db_path)
- self.conn = sqlite3.connect(self.db_path)
- if not self.IsDatabaseUpToDate():
- logging.warning("Rebuilding the package cache, can take a few minutes.")
- self.PurgeDatabase()
- self.PopulateDatabase()
- else:
- print "Building a cache of /var/sadm/install/contents."
- print "The cache will be kept in %s." % self.db_path
- if not os.path.exists(self.checkpkg_dir):
- logging.debug("Creating %s", self.checkpkg_dir)
- os.mkdir(self.checkpkg_dir)
- self.conn = sqlite3.connect(self.db_path)
- c = self.conn.cursor()
- c.execute("""
- CREATE TABLE systempkgmap (
- id INTEGER PRIMARY KEY,
- basename TEXT,
- path TEXT,
- line TEXT
- );
- """)
- logging.debug("Creating the config table.")
- c.execute("""
- CREATE TABLE config (
- key VARCHAR(255) PRIMARY KEY,
- float_value FLOAT,
- str_value VARCHAR(255)
- );
- """)
- self.PopulateDatabase()
-
- def SymlinkDuringInstallation(self, p):
- """Emulates the effect of some symlinks present during installations."""
- p = p.replace("/opt/csw/lib/i386", "/opt/csw/lib")
-
- def PopulateDatabase(self):
- """Imports data into the database.
-
- Original bit of code from checkpkg:
-
- egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
- fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
- """
-
- system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
- stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
- # Creating a data structure:
- # soname - {<path1>: <line1>, <path2>: <line2>, ...}
- logging.debug("Building sqlite3 cache db of the %s file",
- SYSTEM_PKGMAP)
- c = self.conn.cursor()
- count = itertools.count()
- for line in system_pkgmap_fd:
- i = count.next()
- if not i % 1000:
- print "\r%s" % i,
- if stop_re.search(line):
- continue
- fields = re.split(WS_RE, line)
- pkgmap_entry_path = fields[0].split("=")[0]
- pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
- sql = "INSERT INTO systempkgmap (basename, path, line) VALUES (?, ?, ?);"
- c.execute(sql, (pkgmap_entry_base_name, pkgmap_entry_dir, line.strip()))
- print
- print "Creating the main database index."
- sql = "CREATE INDEX basename_idx ON systempkgmap(basename);"
- c.execute(sql)
- self.SetDatabaseMtime()
- self.conn.commit()
-
- def SetDatabaseMtime(self):
- c = self.conn.cursor()
- sql = "DELETE FROM config WHERE key = ?;"
- c.execute(sql, [CONFIG_MTIME])
- mtime = self.GetFileMtime()
- logging.debug("Inserting the mtime (%s) into the database.", mtime)
- sql = """
- INSERT INTO config (key, float_value)
- VALUES (?, ?);
- """
- c.execute(sql, [CONFIG_MTIME, mtime])
-
- def GetPkgmapLineByBasename(self, filename):
- if filename in self.cache:
- return self.cache[filename]
- sql = "SELECT path, line FROM systempkgmap WHERE basename = ?;"
- c = self.conn.cursor()
- c.execute(sql, [filename])
- lines = {}
- for row in c:
- lines[row[0]] = row[1]
- if len(lines) == 0:
- logging.debug("Cache doesn't contain filename %s", filename)
- self.cache[filename] = lines
- return lines
-
- def GetDatabaseMtime(self):
- if not self.cache_mtime:
- sql = """
- SELECT float_value FROM config
- WHERE key = ?;
- """
- c = self.conn.cursor()
- c.execute(sql, [CONFIG_MTIME])
- row = c.fetchone()
- if not row:
- # raise ConfigurationError("Could not find the mtime setting")
- self.cache_mtime = 1
- else:
- self.cache_mtime = row[0]
- return self.cache_mtime
-
- def GetFileMtime(self):
- if not self.file_mtime:
- stat_data = os.stat(SYSTEM_PKGMAP)
- self.file_mtime = stat_data.st_mtime
- return self.file_mtime
-
- def IsDatabaseUpToDate(self):
- f_mtime = self.GetFileMtime()
- d_mtime = self.GetDatabaseMtime()
- logging.debug("f_mtime %s, d_time: %s", f_mtime, d_mtime)
- return self.GetFileMtime() <= self.GetDatabaseMtime()
-
- def PurgeDatabase(self):
- logging.info("Purging the cache database")
- c = self.conn.cursor()
- sql = "DELETE FROM config;"
- c.execute(sql)
- sql = "DELETE FROM systempkgmap;"
- c.execute(sql)
- sql = "DROP INDEX basename_idx;"
- try:
- c.execute(sql)
- except sqlite3.OperationalError, e:
- logging.warn(e)
-
-def SharedObjectDependencies(pkgname,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename):
- """This is one of the more obscure and more important pieces of code.
-
- I tried to make it simpler, but given that the operations here involve
- whole sets of packages, it's not easy.
- """
- so_dependencies = set()
- orphan_sonames = set()
- self_provided = set()
- for binary in binaries_by_pkgname[pkgname]:
- needed_sonames = needed_sonames_by_binary[binary][NEEDED_SONAMES]
- for soname in needed_sonames:
- if soname in filenames_by_soname:
- filename = filenames_by_soname[soname]
- pkg = pkg_by_any_filename[filename]
- self_provided.add(soname)
- so_dependencies.add(pkg)
- elif soname in pkgs_by_soname:
- so_dependencies.add(pkgs_by_soname[soname])
- else:
- orphan_sonames.add(soname)
- return so_dependencies, self_provided, orphan_sonames
-
-
-def GuessDepsByFilename(pkgname, pkg_by_any_filename):
- """Guesses dependencies based on filename regexes."""
- guessed_deps = set()
- for pattern, dep_pkgname in DEPENDENCY_FILENAME_REGEXES:
- # If any file name matches, add the dep, go to the next pattern/pkg
- # combination.
- pattern_re = re.compile("^%s$" % pattern)
- for filename in pkg_by_any_filename:
- if (re.match(pattern_re, filename)
- and
- pkgname == pkg_by_any_filename[filename]):
- guessed_deps.add(dep_pkgname)
- break
- return guessed_deps
-
-
-def GuessDepsByPkgname(pkgname, pkg_by_any_filename):
- # More guessed dependencies: If one package is a substring of another, it
- # might be a hint. For example, CSWmysql51test should depend on CSWmysql51.
- # However, the rt (runtime) packages should not want to depend on the main
- # package.
- guessed_deps = set()
- all_other_pkgs = set(pkg_by_any_filename.values())
- for other_pkg in all_other_pkgs:
- other_pkg = unicode(other_pkg)
- if pkgname == other_pkg:
- continue
- if pkgname.startswith(other_pkg):
- endings = ["devel", "test", "bench", "dev"]
- for ending in endings:
- if pkgname.endswith(ending):
- guessed_deps.add(other_pkg)
- return guessed_deps
-
-
-def AnalyzeDependencies(pkgname,
- declared_dependencies,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename):
- """Gathers and merges dependency results from other functions.
-
- declared_dependencies: Dependencies that the package in question claims to
- have.
-
- binaries_by_pkgname: A dictionary mapping pkgnames (CSWfoo) to binary names
- (without paths)
-
- needed_sonames_by_binary: A dictionary mapping binary file name to
- a dictionary containing: "needed sonames",
- "soname", "rpath". Based on examining the binary
- files within the packages.
-
- pkgs_by_soname: A dictionary mapping sonames to pkgnames, based on the
- contents of the system wide pkgmap
- (/var/sadm/install/contents)
-
- filenames_by_soname: A dictionary mapping shared library sonames to filenames,
- based on files within packages
-
- pkg_by_any_filename: Mapping from file names to packages names, based on the
- contents of the packages under examination.
- """
- declared_dependencies_set = set(declared_dependencies)
-
- so_dependencies, self_provided, orphan_sonames = SharedObjectDependencies(
- pkgname,
- binaries_by_pkgname,
- needed_sonames_by_binary,
- pkgs_by_soname,
- filenames_by_soname,
- pkg_by_any_filename)
- auto_dependencies = reduce(lambda x, y: x.union(y),
- [
- so_dependencies,
- GuessDepsByFilename(pkgname, pkg_by_any_filename),
- GuessDepsByPkgname(pkgname, pkg_by_any_filename),
- ])
- missing_deps = auto_dependencies.difference(declared_dependencies_set)
- # Don't report itself as a suggested dependency.
- missing_deps = missing_deps.difference(set([pkgname]))
- missing_deps = missing_deps.difference(set(DO_NOT_REPORT_MISSING))
- surplus_deps = declared_dependencies_set.difference(auto_dependencies)
- surplus_deps = surplus_deps.difference(DO_NOT_REPORT_SURPLUS)
- orphan_sonames = orphan_sonames.difference(ALLOWED_ORPHAN_SONAMES)
- return missing_deps, surplus_deps, orphan_sonames
-
-
-def ExpandRunpath(runpath, isalist):
- # Emulating $ISALIST expansion
- if '$ISALIST' in runpath:
- expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
- else:
- expanded_list = [runpath]
- return expanded_list
-
-def ExpandSymlink(symlink, target, input_path):
- symlink_re = re.compile(r"%s(/|$)" % symlink)
- if re.search(symlink_re, input_path):
- result = input_path.replace(symlink, target)
- else:
- result = input_path
- return result
-
-def Emulate64BitSymlinks(runpath_list):
- """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
-
- Since we don't know the architecture, we'll adding both amd64 and sparcv9.
- It should be safe.
- """
- symlinked_list = []
- for runpath in runpath_list:
- for symlink, expansion_list in SYSTEM_SYMLINKS:
- for target in expansion_list:
- expanded = ExpandSymlink(symlink, target, runpath)
- if expanded not in symlinked_list:
- symlinked_list.append(expanded)
- return symlinked_list
-
-
-def SanitizeRunpath(runpath):
- ok = False
- while True:
- if runpath.endswith("/"):
- runpath = runpath[:-1]
- elif "//" in runpath:
- runpath = runpath.replace("//", "/")
- else:
- break
- return runpath
-
-
-def GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist):
- """Works out which system pkgmap lines correspond to given sonames."""
- lines_by_soname = {}
- for soname in needed_sonames:
- # This is the critical part of the algorithm: it iterates over the
- # runpath and finds the first matching one.
- runpath_found = False
- for runpath in runpath_by_needed_soname[soname]:
- runpath = SanitizeRunpath(runpath)
- runpath_list = ExpandRunpath(runpath, isalist)
- runpath_list = Emulate64BitSymlinks(runpath_list)
- soname_runpath_data = pkgmap.GetPkgmapLineByBasename(soname)
- # Emulating the install time symlinks, for instance, if the prototype contains
- # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
- # the shared library ends up in /opt/csw/lib/foo.so.0 and should be findable even when
- # RPATH does not contain $ISALIST.
- new_soname_runpath_data = {}
- for p in soname_runpath_data:
- expanded_p_list = Emulate64BitSymlinks([p])
- for expanded_p in expanded_p_list:
- new_soname_runpath_data[expanded_p] = soname_runpath_data[p]
- soname_runpath_data = new_soname_runpath_data
-
- logging.debug("%s: will be looking for %s in %s" %
- (soname, runpath_list, soname_runpath_data.keys()))
- for runpath_expanded in runpath_list:
- if runpath_expanded in soname_runpath_data:
- lines_by_soname[soname] = soname_runpath_data[runpath_expanded]
- runpath_found = True
- # This break only goes out of the inner loop,
- # need another one below to finish the outer loop.
- break
- if runpath_found:
- break
- return lines_by_soname
-
-
-def BuildIndexesBySoname(needed_sonames_by_binary):
- """Builds data structures indexed by soname.
-
- Building indexes
- {"foo.so": ["/opt/csw/lib/gcc4", "/opt/csw/lib", ...],
- ...
- }
- """
- needed_sonames = set()
- binaries_by_soname = {}
- runpath_by_needed_soname = {}
- for binary_name, data in needed_sonames_by_binary.iteritems():
- for soname in data[NEEDED_SONAMES]:
- needed_sonames.add(soname)
- if soname not in runpath_by_needed_soname:
- runpath_by_needed_soname[soname] = []
- runpath_by_needed_soname[soname].extend(data[RUNPATH])
- if soname not in binaries_by_soname:
- binaries_by_soname[soname] = set()
- binaries_by_soname[soname].add(binary_name)
- return needed_sonames, binaries_by_soname, runpath_by_needed_soname
-
-
-def ParseDumpOutput(dump_output):
- binary_data = {RUNPATH: [],
- NEEDED_SONAMES: []}
- for line in dump_output.splitlines():
- fields = re.split(WS_RE, line)
- # TODO: Make it a unit test
- # logging.debug("%s says: %s", DUMP_BIN, fields)
- if len(fields) < 3:
- continue
- if fields[1] == "NEEDED":
- binary_data[NEEDED_SONAMES].append(fields[2])
- elif fields[1] == "RUNPATH":
- binary_data[RUNPATH].extend(fields[2].split(":"))
- elif fields[1] == "SONAME":
- binary_data[SONAME] = fields[2]
- # Adding the default runtime path search option.
- binary_data[RUNPATH].append("/usr/lib/$ISALIST")
- binary_data[RUNPATH].append("/usr/lib")
- binary_data[RUNPATH].append("/lib/$ISALIST")
- binary_data[RUNPATH].append("/lib")
- return binary_data
Copied: csw/mgar/gar/v2/lib/python/checkpkg.py (from rev 8180, csw/mgar/gar/v2-dirpackage/lib/python/checkpkg.py)
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -0,0 +1,569 @@
+# $Id$
+#
+# This is the checkpkg library, common for all checkpkg tests written in
+# Python.
+
+import itertools
+import logging
+import optparse
+import os
+import os.path
+import re
+import socket
+import sqlite3
+import subprocess
+import StringIO
+from Cheetah import Template
+
+SYSTEM_PKGMAP = "/var/sadm/install/contents"
+WS_RE = re.compile(r"\s+")
+NEEDED_SONAMES = "needed sonames"
+RUNPATH = "runpath"
+SONAME = "soname"
+CONFIG_MTIME = "mtime"
+DO_NOT_REPORT_SURPLUS = set([u"CSWcommon", u"CSWcswclassutils", u"CSWisaexec"])
+DO_NOT_REPORT_MISSING = set([u"SUNWlibC", u"SUNWcsl", u"SUNWlibms",
+ u"*SUNWcslr", u"*SUNWlibC", u"*SUNWlibms",
+ u"SUNWcslx"])
+SYSTEM_SYMLINKS = (
+ ("/opt/csw/bdb4", ["/opt/csw/bdb42"]),
+ ("/64", ["/amd64", "/sparcv9"]),
+ ("/opt/csw/lib/i386", ["/opt/csw/lib"]),
+)
+
+# This shared library is present on Solaris 10 on amd64, but it's missing on
+# Solaris 8 on i386. It's okay if it's missing.
+ALLOWED_ORPHAN_SONAMES = set([u"libm.so.2"])
+DEPENDENCY_FILENAME_REGEXES = (
+ (r".*\.pl", u"CSWperl"),
+ (r".*\.pm", u"CSWperl"),
+ (r".*\.py", u"CSWpython"),
+ (r".*\.rb", u"CSWruby"),
+)
+
+REPORT_TMPL = u"""$pkgname:
+#if $missing_deps
+SUGGESTION: you may want to add some or all of the following as depends:
+ (Feel free to ignore SUNW or SPRO packages)
+#for $pkg in $sorted($missing_deps)
+> $pkg
+#end for
+#end if
+#if $surplus_deps
+The following packages might be unnecessary dependencies:
+#for $pkg in $sorted($surplus_deps)
+? $pkg
+#end for
+#end if
+#if $orphan_sonames
+The following sonames don't belong to any package:
+#for $soname in $sorted($orphan_sonames)
+! $soname
+#end for
+#end if
+#if not $missing_deps and not $surplus_deps and not $orphan_sonames
++ Dependencies of $pkgname look good.
+#end if
+"""
+
+class Error(Exception):
+ pass
+
+
+class ConfigurationError(Error):
+ pass
+
+
+class PackageError(Error):
+ pass
+
+
+def GetOptions():
+ parser = optparse.OptionParser()
+ parser.add_option("-e", dest="extractdir",
+ help="The directory into which the package has been extracted")
+ parser.add_option("-d", "--debug", dest="debug",
+ default=False, action="store_true",
+ help="Turn on debugging messages")
+ (options, args) = parser.parse_args()
+ if not options.extractdir:
+ raise ConfigurationError("ERROR: -e option is missing.")
+ # Using set() to make the arguments unique.
+ return options, set(args)
+
+
+class CheckpkgBase(object):
+ """This class has functionality overlapping with DirectoryFormatPackage
+ from the opencsw.py library. The classes should be merged.
+ """
+
+ def __init__(self, extractdir, pkgname):
+ self.extractdir = extractdir
+ self.pkgname = pkgname
+ self.pkgpath = os.path.join(self.extractdir, self.pkgname)
+
+ def CheckPkgpathExists(self):
+ if not os.path.isdir(self.pkgpath):
+ raise PackageError("%s does not exist or is not a directory"
+ % self.pkgpath)
+
+ def ListBinaries(self):
+ """Shells out to list all the binaries from a given package.
+
+ Original checkpkg code:
+
+ # #########################################
+ # # find all executables and dynamic libs,and list their filenames.
+ # listbinaries() {
+ # if [ ! -d $1 ] ; then
+ # print errmsg $1 not a directory
+ # rm -rf $EXTRACTDIR
+ # exit 1
+ # fi
+ #
+ # find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
+ # }
+ """
+ self.CheckPkgpathExists()
+ find_tmpl = "find %s -print | xargs file | grep ELF | nawk -F: '{print $1}'"
+ find_proc = subprocess.Popen(find_tmpl % self.pkgpath,
+ shell=True, stdout=subprocess.PIPE)
+ stdout, stderr = find_proc.communicate()
+ ret = find_proc.wait()
+ if ret:
+ logging.error("The find command returned an error.")
+ return stdout.splitlines()
+
+ def GetAllFilenames(self):
+ self.CheckPkgpathExists()
+ file_basenames = []
+ for root, dirs, files in os.walk(self.pkgpath):
+ file_basenames.extend(files)
+ return file_basenames
+
+ def GetDependencies(self):
+ fd = open(os.path.join(self.pkgpath, "install", "depend"), "r")
+ depends = {}
+ for line in fd:
+ fields = re.split(WS_RE, line)
+ if fields[0] == "P":
+ depends[fields[1]] = " ".join(fields[1:])
+ fd.close()
+ return depends
+
+ def FormatDepsReport(self, missing_deps, surplus_deps, orphan_sonames):
+ """A intermediate version in which StringIO is used."""
+ namespace = {
+ "pkgname": self.pkgname,
+ "missing_deps": missing_deps,
+ "surplus_deps": surplus_deps,
+ "orphan_sonames": orphan_sonames,
+ }
+ t = Template.Template(REPORT_TMPL, searchList=[namespace])
+ return unicode(t)
+
+
+class SystemPkgmap(object):
+ """A class to hold and manipulate the /var/sadm/install/contents file.
+
+ TODO: Implement timestamp checking and refreshing the cache.
+ """
+
+ STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
+ CHECKPKG_DIR = ".checkpkg"
+ SQLITE3_DBNAME_TMPL = "var-sadm-install-contents-cache-%s"
+
+ def __init__(self):
+ """There is no need to re-parse it each time.
+
+ Read it slowly the first time and cache it for later."""
+ self.cache = {}
+ self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
+ self.fqdn = socket.getfqdn()
+ self.db_path = os.path.join(self.checkpkg_dir,
+ self.SQLITE3_DBNAME_TMPL % self.fqdn)
+ self.file_mtime = None
+ self.cache_mtime = None
+ if os.path.exists(self.db_path):
+ logging.debug("Connecting to the %s database.", self.db_path)
+ self.conn = sqlite3.connect(self.db_path)
+ if not self.IsDatabaseUpToDate():
+ logging.warning("Rebuilding the package cache, can take a few minutes.")
+ self.PurgeDatabase()
+ self.PopulateDatabase()
+ else:
+ print "Building a cache of /var/sadm/install/contents."
+ print "The cache will be kept in %s." % self.db_path
+ if not os.path.exists(self.checkpkg_dir):
+ logging.debug("Creating %s", self.checkpkg_dir)
+ os.mkdir(self.checkpkg_dir)
+ self.conn = sqlite3.connect(self.db_path)
+ c = self.conn.cursor()
+ c.execute("""
+ CREATE TABLE systempkgmap (
+ id INTEGER PRIMARY KEY,
+ basename TEXT,
+ path TEXT,
+ line TEXT
+ );
+ """)
+ logging.debug("Creating the config table.")
+ c.execute("""
+ CREATE TABLE config (
+ key VARCHAR(255) PRIMARY KEY,
+ float_value FLOAT,
+ str_value VARCHAR(255)
+ );
+ """)
+ self.PopulateDatabase()
+
+ def SymlinkDuringInstallation(self, p):
+ """Emulates the effect of some symlinks present during installations."""
+ p = p.replace("/opt/csw/lib/i386", "/opt/csw/lib")
+
+ def PopulateDatabase(self):
+ """Imports data into the database.
+
+ Original bit of code from checkpkg:
+
+ egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
+ fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
+ """
+
+ system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
+ stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
+ # Creating a data structure:
+ # soname - {<path1>: <line1>, <path2>: <line2>, ...}
+ logging.debug("Building sqlite3 cache db of the %s file",
+ SYSTEM_PKGMAP)
+ c = self.conn.cursor()
+ count = itertools.count()
+ for line in system_pkgmap_fd:
+ i = count.next()
+ if not i % 1000:
+ print "\r%s" % i,
+ if stop_re.search(line):
+ continue
+ fields = re.split(WS_RE, line)
+ pkgmap_entry_path = fields[0].split("=")[0]
+ pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
+ sql = "INSERT INTO systempkgmap (basename, path, line) VALUES (?, ?, ?);"
+ c.execute(sql, (pkgmap_entry_base_name, pkgmap_entry_dir, line.strip()))
+ print
+ print "Creating the main database index."
+ sql = "CREATE INDEX basename_idx ON systempkgmap(basename);"
+ c.execute(sql)
+ self.SetDatabaseMtime()
+ self.conn.commit()
+
+ def SetDatabaseMtime(self):
+ c = self.conn.cursor()
+ sql = "DELETE FROM config WHERE key = ?;"
+ c.execute(sql, [CONFIG_MTIME])
+ mtime = self.GetFileMtime()
+ logging.debug("Inserting the mtime (%s) into the database.", mtime)
+ sql = """
+ INSERT INTO config (key, float_value)
+ VALUES (?, ?);
+ """
+ c.execute(sql, [CONFIG_MTIME, mtime])
+
+ def GetPkgmapLineByBasename(self, filename):
+ if filename in self.cache:
+ return self.cache[filename]
+ sql = "SELECT path, line FROM systempkgmap WHERE basename = ?;"
+ c = self.conn.cursor()
+ c.execute(sql, [filename])
+ lines = {}
+ for row in c:
+ lines[row[0]] = row[1]
+ if len(lines) == 0:
+ logging.debug("Cache doesn't contain filename %s", filename)
+ self.cache[filename] = lines
+ return lines
+
+ def GetDatabaseMtime(self):
+ if not self.cache_mtime:
+ sql = """
+ SELECT float_value FROM config
+ WHERE key = ?;
+ """
+ c = self.conn.cursor()
+ c.execute(sql, [CONFIG_MTIME])
+ row = c.fetchone()
+ if not row:
+ # raise ConfigurationError("Could not find the mtime setting")
+ self.cache_mtime = 1
+ else:
+ self.cache_mtime = row[0]
+ return self.cache_mtime
+
+ def GetFileMtime(self):
+ if not self.file_mtime:
+ stat_data = os.stat(SYSTEM_PKGMAP)
+ self.file_mtime = stat_data.st_mtime
+ return self.file_mtime
+
+ def IsDatabaseUpToDate(self):
+ f_mtime = self.GetFileMtime()
+ d_mtime = self.GetDatabaseMtime()
+ logging.debug("f_mtime %s, d_time: %s", f_mtime, d_mtime)
+ return self.GetFileMtime() <= self.GetDatabaseMtime()
+
+ def PurgeDatabase(self):
+ logging.info("Purging the cache database")
+ c = self.conn.cursor()
+ sql = "DELETE FROM config;"
+ c.execute(sql)
+ sql = "DELETE FROM systempkgmap;"
+ c.execute(sql)
+ sql = "DROP INDEX basename_idx;"
+ try:
+ c.execute(sql)
+ except sqlite3.OperationalError, e:
+ logging.warn(e)
+
+def SharedObjectDependencies(pkgname,
+ binaries_by_pkgname,
+ needed_sonames_by_binary,
+ pkgs_by_soname,
+ filenames_by_soname,
+ pkg_by_any_filename):
+ """This is one of the more obscure and more important pieces of code.
+
+ I tried to make it simpler, but given that the operations here involve
+ whole sets of packages, it's not easy.
+ """
+ so_dependencies = set()
+ orphan_sonames = set()
+ self_provided = set()
+ for binary in binaries_by_pkgname[pkgname]:
+ needed_sonames = needed_sonames_by_binary[binary][NEEDED_SONAMES]
+ for soname in needed_sonames:
+ if soname in filenames_by_soname:
+ filename = filenames_by_soname[soname]
+ pkg = pkg_by_any_filename[filename]
+ self_provided.add(soname)
+ so_dependencies.add(pkg)
+ elif soname in pkgs_by_soname:
+ so_dependencies.add(pkgs_by_soname[soname])
+ else:
+ orphan_sonames.add(soname)
+ return so_dependencies, self_provided, orphan_sonames
+
+
+def GuessDepsByFilename(pkgname, pkg_by_any_filename):
+ """Guesses dependencies based on filename regexes."""
+ guessed_deps = set()
+ for pattern, dep_pkgname in DEPENDENCY_FILENAME_REGEXES:
+ # If any file name matches, add the dep, go to the next pattern/pkg
+ # combination.
+ pattern_re = re.compile("^%s$" % pattern)
+ for filename in pkg_by_any_filename:
+ if (re.match(pattern_re, filename)
+ and
+ pkgname == pkg_by_any_filename[filename]):
+ guessed_deps.add(dep_pkgname)
+ break
+ return guessed_deps
+
+
+def GuessDepsByPkgname(pkgname, pkg_by_any_filename):
+ # More guessed dependencies: If one package is a substring of another, it
+ # might be a hint. For example, CSWmysql51test should depend on CSWmysql51.
+ # However, the rt (runtime) packages should not want to depend on the main
+ # package.
+ guessed_deps = set()
+ all_other_pkgs = set(pkg_by_any_filename.values())
+ for other_pkg in all_other_pkgs:
+ other_pkg = unicode(other_pkg)
+ if pkgname == other_pkg:
+ continue
+ if pkgname.startswith(other_pkg):
+ endings = ["devel", "test", "bench", "dev"]
+ for ending in endings:
+ if pkgname.endswith(ending):
+ guessed_deps.add(other_pkg)
+ return guessed_deps
+
+
+def AnalyzeDependencies(pkgname,
+ declared_dependencies,
+ binaries_by_pkgname,
+ needed_sonames_by_binary,
+ pkgs_by_soname,
+ filenames_by_soname,
+ pkg_by_any_filename):
+ """Gathers and merges dependency results from other functions.
+
+ declared_dependencies: Dependencies that the package in question claims to
+ have.
+
+ binaries_by_pkgname: A dictionary mapping pkgnames (CSWfoo) to binary names
+ (without paths)
+
+ needed_sonames_by_binary: A dictionary mapping binary file name to
+ a dictionary containing: "needed sonames",
+ "soname", "rpath". Based on examining the binary
+ files within the packages.
+
+ pkgs_by_soname: A dictionary mapping sonames to pkgnames, based on the
+ contents of the system wide pkgmap
+ (/var/sadm/install/contents)
+
+ filenames_by_soname: A dictionary mapping shared library sonames to filenames,
+ based on files within packages
+
+ pkg_by_any_filename: Mapping from file names to packages names, based on the
+ contents of the packages under examination.
+ """
+ declared_dependencies_set = set(declared_dependencies)
+
+ so_dependencies, self_provided, orphan_sonames = SharedObjectDependencies(
+ pkgname,
+ binaries_by_pkgname,
+ needed_sonames_by_binary,
+ pkgs_by_soname,
+ filenames_by_soname,
+ pkg_by_any_filename)
+ auto_dependencies = reduce(lambda x, y: x.union(y),
+ [
+ so_dependencies,
+ GuessDepsByFilename(pkgname, pkg_by_any_filename),
+ GuessDepsByPkgname(pkgname, pkg_by_any_filename),
+ ])
+ missing_deps = auto_dependencies.difference(declared_dependencies_set)
+ # Don't report itself as a suggested dependency.
+ missing_deps = missing_deps.difference(set([pkgname]))
+ missing_deps = missing_deps.difference(set(DO_NOT_REPORT_MISSING))
+ surplus_deps = declared_dependencies_set.difference(auto_dependencies)
+ surplus_deps = surplus_deps.difference(DO_NOT_REPORT_SURPLUS)
+ orphan_sonames = orphan_sonames.difference(ALLOWED_ORPHAN_SONAMES)
+ return missing_deps, surplus_deps, orphan_sonames
+
+
+def ExpandRunpath(runpath, isalist):
+ # Emulating $ISALIST expansion
+ if '$ISALIST' in runpath:
+ expanded_list = [runpath.replace('$ISALIST', isa) for isa in isalist]
+ else:
+ expanded_list = [runpath]
+ return expanded_list
+
+def ExpandSymlink(symlink, target, input_path):
+ symlink_re = re.compile(r"%s(/|$)" % symlink)
+ if re.search(symlink_re, input_path):
+ result = input_path.replace(symlink, target)
+ else:
+ result = input_path
+ return result
+
+def Emulate64BitSymlinks(runpath_list):
+ """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
+
+ Since we don't know the architecture, we'll adding both amd64 and sparcv9.
+ It should be safe.
+ """
+ symlinked_list = []
+ for runpath in runpath_list:
+ for symlink, expansion_list in SYSTEM_SYMLINKS:
+ for target in expansion_list:
+ expanded = ExpandSymlink(symlink, target, runpath)
+ if expanded not in symlinked_list:
+ symlinked_list.append(expanded)
+ return symlinked_list
+
+
+def SanitizeRunpath(runpath):
+ ok = False
+ while True:
+ if runpath.endswith("/"):
+ runpath = runpath[:-1]
+ elif "//" in runpath:
+ runpath = runpath.replace("//", "/")
+ else:
+ break
+ return runpath
+
+
+def GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist):
+ """Works out which system pkgmap lines correspond to given sonames."""
+ lines_by_soname = {}
+ for soname in needed_sonames:
+ # This is the critical part of the algorithm: it iterates over the
+ # runpath and finds the first matching one.
+ runpath_found = False
+ for runpath in runpath_by_needed_soname[soname]:
+ runpath = SanitizeRunpath(runpath)
+ runpath_list = ExpandRunpath(runpath, isalist)
+ runpath_list = Emulate64BitSymlinks(runpath_list)
+ soname_runpath_data = pkgmap.GetPkgmapLineByBasename(soname)
+ # Emulating the install time symlinks, for instance, if the prototype contains
+ # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
+ # the shared library ends up in /opt/csw/lib/foo.so.0 and should be findable even when
+ # RPATH does not contain $ISALIST.
+ new_soname_runpath_data = {}
+ for p in soname_runpath_data:
+ expanded_p_list = Emulate64BitSymlinks([p])
+ for expanded_p in expanded_p_list:
+ new_soname_runpath_data[expanded_p] = soname_runpath_data[p]
+ soname_runpath_data = new_soname_runpath_data
+
+ logging.debug("%s: will be looking for %s in %s" %
+ (soname, runpath_list, soname_runpath_data.keys()))
+ for runpath_expanded in runpath_list:
+ if runpath_expanded in soname_runpath_data:
+ lines_by_soname[soname] = soname_runpath_data[runpath_expanded]
+ runpath_found = True
+ # This break only goes out of the inner loop,
+ # need another one below to finish the outer loop.
+ break
+ if runpath_found:
+ break
+ return lines_by_soname
+
+
+def BuildIndexesBySoname(needed_sonames_by_binary):
+ """Builds data structures indexed by soname.
+
+ Building indexes
+ {"foo.so": ["/opt/csw/lib/gcc4", "/opt/csw/lib", ...],
+ ...
+ }
+ """
+ needed_sonames = set()
+ binaries_by_soname = {}
+ runpath_by_needed_soname = {}
+ for binary_name, data in needed_sonames_by_binary.iteritems():
+ for soname in data[NEEDED_SONAMES]:
+ needed_sonames.add(soname)
+ if soname not in runpath_by_needed_soname:
+ runpath_by_needed_soname[soname] = []
+ runpath_by_needed_soname[soname].extend(data[RUNPATH])
+ if soname not in binaries_by_soname:
+ binaries_by_soname[soname] = set()
+ binaries_by_soname[soname].add(binary_name)
+ return needed_sonames, binaries_by_soname, runpath_by_needed_soname
+
+
+def ParseDumpOutput(dump_output):
+ binary_data = {RUNPATH: [],
+ NEEDED_SONAMES: []}
+ for line in dump_output.splitlines():
+ fields = re.split(WS_RE, line)
+ # TODO: Make it a unit test
+ # logging.debug("%s says: %s", DUMP_BIN, fields)
+ if len(fields) < 3:
+ continue
+ if fields[1] == "NEEDED":
+ binary_data[NEEDED_SONAMES].append(fields[2])
+ elif fields[1] == "RUNPATH":
+ binary_data[RUNPATH].extend(fields[2].split(":"))
+ elif fields[1] == "SONAME":
+ binary_data[SONAME] = fields[2]
+ # Adding the default runtime path search option.
+ binary_data[RUNPATH].append("/usr/lib/$ISALIST")
+ binary_data[RUNPATH].append("/usr/lib")
+ binary_data[RUNPATH].append("/lib/$ISALIST")
+ binary_data[RUNPATH].append("/lib")
+ return binary_data
Deleted: csw/mgar/gar/v2/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2-dirpackage/lib/python/checkpkg_test.py 2010-01-26 10:26:12 UTC (rev 8180)
+++ csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-01-26 10:43:08 UTC (rev 8181)
@@ -1,589 +0,0 @@
-#!/opt/csw/bin/python2.6
-# $Id$
-
-import unittest
-import mox
-import difflib
-import checkpkg
-import testdata.checkpkg_test_data_CSWmysql51rt as d1
-import testdata.checkpkg_test_data_CSWmysql51client as d2
-import testdata.checkpkg_test_data_CSWmysql51 as d3
-import testdata.checkpkg_test_data_CSWmysql51devel as d4
-import testdata.checkpkg_test_data_CSWlibpq_84 as d5
-import testdata.checkpkg_test_data_CSWmysql5client_8x as d6
-import testdata.checkpkg_test_data_CSWpostfix as d7
-import testdata.dump_output_1 as dump_1
-import testdata.dump_output_2 as dump_2
-
-"""A set of unit tests for the library checking code.
-
-A bunch of lines to test in the interactive Python shell.
-
-import sys
-sys.path.append("gar/bin/checkpkg.d")
-import checkpkg
-import testdata.checkpkg_test_data_CSWmysql5client_8x as d6
-
-checkpkg.SharedObjectDependencies("CSWmysql5client",
-d6.DATA_BINARIES_BY_PKGNAME, d6.DATA_NEEDED_SONAMES_BY_BINARY,
-d6.DATA_PKGS_BY_FILENAME, d6.DATA_FILENAMES_BY_SONAME,
-d6.DATA_PKG_BY_ANY_FILENAME)
-
-sqlite3 ~/.checkpkg/var-sadm-install-contents-cache-build8x
-SELECT * FROM systempkgmap WHERE basename = 'libncursesw.so.5';
-"""
-
-class DependenciesUnitTest_1(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d1.DATA_PKGNAME,
- d1.DATA_DECLARED_DEPENDENCIES,
- d1.DATA_BINARIES_BY_PKGNAME,
- d1.DATA_NEEDED_SONAMES_BY_BINARY,
- d1.DATA_PKGS_BY_FILENAME,
- d1.DATA_FILENAMES_BY_SONAME,
- d1.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_2(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d2.DATA_PKGNAME,
- d2.DATA_DECLARED_DEPENDENCIES,
- d2.DATA_BINARIES_BY_PKGNAME,
- d2.DATA_NEEDED_SONAMES_BY_BINARY,
- d2.DATA_PKGS_BY_FILENAME,
- d2.DATA_FILENAMES_BY_SONAME,
- d2.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_3(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d3.DATA_PKGNAME,
- d3.DATA_DECLARED_DEPENDENCIES,
- d3.DATA_BINARIES_BY_PKGNAME,
- d3.DATA_NEEDED_SONAMES_BY_BINARY,
- d3.DATA_PKGS_BY_FILENAME,
- d3.DATA_FILENAMES_BY_SONAME,
- d3.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([u'CSWmysql51client']), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set(['CSWmysql51rt'])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_4(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d4.DATA_PKGNAME,
- d4.DATA_DECLARED_DEPENDENCIES,
- d4.DATA_BINARIES_BY_PKGNAME,
- d4.DATA_NEEDED_SONAMES_BY_BINARY,
- d4.DATA_PKGS_BY_FILENAME,
- d4.DATA_FILENAMES_BY_SONAME,
- d4.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_5(unittest.TestCase):
-
- def setUp(self):
- self.missing_deps, self.surplus_deps, self.orphan_sonames = checkpkg.AnalyzeDependencies(
- d5.DATA_PKGNAME,
- d5.DATA_DECLARED_DEPENDENCIES,
- d5.DATA_BINARIES_BY_PKGNAME,
- d5.DATA_NEEDED_SONAMES_BY_BINARY,
- d5.DATA_PKGS_BY_FILENAME,
- d5.DATA_FILENAMES_BY_SONAME,
- d5.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- # This tends to report itself...
- expected = set([u'SUNWgss'])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_6(unittest.TestCase):
-
- def setUp(self):
- (self.missing_deps,
- self.surplus_deps,
- self.orphan_sonames) = checkpkg.AnalyzeDependencies(
- d6.DATA_PKGNAME,
- d6.DATA_DECLARED_DEPENDENCIES,
- d6.DATA_BINARIES_BY_PKGNAME,
- d6.DATA_NEEDED_SONAMES_BY_BINARY,
- d6.DATA_PKGS_BY_FILENAME,
- d6.DATA_FILENAMES_BY_SONAME,
- d6.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class DependenciesUnitTest_7(unittest.TestCase):
-
- def setUp(self):
- (self.missing_deps,
- self.surplus_deps,
- self.orphan_sonames) = checkpkg.AnalyzeDependencies(
- d7.DATA_PKGNAME,
- d7.DATA_DECLARED_DEPENDENCIES,
- d7.DATA_BINARIES_BY_PKGNAME,
- d7.DATA_NEEDED_SONAMES_BY_BINARY,
- d7.DATA_PKGS_BY_FILENAME,
- d7.DATA_FILENAMES_BY_SONAME,
- d7.DATA_PKG_BY_ANY_FILENAME,
- )
-
- def testSurplusDeps(self):
- self.assertEquals(set([]), self.surplus_deps)
-
- def testOrphanSonames(self):
- self.assertEquals(set([]), self.orphan_sonames)
-
- def testMissingDeps(self):
- expected = set([])
- self.assertEquals(expected, self.missing_deps)
-
-
-class GuessDepsUnitTest(unittest.TestCase):
-
- def testGuessDepsByFilename1(self):
- expected = set([u"CSWpython"])
- pkgname = u"CSWfoo"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByFilename(pkgname, pkg_by_filename))
-
- def testGuessDepsByFilename2(self):
- expected = set([])
- pkgname = u"CSWfoo"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWbar",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByFilename(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname1(self):
- expected = set([u"CSWfoo"])
- pkgname = u"CSWfoo-devel"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/bin/barfoo": u"CSWfoobar",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByPkgname(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname2(self):
- expected = set([])
- pkgname = u"CSWzfoo-devel"
- pkg_by_filename = {
- "/opt/csw/bin/bar": u"CSWfoo",
- "/opt/csw/bin/barfoo": u"CSWfoobar",
- "/opt/csw/lib/python/site-packages/foo.py": u"CSWfoo",
- }
- self.assertEqual(expected,
- checkpkg.GuessDepsByPkgname(pkgname, pkg_by_filename))
-
- def testGuessDepsByPkgname3(self):
- self.assertEqual(set([u"CSWmysql51"]),
- checkpkg.GuessDepsByPkgname(u"CSWmysql51devel",
- d4.DATA_PKG_BY_ANY_FILENAME))
-
- def testGuessDepsByPkgname4(self):
- data1 = set(['CSWmysql51', 'CSWmysql51rt', 'CSWmysql51test',
- 'CSWmysql51client', 'CSWmysql51bench', 'CSWmysql51devel'])
- data2 = dict(((x, x) for x in data1))
- self.assertEqual(set([u"CSWmysql51"]), checkpkg.GuessDepsByPkgname(u"CSWmysql51devel", data2))
-
- def testGuessDepsByPkgname4(self):
- data1 = set(['CSWmysql51', 'CSWmysql51rt', 'CSWmysql51test',
- 'CSWmysql51client', 'CSWmysql51bench', 'CSWmysql51devel'])
- data2 = dict(((x, x) for x in data1))
- self.assertEqual(set([]), checkpkg.GuessDepsByPkgname(u"CSWmysql51rt", data2))
-
-
-class GetLinesBySonameUnitTest(unittest.TestCase):
-
- class PkgmapStub(object):
-
- def __init__(self, cache):
- self.cache = cache
-
- def GetPkgmapLineByBasename(self, soname):
- return self.cache[soname]
-
- def setUp(self):
- self.pkgmap_mocker = mox.Mox()
-
- def testExpandRunpath_1(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/lib/$ISALIST"
- expected = ["/opt/csw/lib/foo", "/opt/csw/lib/bar"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
-
- def testExpandRunpath_2(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/mysql5/lib/$ISALIST/mysql"
- expected = ["/opt/csw/mysql5/lib/foo/mysql", "/opt/csw/mysql5/lib/bar/mysql"]
- self.assertEquals(expected, checkpkg.ExpandRunpath(runpath, isalist))
-
- def testEmulate64BitSymlinks_1(self):
- runpath_list = ["/opt/csw/mysql5/lib/foo/mysql/64"]
- expected = "/opt/csw/mysql5/lib/foo/mysql/amd64"
- self.assertTrue(expected in checkpkg.Emulate64BitSymlinks(runpath_list))
-
- def testEmulate64BitSymlinks_2(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/amd64/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_3(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/sparcv9/mysql/foo"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_4(self):
- """No repeated paths because of symlink expansion"""
- runpath_list = ["/opt/csw/lib"]
- expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertEquals(1, len(result), "len(%s) != %s" % (result, 1))
-
- def testEmulateSymlinks_3(self):
- runpath_list = ["/opt/csw/bdb4"]
- expected = "/opt/csw/bdb42"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_4(self):
- runpath_list = ["/opt/csw/bdb42"]
- expected = "/opt/csw/bdb42"
- not_expected = "/opt/csw/bdb422"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result, "%s is in %s" % (not_expected, result))
-
- def testEmulateSymlinks_5(self):
- """Install time symlink expansion."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- result = checkpkg.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_6(self):
- """ExpandSymlink for /opt/csw/lib/i386."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- not_expected = "/opt/csw/lib/i386"
- result = checkpkg.ExpandSymlink("/opt/csw/lib/i386", "/opt/csw/lib", "/opt/csw/lib/i386")
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result, "%s is in %s" % (not_expected, result))
-
- def testGetLinesBySoname(self):
- expected = {'foo.so.1': '/opt/csw/lib/isa-value-1/foo.so.1 foo'}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo")
- lines1 = {"/opt/csw/lib/isa-value-1": "/opt/csw/lib/isa-value-1/foo.so.1 foo",
- "/usr/lib": "/usr/lib/foo.so.1 foo"}
- # pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- pkgmap.GetPkgmapLineByBasename("foo")
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {"foo.so.1": ["/opt/csw/lib/$ISALIST", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_3(self):
- expected = {'foo.so.1': '/opt/csw/lib/isa-value-1/foo.so.1 foo'}
- pkgmap = self.pkgmap_mocker.CreateMock(checkpkg.SystemPkgmap)
- pkgmap.GetPkgmapLineByBasename("foo")
- lines1 = {
- "/opt/csw/lib/isa-value-1": "/opt/csw/lib/isa-value-1/foo.so.1 foo",
- "/opt/csw/lib": "/opt/csw/lib/foo.so.1 foo",
- "/usr/lib": "/usr/lib/foo.so.1 foo"}
- # pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- pkgmap.GetPkgmapLineByBasename("foo.so.1").AndReturn(lines1)
- self.pkgmap_mocker.ReplayAll()
- pkgmap.GetPkgmapLineByBasename("foo")
- needed_sonames = set(["foo.so.1"])
- runpath_by_needed_soname = {
- "foo.so.1": ["/opt/csw/lib/$ISALIST", "/usr/lib"]}
- isalist = ["isa-value-1", "isa-value-2"]
- result = checkpkg.GetLinesBySoname(
- pkgmap, needed_sonames, runpath_by_needed_soname, isalist)
- self.pkgmap_mocker.VerifyAll()
- self.assertEqual(expected, result)
-
- def testGetLinesBySoname_4(self):
- """A more complex test, four ISAs."""
@@ Diff output truncated at 100000 characters. @@
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list