[csw-devel] SF.net SVN: gar:[7741] csw/mgar/gar/v2-checkpkg/bin
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Thu Dec 24 16:41:02 CET 2009
Revision: 7741
http://gar.svn.sourceforge.net/gar/?rev=7741&view=rev
Author: wahwah
Date: 2009-12-24 15:41:02 +0000 (Thu, 24 Dec 2009)
Log Message:
-----------
mGAR v2-checkpkg: Dependency reporting done, error code returning done.
Modified Paths:
--------------
csw/mgar/gar/v2-checkpkg/bin/checkpkg
csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg 2009-12-24 14:01:28 UTC (rev 7740)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg 2009-12-24 15:41:02 UTC (rev 7741)
@@ -83,6 +83,11 @@
pkgnames="$pkgnames $pkgname"
}
+if [[ "$1" == "-d" ]] ; then
+ DEBUG=1
+ shift
+fi
+
if [[ "$1" == "-e" ]] ; then
quit_on_warn=1;
shift
@@ -422,7 +427,7 @@
#cat $EXTRACTDIR/elflist| xargs ldd 2>/dev/null |fgrep '.so' |
# sed 's:^.*=>[^/]*::' | nawk '{print $1}' |sort -u >$EXTRACTDIR/liblist
- cat $EXTRACTDIR/elflist| xargs dump -Lv |nawk '$2=="NEEDED"{print $3}' |
+ cat $EXTRACTDIR/elflist| xargs /usr/ccs/bin/dump -Lv |nawk '$2=="NEEDED"{print $3}' |
sort -u | egrep -v $EXTRACTDIR >$EXTRACTDIR/liblist
@@ -564,13 +569,16 @@
checkpkg_scriptname=`basename $0`
checkpkg_basedir=${0%/${checkpkg_scriptname}}
plugindir=${checkpkg_basedir}/checkpkg.d
+if [[ "${DEBUG}" != "" ]]; then
+ extra_options="--debug"
+fi
echo "plugindir: '$plugindir'"
if [[ -d "$plugindir" ]]; then
# echo plugin dir exists
for plugin in "${plugindir}"/checkpkg-*; do
if [[ -x "${plugin}" ]]; then
- echo "Executing: ${plugin} -e \"${EXTRACTDIR}\" ${pkgnames}"
- ${plugin} -e "${EXTRACTDIR}" ${pkgnames}
+ echo "Executing: ${plugin} $extra_options -e \"${EXTRACTDIR}\" ${pkgnames}"
+ ${plugin} $extra_options -e "${EXTRACTDIR}" ${pkgnames}
if [[ "$?" -ne 0 ]]; then
errmsg "Plugin ${plugin} has returned an error."
fi
@@ -581,11 +589,10 @@
else
echo plugin dir does not exist
fi
+# echo "Profiling."
+# python -m cProfile gar/bin/checkpkg.d/checkpkg-libs.py --debug -e "${EXTRACTDIR}" ${pkgnames}
# End of plugin section
-# Not doing the cleanup now
-# cleanup
-
print ""
# Cleaning up after all packages
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py 2009-12-24 14:01:28 UTC (rev 7740)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg-libs.py 2009-12-24 15:41:02 UTC (rev 7741)
@@ -11,14 +11,20 @@
import re
import subprocess
import logging
+import sys
DUMP_BIN = "/usr/ccs/bin/dump"
NEEDED_SONAMES = "needed sonames"
RUNPATH = "runpath"
+TYPICAL_DEPENDENCIES = set(["CSWcommon", "CSWcswclassutils"])
def main():
- logging.basicConfig(level=logging.DEBUG)
options, args = checkpkg.GetOptions()
+ if options.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+ result_ok = True
pkgnames = args
checkers = []
for pkgname in pkgnames:
@@ -31,39 +37,18 @@
binaries_base = [os.path.split(x)[1] for x in pkg_binary_paths]
binaries_by_pkgname[checker.pkgname] = binaries_base
binaries.extend(pkg_binary_paths)
- # Make the binaries unique
+ # Making the binaries unique
binaries = set(binaries)
ws_re = re.compile(r"\s+")
- # if [[ "$goodarch" = "yes" ]] ; then
- # # man ld.so.1 for more info on this hack
- # export LD_NOAUXFLTR=1
- #
- # listbinaries $EXTRACTDIR/$pkgname >$EXTRACTDIR/elflist
- # # have to do this for ldd to work. arrg.
- # if [ -s "$EXTRACTDIR/elflist" ] ; then
- # chmod 0755 `cat $EXTRACTDIR/elflist`
- #
- # cat $EXTRACTDIR/elflist| xargs dump -Lv |nawk '$2=="NEEDED"{print $3}' |
- # sort -u | egrep -v $EXTRACTDIR >$EXTRACTDIR/liblist
- #
- #
- #
- # print libraries used are:
- # cat $EXTRACTDIR/liblist
- # print "cross-referencing with depend file (May take a while)"
- # else
- # print No dynamic libraries in the package
- # fi
- # fi
-
+ # man ld.so.1 for more info on this hack
env = copy.copy(os.environ)
env["LD_NOAUXFLTR"] = "1"
needed_sonames_by_binary = {}
# Assembling a data structure with the data about binaries.
# {
- # <binary1 name>: {NEEDED_SONAMES: [...],
- # RUNPATH: [...]},
+ # <binary1 name>: { NEEDED_SONAMES: [...],
+ # RUNPATH: [...]},
# <binary2 name>: ...,
# ...
# }
@@ -74,7 +59,7 @@
needed_sonames_by_binary[binary_base_name] = {}
binary_data = needed_sonames_by_binary[binary_base_name]
args = [DUMP_BIN, "-Lv", binary]
- dump_proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+ dump_proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env)
stdout, stderr = dump_proc.communicate()
ret = dump_proc.wait()
for line in stdout.splitlines():
@@ -114,23 +99,19 @@
binaries_by_soname[soname].append(binary_name)
pkgmap = checkpkg.SystemPkgmap()
- paths_by_soname = pkgmap.paths_by_soname
-
- logging.debug("Determining soname-file relationships.")
+ logging.debug("Determining the soname-package relationships.")
# lines by soname is an equivalent of $EXTRACTDIR/shortcatalog
lines_by_soname = {}
for soname in needed_sonames:
- if soname in paths_by_soname:
- # logging.debug("%s found", repr(soname))
- # Finding the first matching path
+ try:
for runpath in runpath_by_needed_soname[soname]:
- if runpath in paths_by_soname[soname]:
- # logging.debug("%s found in %s", runpath, paths_by_soname[soname])
- # logging.debug("line found: %s", repr(paths_by_soname[soname][runpath]))
- lines_by_soname[soname] = paths_by_soname[soname][runpath]
+ if runpath in pkgmap.GetPkgmapLineByBasename(soname):
+ # logging.debug("%s found in %s", runpath, pkgmap.GetPkgmapLineByBasename(soname))
+ # logging.debug("line found: %s", repr(pkgmap.GetPkgmapLineByBasename(soname)[runpath]))
+ lines_by_soname[soname] = pkgmap.GetPkgmapLineByBasename(soname)[runpath]
break
- else:
- logging.debug("%s not found in the soname list!", soname)
+ except KeyError, e:
+ logging.debug("KeyError: %s", soname, e)
pkgs_by_soname = {}
for soname, line in lines_by_soname.iteritems():
# TODO: Find all the packages, not just the last field.
@@ -138,8 +119,10 @@
# For now, we'll assume that the last field is the package.
pkgname = fields[-1]
pkgs_by_soname[soname] = pkgname
+
+ # A shared object dependency/provisioning report, plus checking.
for soname in needed_sonames:
- if soname in binaries:
+ if soname in needed_sonames_by_binary:
print "%s is provided by the package itself" % soname
elif soname in lines_by_soname:
print ("%s is required by %s and provided by %s"
@@ -149,91 +132,49 @@
else:
print ("%s is required by %s, but we don't know what provides it."
% (soname, binaries_by_soname[soname]))
+ result_ok = False
+
dependent_pkgs = {}
- for pkgname in binaries_by_pkgname:
+ for checker in checkers:
+ orphan_sonames = set()
+ pkgname = checker.pkgname
# logging.debug("Reporting package %s", pkgname)
- dependencies = set()
+ declared_dependencies = checker.GetDependencies()
+ so_dependencies = set()
for binary in binaries_by_pkgname[pkgname]:
if binary in needed_sonames_by_binary:
for soname in needed_sonames_by_binary[binary][NEEDED_SONAMES]:
- dependencies.add(pkgs_by_soname[soname])
+ if soname in pkgs_by_soname:
+ so_dependencies.add(pkgs_by_soname[soname])
+ else:
+ orphan_sonames.add(soname)
else:
logging.warn("%s not found in needed_sonames_by_binary (%s)",
binary, needed_sonames_by_binary.keys())
- logging.info("%s depends on %s", pkgname, dependencies)
+ declared_dependencies_set = set(declared_dependencies)
+ print "You can consider including the following packages in the dependencies:"
+ for dep_pkgname in sorted(so_dependencies.difference(declared_dependencies_set)):
+ print " ", dep_pkgname,
+ if dep_pkgname.startswith("SUNW"):
+ print "(it's safe to ignore this one)",
+ print
+
+ surplus_dependencies = declared_dependencies_set.difference(so_dependencies)
+ surplus_dependencies = surplus_dependencies.difference(TYPICAL_DEPENDENCIES)
+ if surplus_dependencies:
+ print "The following packages might be unnecessary dependencies:"
+ for dep_pkgname in surplus_dependencies:
+ print " ", dep_pkgname
+ if orphan_sonames:
+ print "The following sonames don't belong to any package:"
+ for soname in sorted(orphan_sonames):
+ print " ", soname
- # binaries_by_pkgname --> needed_sonames_by_binary --> pkgs_by_soname
- # TODO: print per-package deps (requires the transition: pkgname -> soname ->
- # pkgname)
+ if result_ok:
+ sys.exit(0)
+ else:
+ sys.exit(1)
- # for lib in `cat $EXTRACTDIR/liblist` ; do
- # grep "[/=]$lib[ =]" $EXTRACTDIR/$pkgname/pkgmap
- # if [[ $? -eq 0 ]] ; then
- # echo $lib provided by package itself
- # continue
- # else
- # grep "[/=]$lib[ =]" $SETLIBS
- # if [[ $? -eq 0 ]]; then
- # echo "$lib provided by package set being evaluated."
- # continue
- # fi
- # fi
- #
- # libpkg=`grep /$lib $EXTRACTDIR/shortcatalog |
- # sed 's/^.* \([^ ]*\)$/\1/' |sort -u`
- #
- # if [[ -z "$libpkg" ]] ; then
- # echo "$lib $pkgname" >> $SETLIBS.missing
- # print Cannot find package providing $lib. Storing for delayed validation.
- # else
- # print $libpkg | fmt -1 >>$EXTRACTDIR/libpkgs
- # fi
- # done
- #
- # sort -u $EXTRACTDIR/libpkgs >$EXTRACTDIR/libpkgs.x
- # mv $EXTRACTDIR/libpkgs.x $EXTRACTDIR/libpkgs
- #
- # diff $EXTRACTDIR/deppkgs $EXTRACTDIR/libpkgs >/dev/null
- # if [[ $? -ne 0 ]] ; then
- # print SUGGESTION: you may want to add some or all of the following as depends:
- # print ' (Feel free to ignore SUNW or SPRO packages)'
- # diff $EXTRACTDIR/deppkgs $EXTRACTDIR/libpkgs | fgrep '>'
- # fi
- #
- #
- #
- # if [[ "$basedir" != "" ]] ; then
- # print
- # if [[ -f $EXTRACTDIR/elflist ]] ; then
- # print "Checking relocation ability..."
- # xargs strings < $EXTRACTDIR/elflist| grep /opt/csw
- # if [[ $? -eq 0 ]] ; then
- # errmsg package build as relocatable, but binaries have hardcoded /opt/csw paths in them
- # else
- # print trivial check passed
- # fi
- # else
- # echo No relocation check done for non-binary relocatable package.
- # fi
- # fi
- #
- # ...
- #
- # if [ -s $SETLIBS.missing ]; then
- # print "Doing late evaluations of package library dependencies."
- # while read ldep; do
- # lib=`echo $ldep | nawk '{print $1}'`
- # [ "$lib" = "libm.so.2" ] && continue
- # pkg=`echo $ldep | nawk '{print $2}'`
- # /usr/bin/grep "[/=]$lib[ =]" $SETLIBS >/dev/null
- # if [ $? -ne 0 ]; then
- # errmsg "Couldn't find a package providing $lib"
- # else
- # print "A package in the set being evaluated provides $lib"
- # fi
- # done < $SETLIBS.missing
- # fi
-
if __name__ == '__main__':
main()
Modified: csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py
===================================================================
--- csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py 2009-12-24 14:01:28 UTC (rev 7740)
+++ csw/mgar/gar/v2-checkpkg/bin/checkpkg.d/checkpkg.py 2009-12-24 15:41:02 UTC (rev 7741)
@@ -12,6 +12,7 @@
import re
SYSTEM_PKGMAP = "/var/sadm/install/contents"
+WS_RE = re.compile(r"\s+")
class Error(Exception):
pass
@@ -40,6 +41,9 @@
class CheckpkgBase(object):
+ """This class has functionality overlapping with DirectoryFormatPackage
+ from the opencsw.py library. The classes should be merged.
+ """
def __init__(self, extractdir, pkgname):
self.extractdir = extractdir
@@ -50,73 +54,92 @@
# #########################################
# # find all executables and dynamic libs,and list their filenames.
# listbinaries() {
- # if [ ! -d $1 ] ; then
- # print errmsg $1 not a directory
- # rm -rf $EXTRACTDIR
- # exit 1
- # fi
+ # if [ ! -d $1 ] ; then
+ # print errmsg $1 not a directory
+ # rm -rf $EXTRACTDIR
+ # exit 1
+ # fi
#
- # find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
+ # find $1 -print | xargs file |grep ELF |nawk -F: '{print $1}'
# }
find_tmpl = "find %s -print | xargs file | grep ELF | nawk -F: '{print $1}'"
if not os.path.isdir(self.pkgpath):
- raise PackageError("%s does not exist or is not a directory"
- % self.pkgpath)
+ raise PackageError("%s does not exist or is not a directory"
+ % self.pkgpath)
find_proc = subprocess.Popen(find_tmpl % self.pkgpath,
shell=True, stdout=subprocess.PIPE)
stdout, stderr = find_proc.communicate()
ret = find_proc.wait()
if ret:
- logging.error("The find command returned an error.")
+ logging.error("The find command returned an error.")
return stdout.splitlines()
+ def GetDependencies(self):
+ fd = open(os.path.join(self.pkgpath, "install", "depend"), "r")
+ depends = {}
+ for line in fd:
+ fields = re.split(WS_RE, line)
+ if fields[0] == "P":
+ depends[fields[1]] = " ".join(fields[1:])
+ fd.close()
+ return depends
+
+
+
class SystemPkgmap(object):
"""A class to hold and manipulate the /var/sadm/install/contents file."""
PICKLE_NAME = "var-sadm-install-contents.pickle"
- WS_RE = re.compile(r"\s+")
+ STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
+ CHECKPKG_DIR = ".checkpkg"
+
def __init__(self):
"""There is no need to re-parse it each time.
Read it slowly the first time and cache it for later."""
- self.checkpkg_dir = os.path.join(os.environ["HOME"], ".checkpkg")
+ self.checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
self.pickle_path = os.path.join(self.checkpkg_dir, self.PICKLE_NAME)
if os.path.exists(self.pickle_path):
- logging.debug("Unpickling %s, please wait.", self.pickle_path)
- pickle_fd = open(self.pickle_path, "r")
- self.paths_by_soname = cPickle.load(pickle_fd)
- pickle_fd.close()
+ logging.info("Unpickling %s, this can take up to 30s.", self.pickle_path)
+ pickle_fd = open(self.pickle_path, "r")
+ self.pkmap_lines_by_basename = cPickle.load(pickle_fd)
+ pickle_fd.close()
else:
# The original checkpkg code to port is in the comments.
#
# egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
# fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
- stop_pkgs = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
system_pkgmap_fd = open(SYSTEM_PKGMAP, "r")
- stop_re = re.compile("(%s)" % "|".join(stop_pkgs))
+ stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
# Creating a data structure:
# soname - {<path1>: <line1>, <path2>: <line2>, ...}
logging.debug("Building in-memory data structure for the %s file",
SYSTEM_PKGMAP)
- paths_by_soname = {}
+ pkmap_lines_by_basename = {}
for line in system_pkgmap_fd:
if stop_re.search(line):
continue
- fields = re.split(self.WS_RE, line)
+ fields = re.split(WS_RE, line)
pkgmap_entry_path = fields[0].split("=")[0]
pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
- if pkgmap_entry_base_name not in paths_by_soname:
- paths_by_soname[pkgmap_entry_base_name] = {}
- paths_by_soname[pkgmap_entry_base_name][pkgmap_entry_dir] = line
+ if pkgmap_entry_base_name not in pkmap_lines_by_basename:
+ pkmap_lines_by_basename[pkgmap_entry_base_name] = {}
+ pkmap_lines_by_basename[pkgmap_entry_base_name][pkgmap_entry_dir] = line
logging.debug("The data structure contains %s files",
- len(paths_by_soname))
- self.paths_by_soname = paths_by_soname
+ len(pkmap_lines_by_basename))
+ self.pkmap_lines_by_basename = pkmap_lines_by_basename
if not os.path.exists(self.checkpkg_dir):
- logging.debug("Creating %s", self.checkpkg_dir)
- os.mkdir(self.checkpkg_dir)
+ logging.debug("Creating %s", self.checkpkg_dir)
+ os.mkdir(self.checkpkg_dir)
logging.debug("Pickling to %s", self.pickle_path)
pickle_fd = open(self.pickle_path, "w")
- cPickle.dump(self.paths_by_soname, pickle_fd)
+ cPickle.dump(self.pkmap_lines_by_basename, pickle_fd)
pickle_fd.close()
+
+ def GetPkgmapLineByBasename(self, filename):
+ if filename in self.pkmap_lines_by_basename:
+ return self.pkmap_lines_by_basename[filename]
+ else:
+ raise KeyError, "%s not found in self.pkmap_lines_by_basename" % filename
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list