[csw-devel] SF.net SVN: gar:[11882] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sat Dec 11 19:01:58 CET 2010
Revision: 11882
http://gar.svn.sourceforge.net/gar/?rev=11882&view=rev
Author: wahwah
Date: 2010-12-11 18:01:58 +0000 (Sat, 11 Dec 2010)
Log Message:
-----------
Merge branch 'collisions' into svn-merge
Modified Paths:
--------------
csw/mgar/gar/v2/bin/checkpkg
csw/mgar/gar/v2/bin/mkpackage
csw/mgar/gar/v2/gar.pkg.mk
csw/mgar/gar/v2/lib/python/README
csw/mgar/gar/v2/lib/python/catalog.py
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/checkpkg_test.py
csw/mgar/gar/v2/lib/python/configuration.py
csw/mgar/gar/v2/lib/python/database.py
csw/mgar/gar/v2/lib/python/dependency_checks.py
csw/mgar/gar/v2/lib/python/dependency_checks_test.py
csw/mgar/gar/v2/lib/python/inspective_package.py
csw/mgar/gar/v2/lib/python/models.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/package.py
csw/mgar/gar/v2/lib/python/package_checks.py
csw/mgar/gar/v2/lib/python/package_checks_test.py
csw/mgar/gar/v2/lib/python/package_stats.py
csw/mgar/gar/v2/lib/python/package_stats_test.py
csw/mgar/gar/v2/lib/python/pkgdb.py
csw/mgar/gar/v2/lib/python/sharedlib_utils.py
csw/mgar/gar/v2/lib/python/sharedlib_utils_test.py
csw/mgar/gar/v2/lib/python/tag.py
csw/mgar/gar/v2/lib/python/tag_test.py
csw/mgar/gar/v2/lib/python/testdata/neon_stats.py
csw/mgar/gar/v2/lib/python/testdata/tree_stats.py
csw/mgar/gar/v2/tests/run_tests.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/checkpkg2.py
csw/mgar/gar/v2/lib/python/checkpkg_defaults.ini
csw/mgar/gar/v2/lib/python/checkpkg_lib.py
csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py
csw/mgar/gar/v2/lib/python/common_constants.py
csw/mgar/gar/v2/lib/python/ldd_emul.py
csw/mgar/gar/v2/lib/python/ldd_emul_test.py
csw/mgar/gar/v2/lib/python/models_test.py
csw/mgar/gar/v2/lib/python/mute_progressbar.py
csw/mgar/gar/v2/lib/python/pkgdb_test.py
csw/mgar/gar/v2/lib/python/shell.py
csw/mgar/gar/v2/lib/python/system_pkgmap.py
csw/mgar/gar/v2/lib/python/system_pkgmap_test.py
csw/mgar/gar/v2/lib/python/test_base.py
Removed Paths:
-------------
csw/mgar/gar/v2/bin/analyze_module_results.py
csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
csw/mgar/gar/v2/bin/checkpkg_run_modules.py
Property Changed:
----------------
csw/mgar/gar/v2/bin/checkpkg
csw/mgar/gar/v2/lib/python/package_stats_test.py
Deleted: csw/mgar/gar/v2/bin/analyze_module_results.py
===================================================================
--- csw/mgar/gar/v2/bin/analyze_module_results.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/bin/analyze_module_results.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -1,73 +0,0 @@
-#!/opt/csw/bin/python2.6
-# $Id$
-
-import itertools
-import operator
-import optparse
-import os
-import pprint
-import progressbar
-import sys
-import textwrap
-
-# The following bit of code sets the correct path to Python libraries
-# distributed with GAR.
-path_list = [os.path.dirname(__file__),
- "..", "lib", "python"]
-sys.path.append(os.path.join(*path_list))
-import checkpkg
-import overrides
-import package_stats
-
-BEFORE_OVERRIDES = """If any of the reported errors were false positives, you
-can override them pasting the lines below to the GAR recipe."""
-
-AFTER_OVERRIDES = """Please note that checkpkg isn't suggesting you should
-simply add these overrides do the Makefile. It only informs what the overrides
-could look like. You need to understand what are the reported issues about and
-use your best judgement to decide whether to fix the underlying problems or
-override them. For more information, scroll up and read the detailed
-messages."""
-
-UNAPPLIED_OVERRIDES = """WARNING: Some overrides did not match any errors.
-They can be removed, as they don't take any effect anyway. If you're getting
-errors at the same time, maybe you didn't specify the overrides correctly."""
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option("-c", "--catalog_file", dest="catalog",
- help="Optional catalog file")
- parser.add_option("-q", "--quiet", dest="quiet",
- default=False, action="store_true",
- help=("Display less messages"))
- options, args = parser.parse_args()
- filenames = args
-
- # This might be bottleneck. Perhaps a list of md5 sums can be given to this
- # script instead.
-
- # It might be a good idea to store the error tags in the database and
- # eliminate the need to access the directory with the error tag files.
-
- pkgstats = package_stats.StatsListFromCatalog(filenames, options.catalog)
- overrides_list = [pkg.GetSavedOverrides() for pkg in pkgstats]
- override_list = reduce(operator.add, overrides_list)
- error_tags = reduce(operator.add, [stat.GetSavedErrorTags() for stat in pkgstats])
- (tags_after_overrides,
- unapplied_overrides) = overrides.ApplyOverrides(error_tags, override_list)
- if not options.quiet:
- if tags_after_overrides:
- print textwrap.fill(BEFORE_OVERRIDES, 80)
- for checkpkg_tag in tags_after_overrides:
- print checkpkg_tag.ToGarSyntax()
- print textwrap.fill(AFTER_OVERRIDES, 80)
- if unapplied_overrides:
- print textwrap.fill(UNAPPLIED_OVERRIDES, 80)
- for override in unapplied_overrides:
- print "* Unused %s" % override
- exit_code = bool(tags_after_overrides)
- sys.exit(exit_code)
-
-
-if __name__ == '__main__':
- main()
Modified: csw/mgar/gar/v2/bin/checkpkg
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/bin/checkpkg 2010-12-11 18:01:58 UTC (rev 11882)
@@ -1,322 +1 @@
-#!/bin/ksh -p
-#
-# $Id$
-#
-# checkpkg 1.51
-#
-# diff to 1.46a
-# - check multiple package files
-# - checkpkg.d plugin support
-# - getopts support for command line options
-# - colors
-# - modular architecture + unit tests
-# - reliable shared library checking
-#
-# This script examines a package that has been put together
-# for submittal to the CSW archive at opencsw.org
-#
-# It examines it for compliance with the packaging standards at
-# http://www.opencsw.org/standards/
-# It DOES NOT CATCH EVERYTHING. However, the package will be
-# tested with this script before acceptance, so you may as well
-# save yourself some time, and run the script yourself!
-#
-# Be sure to occasionally do a "pkg-get update cswutils" so that
-# you know you are tracking the most current version.
-#
-# TODO:
-# - add message handlig to the CheckInterface class.
-#
-
-PATH=$PATH:/usr/sbin
-readonly NAME_MAX_LENGTH=${NAME_MAX_LENGTH:-20}
-
-command_basename=`basename $0`
-command_basedir="${0%/${command_basename}}"
-libshdir="${command_basedir}/../lib/sh"
-readonly command_basename command_basedir libshdir
-. "${libshdir}/libcheckpkg.sh"
-
-LOCAL_ARCH=`uname -p`
-CHECKPKG_TMPDIR=${CHECKPKG_TMPDIR:-/var/tmp}
-readonly CHECKPKG_TMPDIR
-
-# Colors only when running interactively
-if [[ -t 1 ]]; then
- GREEN="\\033[0;32;40m"
- RED="\\033[1;31;40m"
- BOLD="\\033[1m"
- COLOR_RESET="\\033[00m"
-else
- GREEN=""
- RED=""
- BOLD=""
- COLOR_RESET=""
-fi
-readonly GREEN RED BOLD COLOR_RESET
-
-readonly selfpath="$0"
-readonly selfargs="$@"
-
-cleanup(){
- if [[ -d "$EXTRACTDIR" ]] ; then
- rm -rf $EXTRACTDIR
- fi
- cleantmparchives
-}
-
-cleantmparchives() {
- for TMPARCHIVE in $tmparchives; do
- if [[ "$TMPARCHIVE" != "" ]]; then
- [ -f "$TMPARCHIVE" ] && rm $TMPARCHIVE
- fi
- done
-}
-
-cleanupset(){
- if [ "`echo $SETINF*`" != "$SETINF*" ]; then
- rm $SETINF*
- fi
-}
-
-# Print error message, and quit program.
-errmsg(){
- print ERROR: $* >/dev/fd/2
- cleanup
- cleanupset
- print "To run checkpkg in the debug mode, add the '-d' flag, for example:"
- # selfargs can be very, very long. Find a way to truncate it.
- # print "${selfpath} -d ${selfargs}"
- print "After you modify any overrides, you need to do gmake remerge repackage"
- print "or gmake platforms-remerge platforms-repackage."
- exit 1
-}
-
-debugmsg() {
- if [[ "${DEBUG}" != "" ]]; then
- print "DEBUG: $*" > /dev/fd/2
- fi
-}
-
-# TODO: Options to add:
-# - Use an pre-cached (from a catalog file?) list of md5 sums
-# - Don't use the data from /var/sadm/install/contents
-display_help=0
-SKIP_STATS_COLLECTION=0
-MD5_SUMS_CATALOG_FILE=""
-INSTALL_CONTENTS_FILES="/var/sadm/install/contents"
-ANALYZE=1
-PROFILE=0
-QUIET=0
-
-while getopts hsdNM:o:c:Apq opt; do
- case "${opt}" in
- c)
- INSTALL_CONTENTS_FILES="${INSTALL_CONTENTS_FILES} ${OPTARG}"
- ;;
- d)
- DEBUG=1
- ;;
- h)
- display_help=1
- ;;
- N)
- SKIP_STATS_COLLECTION=1
- ;;
- M)
- MD5_SUMS_CATALOG_FILE="${OPTARG}"
- ;;
- A)
- ANALYZE=0
- ;;
- p)
- PROFILE=1
- ;;
- q) QUIET=1
- ;;
- *)
- echo "Unknown option '${opt}'"
- ;;
- esac
-done
-shift $(( $OPTIND -1 ))
-
-readonly INSTALL_CONTENTS_FILES
-readonly MD5_SUMS_CATALOG_FILE
-readonly SKIP_STATS_COLLECTION
-readonly ANALYZE
-readonly PROFILE
-readonly QUIET
-
-if [[ "${display_help}" -eq 1 ]] ; then
- print 'Usage: checkpkg [options] pkg1 [pkg2 ....]'
- print 'Options:'
- print ' -c <file> use an additional install/contents file'
- print ' -d display debug messages'
- print ' -N skip statistics collection'
- print ' -M <file> use package md5sums from a catalog file'
- print ' -A Do not analyze the results.'
- print ' -p Enable profiling'
- print ' -q Display less messages'
- print ''
- print 'Error tags are saved to the sqlite database.'
- exit 0
-fi
-
-# a unique filename for the list of package deps and libs we see in a 'set'
-SETINF=$CHECKPKG_TMPDIR/checkpkg.$$.`date +%Y%m%d%H%M%S`
-SETLIBS=$SETINF.libs
-SETDEPS=$SETINF.deps
-pkgnames=""
-tmparchives=""
-
-EXTRACTDIR=$CHECKPKG_TMPDIR/dissect.$$
-
-if [ -d $EXTRACTDIR ] ; then
- errmsg ERROR: $EXTRACTDIR already exists
-fi
-
-for f in "$@"
-do
-
- if [[ ! -f $f ]] ; then
- errmsg ERROR: $f does not exist
- fi
-
-
-[ -d ${EXTRACTDIR} ] || mkdir ${EXTRACTDIR}
-
-########################################
-# Check for some common errors
-#########################################
-
-# TODO: To be ported.
-#
-# # find all executables and dynamic libs,and list their filenames.
-# if [[ "$basedir" != "" ]] ; then
-# print
-# if [[ -f $EXTRACTDIR/elflist ]] ; then
-# print "Checking relocation ability..."
-# xargs strings < $EXTRACTDIR/elflist| grep /opt/csw
-# if [[ $? -eq 0 ]] ; then
-# errmsg package build as relocatable, but binaries have hardcoded /opt/csw paths in them
-# else
-# print trivial check passed
-# fi
-# else
-# echo No relocation check done for non-binary relocatable package.
-# fi
-# fi
-
-tmparchives="$tmparchives $TMPARCHIVE"
-done
-
-# Plugin section. This is here for support for other programming languages
-# than Python. As of 2010-03-16 there are no checks in there. If this keeps
-# empty, if no checks in other languages get written, it could be removed.
-#
-# Plugins should live in checkpkg.d subdirectory in the same directory in which
-# checkpkg is. Each plugin file name should be an executable and begin with
-# "checkpkg-".
-
-test_suite_ok=1
-checkpkg_module_dir="${command_basedir}/../lib/checkpkg.d"
-checkpkg_module_tag="checkpkg-"
-checkpkg_stats_basedir="${HOME}/.checkpkg/stats"
-
-# Cleaning up old *.pyc files which can cause grief. This is because of the
-# move of Python libraries.
-for pyc_file in ${checkpkg_module_dir}/opencsw.pyc \
- ${checkpkg_module_dir}/checkpkg.pyc; do
- if [ -f "${pyc_file}" ]; then
- echo "Removing old pyc file: '${pyc_file}'"
- rm "${pyc_file}"
- fi
-done
-
-if [[ "${DEBUG}" != "" ]]; then
- extra_options="--debug"
-fi
-if [[ "${PROFILE}" -eq 1 ]]; then
- extra_options="${extra_options} --profile"
-fi
-if [[ "${QUIET}" -eq 1 ]]; then
- quiet_options="--quiet"
-else
- quiet_options=""
-fi
-
-if [[ -n "${MD5_SUMS_CATALOG_FILE}" ]]; then
- catalog_options="--catalog=${MD5_SUMS_CATALOG_FILE}"
-else
- catalog_options=""
-fi
-
-# /var/sadm/install/contents cache update
-# TODO: Either remove this section or stop the stats collection phase from
-# updating the cache.
-${command_basedir}/update_contents_cache.py ${extra_options}
-if [[ $? -ne 0 ]]; then
- errmsg "Updating the contents cache has failed."
-fi
-if [[ "${SKIP_STATS_COLLECTION}" -eq 0 ]]; then
- # Collects package stats to be analyzed later
- ${command_basedir}/checkpkg_collect_stats.py \
- ${catalog_options} \
- ${extra_options} \
- "$@"
- if [[ "$?" -ne 0 ]]; then
- errmsg "Stats collection phase has failed."
- fi
-fi
-
-# TODO: A performance problem. The following line means that the md5sums are
-# calculated once more.
-if [ "${MD5_SUMS_CATALOG_FILE}" ]; then
- debugmsg "Reading md5sums from ${MD5_SUMS_CATALOG_FILE}"
- md5sums=`cat "${MD5_SUMS_CATALOG_FILE}" \
- | awk '{print $5}' \
- | ggrep -E '[0-9abcdef]{32}'`
-else
- debugmsg "Calculating md5 sums of all the package files."
- md5sums=`gmd5sum "$@" | awk '{print $1}'`
-fi
-debugmsg "All md5 sums: ${md5sums}"
-
-# Running the checks.
-${command_basedir}/checkpkg_run_modules.py \
- ${extra_options} \
- -b "${checkpkg_stats_basedir}" \
- ${quiet_options} \
- ${md5sums}
-if [[ "$?" -ne 0 ]]; then
- print "There was a problem analyzing package stats."
- test_suite_ok=0
-fi
-
-if [[ ${test_suite_ok} -ne 1 ]]; then
- errmsg "One or more tests have finished with an error."
-fi
-
-if [[ "${ANALYZE}" -eq 1 ]]; then
-# Collecting errors and applying the overrides.
-# This has to use the original files.
- ${command_basedir}/analyze_module_results.py \
- ${catalog_options} \
- ${quiet_options} \
- "$@"
- if [[ "$?" -ne 0 ]]; then
- errmsg "${RED}Checkpkg has reported errors.${COLOR_RESET}"
- else
- print "${GREEN}Checkpkg reports no errors.${COLOR_RESET}"
- fi
-else
- echo "Skipping result analysis."
-fi
-
-print ""
-
-# Cleaning up after all packages
-cleanup
-
-cleanupset
+link ../lib/python/checkpkg2.py
\ No newline at end of file
Property changes on: csw/mgar/gar/v2/bin/checkpkg
___________________________________________________________________
Deleted: svn:executable
- *
Added: svn:special
+ *
Deleted: csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -1,69 +0,0 @@
-#!/opt/csw/bin/python2.6
-#
-# $Id$
-#
-# Collects statistics about a package and saves to a directory, for later use
-# by checkpkg modules.
-
-import itertools
-import logging
-import optparse
-import os
-import os.path
-import subprocess
-import sys
-import progressbar
-
-# The following bit of code sets the correct path to Python libraries
-# distributed with GAR.
-path_list = [os.path.dirname(__file__),
- "..", "lib", "python"]
-sys.path.append(os.path.join(*path_list))
-import checkpkg
-import opencsw
-import package_stats
-
-def main():
- parser = optparse.OptionParser()
- parser.add_option("-d", "--debug", dest="debug",
- default=False, action="store_true",
- help="Turn on debugging messages")
- parser.add_option("-c", "--catalog", dest="catalog",
- help="Catalog file")
- parser.add_option("-p", "--profile", dest="profile",
- default=False, action="store_true",
- help="A disabled option")
- options, args = parser.parse_args()
- if options.debug:
- logging.basicConfig(level=logging.DEBUG)
- else:
- logging.basicConfig(level=logging.INFO)
- logging.debug("Collecting statistics about given package files.")
- args_display = args
- if len(args_display) > 5:
- args_display = args_display[:5] + ["...more..."]
- file_list = args
- logging.debug("Processing: %s, please be patient", args_display)
- stats_list = package_stats.StatsListFromCatalog(
- file_list, options.catalog, options.debug)
- # Reversing the item order in the list, so that the pop() method can be used
- # to get packages, and the order of processing still matches the one in the
- # catalog file.
- stats_list.reverse()
- total_packages = len(stats_list)
- counter = itertools.count(1)
- logging.info("Juicing the srv4 package stream files...")
- bar = progressbar.ProgressBar()
- bar.maxval = total_packages
- bar.start()
- while stats_list:
- # This way objects will get garbage collected as soon as they are removed
- # from the list by pop(). The destructor (__del__()) of the srv4 class
- # removes the temporary directory from the disk. This allows to process
- # the whole catalog.
- stats_list.pop().CollectStats()
- bar.update(counter.next())
- bar.finish()
-
-if __name__ == '__main__':
- main()
Deleted: csw/mgar/gar/v2/bin/checkpkg_run_modules.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg_run_modules.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/bin/checkpkg_run_modules.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -1,56 +0,0 @@
-#!/opt/csw/bin/python2.6
-# $Id$
-
-"""This script runs all the checks written in Python."""
-
-import datetime
-import logging
-import os
-import os.path
-import sys
-import re
-import cProfile
-
-CHECKPKG_MODULE_NAME = "Second checkpkg API version"
-
-# The following bit of code sets the correct path to Python libraries
-# distributed with GAR.
-path_list = [os.path.dirname(__file__),
- "..", "lib", "python"]
-sys.path.append(os.path.join(*path_list))
-import checkpkg
-import opencsw
-
-
-def main():
- options, args = checkpkg.GetOptions()
- if options.debug:
- logging.basicConfig(level=logging.DEBUG)
- else:
- logging.basicConfig(level=logging.INFO)
- md5sums = args
- # CheckpkgManager2 class abstracts away things such as the collection of
- # results.
- check_manager = checkpkg.CheckpkgManager2(CHECKPKG_MODULE_NAME,
- options.stats_basedir,
- md5sums,
- options.debug)
- # Running the checks, reporting and exiting.
- exit_code, screen_report, tags_report = check_manager.Run()
- screen_report = unicode(screen_report)
- if not options.quiet and screen_report:
- sys.stdout.write(screen_report)
- else:
- logging.debug("No screen report.")
- sys.exit(exit_code)
-
-
-if __name__ == '__main__':
- if "--profile" in sys.argv:
- t_str = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
- home = os.environ["HOME"]
- cprof_file_name = os.path.join(
- home, ".checkpkg", "run-modules-%s.cprof" % t_str)
- cProfile.run("main()", sort=1, filename=cprof_file_name)
- else:
- main()
Modified: csw/mgar/gar/v2/bin/mkpackage
===================================================================
--- csw/mgar/gar/v2/bin/mkpackage 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/bin/mkpackage 2010-12-11 18:01:58 UTC (rev 11882)
@@ -25,6 +25,9 @@
# Tool Version/Revision Information
$TOOLVERSION = "1.4";
($REVISION) = q/$Revision$/ =~ /(\d+)/;
+# This shows a warning:
+# "Use of uninitialized value $REVISION in sprintf at
+# /home/maciej/src/opencsw/pkg/nspr/trunk/gar/bin/mkpackage line 31."
$VERSION = sprintf '%s (r%d)', $TOOLVERSION, $REVISION;
# Discover network support
Modified: csw/mgar/gar/v2/gar.pkg.mk
===================================================================
--- csw/mgar/gar/v2/gar.pkg.mk 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/gar.pkg.mk 2010-12-11 18:01:58 UTC (rev 11882)
@@ -52,6 +52,7 @@
GARPKG_v1 = CSWgar-v1
GARPKG_v2 = CSWgar-v2
RUNTIME_DEP_PKGS_$(SRCPACKAGE) ?= $(or $(GARPKG_$(GARSYSTEMVERSION)),$(error GAR version $(GARSYSTEMVERSION) unknown))
+CATALOG_RELEASE ?= current
_PKG_SPECS = $(filter-out $(NOPACKAGE),$(SPKG_SPECS))
@@ -842,7 +843,11 @@
# pkgcheck - check if the package is compliant
#
pkgcheck: $(foreach SPEC,$(_PKG_SPECS),package-$(SPEC))
- $(_DBG)( LC_ALL=C $(GARBIN)/checkpkg $(foreach SPEC,$(_PKG_SPECS),$(SPKG_EXPORT)/`$(call _PKG_ENV,$(SPEC)) mkpackage --tmpdir $(SPKG_TMPDIR) -qs $(WORKDIR)/$(SPEC).gspec -D pkgfile`.gz ) || exit 2;)
+ $(_DBG)( LC_ALL=C $(GARBIN)/checkpkg \
+ --architecture "$(GARCH)" \
+ --os-releases "$(SPKG_OSNAME)" \
+ --catalog-release "$(CATALOG_RELEASE)" \
+ $(foreach SPEC,$(_PKG_SPECS),$(SPKG_EXPORT)/`$(call _PKG_ENV,$(SPEC)) mkpackage --tmpdir $(SPKG_TMPDIR) -qs $(WORKDIR)/$(SPEC).gspec -D pkgfile`.gz ) || exit 2;)
@$(MAKECOOKIE)
pkgcheck-p:
Modified: csw/mgar/gar/v2/lib/python/README
===================================================================
--- csw/mgar/gar/v2/lib/python/README 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/lib/python/README 2010-12-11 18:01:58 UTC (rev 11882)
@@ -21,9 +21,11 @@
tests as means to run various bits of code. Here's the dependency list
for Ubuntu.
- python-cheetah
- python-hachoir-parser
- python-magic
- python-mox
- python-progressbar
+sudo aptitude install \
+ python-cheetah \
+ python-hachoir-parser \
+ python-magic \
+ python-mox \
+ python-progressbar \
+ python-sqlobject \
python-yaml
Modified: csw/mgar/gar/v2/lib/python/catalog.py
===================================================================
--- csw/mgar/gar/v2/lib/python/catalog.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/lib/python/catalog.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -123,6 +123,7 @@
def _GetCatalogData(self, fd):
catalog_data = []
for line in fd:
+ if line.startswith("#"): continue
try:
parsed = self._ParseCatalogLine(line)
catalog_data.append(parsed)
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -3,44 +3,22 @@
# This is the checkpkg library, common for all checkpkg tests written in
# Python.
-import copy
-import cPickle
-import errno
import itertools
import logging
-import operator
import optparse
-import os
import os.path
import re
import pprint
import progressbar
-import socket
-import sqlite3
import sqlobject
-import time
-from sqlobject import sqlbuilder
import subprocess
-import textwrap
-from Cheetah import Template
import database
-import package
import inspective_package
-import package_checks
-import package_stats
import models as m
-import configuration as c
-import tag
+import common_constants
-DEBUG_BREAK_PKGMAP_AFTER = False
-SYSTEM_PKGMAP = "/var/sadm/install/contents"
-NEEDED_SONAMES = "needed sonames"
-RUNPATH = "runpath"
-SONAME = "soname"
-CONFIG_MTIME = "mtime"
-CONFIG_DB_SCHEMA = "db_schema_version"
DO_NOT_REPORT_SURPLUS = set([u"CSWcommon", u"CSWcswclassutils", u"CSWisaexec"])
DO_NOT_REPORT_MISSING = set([])
DO_NOT_REPORT_MISSING_RE = [r"\*?SUNW.*"]
@@ -48,17 +26,11 @@
PSTAMP_RE = r"(?P<username>\w+)@(?P<hostname>[\w\.-]+)-(?P<timestamp>\d+)"
DESCRIPTION_RE = r"^([\S]+) - (.*)$"
BAD_CONTENT_REGEXES = (
- # Slightly obfuscating these by using the default concatenation of
- # strings.
+ # Slightly obfuscating these by using concatenation of strings.
r'/export' r'/medusa',
r'/opt' r'/build',
)
-SYSTEM_SYMLINKS = (
- ("/opt/csw/bdb4", ["/opt/csw/bdb42"]),
- ("/64", ["/amd64", "/sparcv9"]),
- ("/opt/csw/lib/i386", ["/opt/csw/lib"]),
-)
INSTALL_CONTENTS_AVG_LINE_LENGTH = 102.09710677919261
SYS_DEFAULT_RUNPATH = [
"/usr/lib/$ISALIST",
@@ -67,8 +39,7 @@
"/lib",
]
-CONTENT_PKG_RE = r"^\*?(CSW|SUNW)[0-9a-zA-Z\-]?[0-9a-z\-]+$"
-MD5_RE = r"^[0123456789abcdef]{32}$"
+MD5_RE = re.compile(r"^[0123456789abcdef]{32}$")
REPORT_TMPL = u"""#if $missing_deps or $surplus_deps or $orphan_sonames
Dependency issues of $pkgname:
@@ -90,57 +61,6 @@
#end if
"""
-SCREEN_ERROR_REPORT_TMPL = u"""#if $errors
-#if $debug
-ERROR: One or more errors have been found by $name.
-#end if
-#for $pkgname in $errors
-$pkgname:
-#for $error in $errors[$pkgname]
-#if $debug
- $repr($error)
-#elif $error.msg
-$textwrap.fill($error.msg, 78, initial_indent="# ", subsequent_indent="# ")
-# -> $repr($error)
-
-#end if
-#end for
-#end for
-#else
-#if $debug
-OK: $repr($name) module found no problems.
-#end if
-#end if
-#if $messages
-#for $msg in $messages
-$textwrap.fill($msg, 78, initial_indent=" * ", subsequent_indent=" ")
-#end for
-#end if
-#if $gar_lines
-
-# Checkpkg suggests adding the following lines to the GAR recipe:
-# This is a summary; see above for details.
-#for $line in $gar_lines
-$line
-#end for
-#end if
-"""
-
-# http://www.cheetahtemplate.org/docs/users_guide_html_multipage/language.directives.closures.html
-TAG_REPORT_TMPL = u"""#if $errors
-# Tags reported by $name module
-#for $pkgname in $errors
-#for $tag in $errors[$pkgname]
-#if $tag.msg
-$textwrap.fill($tag.msg, 70, initial_indent="# ", subsequent_indent="# ")
-#end if
-$pkgname: ${tag.tag_name}#if $tag.tag_info# $tag.tag_info#end if#
-#end for
-#end for
-#end if
-"""
-
-
class Error(Exception):
pass
@@ -157,11 +77,12 @@
pass
+class SetupError(Error):
+ pass
+
+
def GetOptions():
parser = optparse.OptionParser()
- parser.add_option("-b", "--stats-basedir", dest="stats_basedir",
- help=("The base directory with package statistics "
- "in yaml format, e.g. ~/.checkpkg/stats"))
parser.add_option("-d", "--debug", dest="debug",
default=False, action="store_true",
help="Turn on debugging messages")
@@ -172,8 +93,6 @@
default=False, action="store_true",
help=("Print less messages"))
(options, args) = parser.parse_args()
- if not options.stats_basedir:
- raise ConfigurationError("ERROR: the -b option is missing.")
# Using set() to make the arguments unique.
return options, set(args)
@@ -195,859 +114,9 @@
return m.group("username") if m else None
-class SystemPkgmap(database.DatabaseClient):
- """A class to hold and manipulate the /var/sadm/install/contents file."""
-
- STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
-
- def __init__(self, system_pkgmap_files=None, debug=False):
- """There is no need to re-parse it each time.
-
- Read it slowly the first time and cache it for later."""
- super(SystemPkgmap, self).__init__(debug=debug)
- self.cache = {}
- self.pkgs_by_path_cache = {}
- self.file_mtime = None
- self.cache_mtime = None
- self.initialized = False
- if not system_pkgmap_files:
- self.system_pkgmap_files = [SYSTEM_PKGMAP]
- else:
- self.system_pkgmap_files = system_pkgmap_files
- self.csw_pkg_re = re.compile(CONTENT_PKG_RE)
- self.digits_re = re.compile(r"^[0-9]+$")
-
- def _LazyInitializeDatabase(self):
- if not self.initialized:
- self.InitializeDatabase()
-
- def InitializeRawDb(self):
- """It's necessary for low level operations."""
- if True:
- logging.debug("Connecting to sqlite")
- self.sqlite_conn = sqlite3.connect(self.GetDatabasePath())
-
- def InitializeDatabase(self):
- """Established the connection to the database.
-
- TODO: Refactor this class to first create CswFile with no primary key and
- no indexes.
- """
- need_to_create_tables = False
- db_path = self.GetDatabasePath()
- checkpkg_dir = os.path.join(os.environ["HOME"], self.CHECKPKG_DIR)
- if not os.path.exists(db_path):
- logging.info("Building the cache database %s.", self.system_pkgmap_files)
- logging.info("The cache will be kept in %s.", db_path)
- if not os.path.exists(checkpkg_dir):
- logging.debug("Creating %s", checkpkg_dir)
- os.mkdir(checkpkg_dir)
- need_to_create_tables = True
- self.InitializeRawDb()
- self.InitializeSqlobject()
- if not self.IsDatabaseGoodSchema():
- logging.info("Old database schema detected.")
- self.PurgeDatabase(drop_tables=True)
- need_to_create_tables = True
- if need_to_create_tables:
- self.CreateTables()
- self.PerformInitialDataImport()
- if not self.IsDatabaseUpToDate():
- logging.debug("Rebuilding the package cache, can take a few minutes.")
- self.ClearTablesForUpdates()
- self.RefreshDatabase()
- self.initialized = True
-
- def RefreshDatabase(self):
- for pkgmap_path in self.system_pkgmap_files:
- self._ProcessSystemPkgmap(pkgmap_path)
- self.PopulatePackagesTable()
- self.SetDatabaseMtime()
-
- def PerformInitialDataImport(self):
- """Imports data into the database.
-
- Original bit of code from checkpkg:
- egrep -v 'SUNWbcp|SUNWowbcp|SUNWucb' /var/sadm/install/contents |
- fgrep -f $EXTRACTDIR/liblist >$EXTRACTDIR/shortcatalog
- """
- for pkgmap_path in self.system_pkgmap_files:
- self._ProcessSystemPkgmap(pkgmap_path)
- self.SetDatabaseSchemaVersion()
- self.PopulatePackagesTable()
- self.SetDatabaseMtime()
-
- def _ProcessSystemPkgmap(self, pkgmap_path):
- """Update the database using data from pkgmap.
-
- The strategy to only update the necessary bits:
- - for each new row
- - look it up in the db
- - if doesn't exist, create it
- - if exists, check the
- TODO: continue this description
- """
- INSERT_SQL = """
- INSERT INTO csw_file (basename, path, line)
- VALUES (?, ?, ?);
- """
- sqlite_cursor = self.sqlite_conn.cursor()
- break_after = DEBUG_BREAK_PKGMAP_AFTER
- contents_length = os.stat(pkgmap_path).st_size
- if break_after:
- estimated_lines = break_after
- else:
- estimated_lines = contents_length / INSTALL_CONTENTS_AVG_LINE_LENGTH
- # The progressbar library doesn't like handling larger numbers
- # It displays up to 99% if we feed it a maxval in the range of hundreds of
- # thousands.
- progressbar_divisor = int(estimated_lines / 1000)
- if progressbar_divisor < 1:
- progressbar_divisor = 1
- update_period = 1L
- # To help delete old records
- system_pkgmap_fd = open(pkgmap_path, "r")
- stop_re = re.compile("(%s)" % "|".join(self.STOP_PKGS))
- # Creating a data structure:
- # soname - {<path1>: <line1>, <path2>: <line2>, ...}
- logging.debug("Building database cache db of the %s file",
- pkgmap_path)
- logging.info("Processing %s, it can take a few minutes", pkgmap_path)
- count = itertools.count()
- bar = progressbar.ProgressBar()
- bar.maxval = estimated_lines / progressbar_divisor
- bar.start()
- # I tried dropping the csw_file_basename_idx index to speed up operation,
- # but after I measured the times, it turned out that it doesn't make any
- # difference to the total runnng time.
- # logging.info("Dropping csw_file_basename_idx")
- # sqlite_cursor.execute("DROP INDEX csw_file_basename_idx;")
- for line in system_pkgmap_fd:
- i = count.next()
- if not i % update_period and (i / progressbar_divisor) <= bar.maxval:
- bar.update(i / progressbar_divisor)
- if stop_re.search(line):
- continue
- if line.startswith("#"):
- continue
- fields = re.split(c.WS_RE, line)
- pkgmap_entry_path = fields[0].split("=")[0]
- pkgmap_entry_dir, pkgmap_entry_base_name = os.path.split(pkgmap_entry_path)
- # The following SQLObject-driven inserts are 60 times slower than the raw
- # sqlite API.
- # pkgmap_entry = m.CswFile(basename=pkgmap_entry_base_name,
- # path=pkgmap_entry_dir, line=line.strip())
- # This page has some hints:
- # http://www.mail-archive.com/sqlobject-discuss@lists.sourceforge.net/msg04641.html
- # "These are simple straightforward INSERTs without any additional
- # high-level burden - no SELECT, no caching, nothing. Fire and forget."
- # sql = self.sqo_conn.sqlrepr(
- # sqlobject.sqlbuilder.Insert(m.CswFile.sqlmeta.table, values=record))
- # self.sqo_conn.query(sql)
- # ...unfortunately, it isn't any faster in practice.
- # The fastest way is:
- sqlite_cursor.execute(INSERT_SQL, [pkgmap_entry_base_name,
- pkgmap_entry_dir,
- line.strip()])
- if break_after and i > break_after:
- logging.warning("Breaking after %s for debugging purposes.", break_after)
- break
- bar.finish()
- self.sqlite_conn.commit()
- logging.debug("All lines of %s were processed.", pkgmap_path)
-
- def _ParsePkginfoLine(self, line):
- fields = re.split(c.WS_RE, line)
- pkgname = fields[1]
- pkg_desc = u" ".join(fields[2:])
- return pkgname, pkg_desc
-
- def PopulatePackagesTable(self):
- logging.info("Updating the packages table")
- args = ["pkginfo"]
- pkginfo_proc = subprocess.Popen(args, stdout=subprocess.PIPE)
- stdout, stderr = pkginfo_proc.communicate()
- ret = pkginfo_proc.wait()
- lines = stdout.splitlines()
- bar = progressbar.ProgressBar()
- bar.maxval = len(lines)
- bar.start()
- count = itertools.count()
- INSERT_SQL = """
- INSERT INTO pkginst (pkgname, pkg_desc)
- VALUES (?, ?);
- """
- # If self.GetInstalledPackages calls out to the initialization,
- # the result is an infinite recursion.
- installed_pkgs = self.GetInstalledPackages(initialize=False)
- for line in stdout.splitlines():
- pkgname, pkg_desc = self._ParsePkginfoLine(line)
- if pkgname not in installed_pkgs:
- # This is slow:
- # pkg = m.Pkginst(pkgname=pkgname, pkg_desc=pkg_desc)
- # This is much faster:
- self.sqlite_conn.execute(INSERT_SQL, [pkgname, pkg_desc])
- i = count.next()
- bar.update(i)
- # Need to commit, otherwise subsequent SQLObject calls will fail.
- self.sqlite_conn.commit()
- bar.finish()
-
- def SetDatabaseMtime(self):
- mtime = self.GetFileMtime()
- res = m.CswConfig.select(m.CswConfig.q.option_key==CONFIG_MTIME)
- if res.count() == 0:
- logging.debug("Inserting the mtime (%s) into the database.", mtime)
- config_record = m.CswConfig(option_key=CONFIG_MTIME, float_value=mtime)
- else:
- logging.debug("Updating the mtime (%s) in the database.", mtime)
- res.getOne().float_value = mtime
-
- def SetDatabaseSchemaVersion(self):
- try:
- config_option = m.CswConfig.select(
- m.CswConfig.q.option_key==CONFIG_DB_SCHEMA).getOne()
- config_option.int_value = database.DB_SCHEMA_VERSION
- except sqlobject.main.SQLObjectNotFound, e:
- version = m.CswConfig(option_key=CONFIG_DB_SCHEMA,
- int_value=database.DB_SCHEMA_VERSION)
-
- def GetPkgmapLineByBasename(self, filename):
- """Returns pkgmap lines by basename:
- {
- path1: line1,
- path2: line2,
- }
- """
- if filename in self.cache:
- return self.cache[filename]
- self._LazyInitializeDatabase()
- res = m.CswFile.select(m.CswFile.q.basename==filename)
- lines = {}
- for obj in res:
- lines[obj.path] = obj.line
- if len(lines) == 0:
- logging.debug("Cache doesn't contain filename %s", filename)
- self.cache[filename] = lines
- return lines
-
- def _InferPackagesFromPkgmapLine(self, line):
- """Given a pkgmap line, return all packages it contains."""
- line = line.strip()
- parts = re.split(c.WS_RE, line)
- pkgs = []
- if parts[1] == 'd':
- parts = parts[6:]
- while parts:
- part = parts.pop()
- if self.digits_re.match(part):
- break
- elif "none" == part:
- break
- pkgs.append(part)
- # Make the packages appear in the same order as in the install/contents
- # file.
- pkgs.reverse()
- return pkgs
-
- def GetPathsAndPkgnamesByBasename(self, filename):
- """Returns paths and packages by basename.
-
- e.g.
- {"/opt/csw/lib": ["CSWfoo", "CSWbar"],
- "/opt/csw/1/lib": ["CSWfoomore"]}
- """
- lines = self.GetPkgmapLineByBasename(filename)
- pkgs = {}
- # Infer packages
- for file_path in lines:
- pkgs[file_path] = self._InferPackagesFromPkgmapLine(lines[file_path])
- # self.error_mgr_mock.GetPathsAndPkgnamesByBasename('libc.so.1').AndReturn({
- # "/usr/lib": (u"SUNWcsl",)})
- logging.debug("self.error_mgr_mock.GetPathsAndPkgnamesByBasename(%s).AndReturn(%s)",
- repr(filename), pprint.pformat(pkgs))
- return pkgs
-
- def GetPkgByPath(self, full_path):
- if full_path not in self.pkgs_by_path_cache:
- self._LazyInitializeDatabase()
- path, basename = os.path.split(full_path)
- try:
- obj = m.CswFile.select(
- sqlobject.AND(
- m.CswFile.q.path==path,
- m.CswFile.q.basename==basename)).getOne()
- self.pkgs_by_path_cache[full_path] = self._InferPackagesFromPkgmapLine(
- obj.line)
- except sqlobject.main.SQLObjectNotFound, e:
- logging.debug("Couldn't find in the db: %s/%s", path, basename)
- logging.debug(e)
- self.pkgs_by_path_cache[full_path] = []
- logging.debug("self.error_mgr_mock.GetPkgByPath(%s).AndReturn(%s)",
- repr(full_path), pprint.pformat(self.pkgs_by_path_cache[full_path]))
- return self.pkgs_by_path_cache[full_path]
-
- def GetDatabaseMtime(self):
- if not self.cache_mtime:
- res = m.CswConfig.select(m.CswConfig.q.option_key==CONFIG_MTIME)
- if res.count() == 1:
- self.cache_mtime = res.getOne().float_value
- elif res.count() < 1:
- self.cache_mtime = 1
- logging.debug("GetDatabaseMtime() --> %s", self.cache_mtime)
- return self.cache_mtime
-
- def GetFileMtime(self):
- if not self.file_mtime:
- stat_data = os.stat(SYSTEM_PKGMAP)
- self.file_mtime = stat_data.st_mtime
- return self.file_mtime
-
- def GetDatabaseSchemaVersion(self):
- schema_on_disk = 1L
- if not m.CswConfig.tableExists():
- return schema_on_disk;
- res = m.CswConfig.select(m.CswConfig.q.option_key == CONFIG_DB_SCHEMA)
- if res.count() < 1:
- logging.debug("No db schema value found, assuming %s.",
- schema_on_disk)
- elif res.count() == 1:
- schema_on_disk = res.getOne().int_value
- return schema_on_disk
-
- def IsDatabaseUpToDate(self):
- f_mtime_epoch = self.GetFileMtime()
- d_mtime_epoch = self.GetDatabaseMtime()
- f_mtime = time.gmtime(int(f_mtime_epoch))
- d_mtime = time.gmtime(int(d_mtime_epoch))
- logging.debug("IsDatabaseUpToDate: f_mtime %s, d_time: %s", f_mtime, d_mtime)
- # Rounding up to integer seconds. There is a race condition:
- # pkgadd finishes at 100.1
- # checkpkg reads /var/sadm/install/contents at 100.2
- # new pkgadd runs and finishes at 100.3
- # subsequent checkpkg runs won't pick up the last change.
- # I don't expect pkgadd to run under 1s.
- fresh = f_mtime <= d_mtime
- good_version = self.GetDatabaseSchemaVersion() >= database.DB_SCHEMA_VERSION
- logging.debug("IsDatabaseUpToDate: good_version=%s, fresh=%s",
- repr(good_version), repr(fresh))
- return fresh and good_version
-
- def ClearTablesForUpdates(self):
- for table in self.TABLES_THAT_NEED_UPDATES:
- table.clearTable()
-
- def PurgeDatabase(self, drop_tables=False):
- if drop_tables:
- for table in self.TABLES:
- if table.tableExists():
- table.dropTable()
- else:
- logging.debug("Truncating all tables")
- for table in self.TABLES:
- table.clearTable()
-
- def GetInstalledPackages(self, initialize=True):
- """Returns a dictionary of all installed packages."""
- if initialize:
- self._LazyInitializeDatabase()
- res = m.Pkginst.select()
- return dict([[str(x.pkgname), str(x.pkg_desc)] for x in res])
-
-
-class LddEmulator(object):
- """A class to emulate ldd(1)
-
- Used primarily to resolve SONAMEs and detect package dependencies.
- """
- def __init__(self):
- self.runpath_expand_cache = {}
- self.runpath_origin_expand_cache = {}
- self.symlink_expand_cache = {}
- self.symlink64_cache = {}
- self.runpath_sanitize_cache = {}
-
- def ExpandRunpath(self, runpath, isalist, binary_path):
- """Expands a signle runpath element.
-
- Args:
- runpath: e.g. "/opt/csw/lib/$ISALIST"
- isalist: isalist elements
- binary_path: Necessary to expand $ORIGIN
- """
- key = (runpath, tuple(isalist))
- if key not in self.runpath_expand_cache:
- origin_present = False
- # Emulating $ISALIST and $ORIGIN expansion
- if '$ORIGIN' in runpath:
- origin_present = True
- if origin_present:
- key_o = (runpath, tuple(isalist), binary_path)
- if key_o in self.runpath_origin_expand_cache:
- return self.runpath_origin_expand_cache[key_o]
- else:
- if not binary_path.startswith("/"):
- binary_path = "/" + binary_path
- runpath = runpath.replace('$ORIGIN', binary_path)
- if '$ISALIST' in runpath:
- expanded_list = [runpath.replace('/$ISALIST', '')]
- expanded_list += [runpath.replace('$ISALIST', isa) for isa in isalist]
- else:
- expanded_list = [runpath]
- expanded_list = [os.path.abspath(p) for p in expanded_list]
- if not origin_present:
- self.runpath_expand_cache[key] = expanded_list
- else:
- self.runpath_origin_expand_cache[key_o] = expanded_list
- return self.runpath_origin_expand_cache[key_o]
- return self.runpath_expand_cache[key]
-
- def ExpandSymlink(self, symlink, target, input_path):
- key = (symlink, target, input_path)
- if key not in self.symlink_expand_cache:
- symlink_re = re.compile(r"%s(/|$)" % symlink)
- if re.search(symlink_re, input_path):
- result = input_path.replace(symlink, target)
- else:
- result = input_path
- self.symlink_expand_cache[key] = result
- return self.symlink_expand_cache[key]
-
- def Emulate64BitSymlinks(self, runpath_list):
- """Need to emulate the 64 -> amd64, 64 -> sparcv9 symlink
-
- Since we don't know the architecture, we are adding both amd64 and
- sparcv9. It should be safe - there are other checks that make sure
- that right architectures are in the right directories.
- """
- key = tuple(runpath_list)
- if key not in self.symlink64_cache:
- symlinked_list = []
- for runpath in runpath_list:
- for symlink, expansion_list in SYSTEM_SYMLINKS:
- for target in expansion_list:
- expanded = self.ExpandSymlink(symlink, target, runpath)
- if expanded not in symlinked_list:
- symlinked_list.append(expanded)
- self.symlink64_cache[key] = symlinked_list
- return self.symlink64_cache[key]
-
- def SanitizeRunpath(self, runpath):
- if runpath not in self.runpath_sanitize_cache:
- self.runpath_sanitize_cache[runpath] = os.path.normpath(runpath)
- return self.runpath_sanitize_cache[runpath]
-
-
- def ResolveSoname(self, runpath_list, soname, isalist,
- path_list, binary_path):
- """Emulates ldd behavior, minimal implementation.
-
- runpath: e.g. ["/opt/csw/lib/$ISALIST", "/usr/lib"]
- soname: e.g. "libfoo.so.1"
- isalist: e.g. ["sparcv9", "sparcv8"]
- path_list: A list of paths where the soname is present, e.g.
- ["/opt/csw/lib", "/opt/csw/lib/sparcv9"]
-
- The function returns the one path.
- """
- # Emulating the install time symlinks, for instance, if the prototype contains
- # /opt/csw/lib/i386/foo.so.0 and /opt/csw/lib/i386 is a symlink to ".",
- # the shared library ends up in /opt/csw/lib/foo.so.0 and should be
- # findable even when RPATH does not contain $ISALIST.
- original_paths_by_expanded_paths = {}
- for p in path_list:
- expanded_p_list = self.Emulate64BitSymlinks([p])
- # We can't just expand and return; we need to return one of the paths given
- # in the path_list.
- for expanded_p in expanded_p_list:
- original_paths_by_expanded_paths[expanded_p] = p
- logging.debug(
- "%s: looking for %s in %s",
- soname, runpath_list, original_paths_by_expanded_paths.keys())
- for runpath_expanded in runpath_list:
- if runpath_expanded in original_paths_by_expanded_paths:
- # logging.debug("Found %s",
- # original_paths_by_expanded_paths[runpath_expanded])
- return original_paths_by_expanded_paths[runpath_expanded]
-
-
-def ParseDumpOutput(dump_output):
- binary_data = {RUNPATH: [],
- NEEDED_SONAMES: []}
- runpath = []
- rpath = []
- for line in dump_output.splitlines():
- fields = re.split(c.WS_RE, line)
- if len(fields) < 3:
- continue
- if fields[1] == "NEEDED":
- binary_data[NEEDED_SONAMES].append(fields[2])
- elif fields[1] == "RUNPATH":
- runpath.extend(fields[2].split(":"))
- elif fields[1] == "RPATH":
- rpath.extend(fields[2].split(":"))
- elif fields[1] == "SONAME":
- binary_data[SONAME] = fields[2]
- if runpath:
- binary_data[RUNPATH].extend(runpath)
- elif rpath:
- binary_data[RUNPATH].extend(rpath)
-
- # Converting runpath to a tuple, which is a hashable data type and can act as
- # a key in a dict.
- binary_data[RUNPATH] = tuple(binary_data[RUNPATH])
- # the NEEDED list must not be modified, converting to a tuple.
- binary_data[NEEDED_SONAMES] = tuple(binary_data[NEEDED_SONAMES])
- binary_data["RUNPATH RPATH the same"] = (runpath == rpath)
- binary_data["RPATH set"] = bool(rpath)
- binary_data["RUNPATH set"] = bool(runpath)
- return binary_data
-
-
-class CheckpkgManagerBase(object):
- """Common functions between the older and newer calling functions."""
-
- def __init__(self, name, stats_basedir, md5sum_list, debug=False):
- self.debug = debug
- self.name = name
- self.md5sum_list = md5sum_list
- self.stats_basedir = stats_basedir
- self.errors = []
- self.individual_checks = []
- self.set_checks = []
- self.packages = []
-
- def GetPackageStatsList(self):
- return [package_stats.PackageStats(None, self.stats_basedir, x)
- for x in self.md5sum_list]
-
- def FormatReports(self, errors, messages, gar_lines):
- namespace = {
- "name": self.name,
- "errors": errors,
- "debug": self.debug,
- "textwrap": textwrap,
- "messages": messages,
- "gar_lines": gar_lines,
- }
- screen_t = Template.Template(SCREEN_ERROR_REPORT_TMPL, searchList=[namespace])
- tags_report_t = Template.Template(TAG_REPORT_TMPL, searchList=[namespace])
- return screen_t, tags_report_t
-
- def SetErrorsToDict(self, set_errors, a_dict):
- # These were generated by a set, but are likely to be bound to specific
- # packages. We'll try to preserve the package assignments.
- errors = copy.copy(a_dict)
- for tag in set_errors:
- if tag.pkgname:
- if not tag.pkgname in errors:
- errors[tag.pkgname] = []
- errors[tag.pkgname].append(tag)
- else:
- if "package-set" not in errors:
- errors["package-set"] = []
- errors["package-set"].append(tag)
- return errors
-
- def GetOptimizedAllStats(self, stats_obj_list):
- logging.info("Unwrapping candies...")
- pkgs_data = []
- counter = itertools.count()
- length = len(stats_obj_list)
- bar = progressbar.ProgressBar()
- bar.maxval = length
- bar.start()
- for stats_obj in stats_obj_list:
- # pkg_data = {}
- # This bit is tightly tied to the data structures returned by
- # PackageStats.
- #
- # Python strings are already implementing the flyweight pattern. What's
- # left is lists and dictionaries.
- i = counter.next()
- # logging.debug("Loading stats for %s (%s/%s)",
- # stats_obj.md5sum, i, length)
- raw_pkg_data = stats_obj.GetAllStats()
- pkg_data = raw_pkg_data
- pkgs_data.append(pkg_data)
- bar.update(i)
- bar.finish()
- return pkgs_data
-
- def Run(self):
- """Runs all the checks
-
- Returns a tuple of an exit code and a report.
- """
- packages_data = self.GetPackageStatsList()
- db_stat_objs_by_pkgname = {}
- obj_id_list = []
- for pkg in packages_data:
- db_obj = pkg.GetDbObject()
- db_stat_objs_by_pkgname[db_obj.pkginst.pkgname] = db_obj
- obj_id_list.append(db_obj.id)
- logging.debug("Deleting old %s errors from the database.",
- db_obj.pkginst.pkgname)
- conn = sqlobject.sqlhub.processConnection
- # It's the maximum number of ORs in a SQL statement.
- # Slicing the long list up into s-sized segments. 1000 is too much.
- obj_id_lists = SliceList(obj_id_list, 900)
- for obj_id_list in obj_id_lists:
- # WARNING: This is raw SQL, potentially breaking during a transition to
- # another db. It's here for efficiency.
- sql = ("DELETE FROM checkpkg_error_tag WHERE %s;"
- % " OR ".join("srv4_file_id = %s" % x for x in obj_id_list))
- conn.query(sql)
- # Need to construct the predicate by hand. Otherwise:
- # File "/opt/csw/lib/python/site-packages/sqlobject/sqlbuilder.py",
- # line 829, in OR
- # return SQLOp("OR", op1, OR(*ops))
- # RuntimeError: maximum recursion depth exceeded while calling a Python object
- #
- # The following also tries to use recursion and fails.
- # delete_predicate = sqlobject.OR(False)
- # for pred in delete_predicate_list:
- # delete_predicate = sqlobject.OR(delete_predicate, pred)
- # conn.query(
- # conn.sqlrepr(sqlbuilder.Delete(m.CheckpkgErrorTag.sqlmeta.table,
- # delete_predicate
- # )))
- # res = m.CheckpkgErrorTag.select(m.CheckpkgErrorTag.q.srv4_file==db_obj)
- # for obj in res:
- # obj.destroySelf()
- errors, messages, gar_lines = self.GetAllTags(packages_data)
- no_errors = len(errors) + 1
- bar = progressbar.ProgressBar()
- bar.maxval = no_errors
- count = itertools.count(1)
- logging.info("Stuffing the candies under the pillow...")
- bar.start()
- for pkgname, es in errors.iteritems():
- logging.debug("Saving %s errors to the database.", pkgname)
- for e in es:
- db_error = m.CheckpkgErrorTag(srv4_file=db_stat_objs_by_pkgname[e.pkgname],
- pkgname=e.pkgname,
- tag_name=e.tag_name,
- tag_info=e.tag_info,
- msg=e.msg)
- bar.update(count.next())
- bar.finish()
- flat_error_list = reduce(operator.add, errors.values(), [])
- screen_report, tags_report = self.FormatReports(errors, messages, gar_lines)
- exit_code = 0
- return (exit_code, screen_report, tags_report)
-
-
-class CheckInterfaceBase(object):
- """Proxies interaction with checking functions.
-
- It wraps access to the /var/sadm/install/contents cache.
- """
-
- def __init__(self, system_pkgmap=None, lines_dict=None):
- self.system_pkgmap = system_pkgmap
- if not self.system_pkgmap:
- self.system_pkgmap = SystemPkgmap()
- self.common_paths = {}
- if lines_dict:
- self.lines_dict = lines_dict
- else:
- self.lines_dict = {}
-
- def GetPathsAndPkgnamesByBasename(self, basename):
- """Proxies calls to self.system_pkgmap."""
- return self.system_pkgmap.GetPathsAndPkgnamesByBasename(basename)
-
- def GetPkgByPath(self, path):
- """Proxies calls to self.system_pkgmap."""
- return self.system_pkgmap.GetPkgByPath(path)
-
- def GetInstalledPackages(self, initialize=True):
- return self.system_pkgmap.GetInstalledPackages(initialize)
-
- def _GetPathsForArch(self, arch):
- if not arch in self.lines_dict:
- file_name = os.path.join(
- os.path.dirname(__file__), "..", "..", "etc", "commondirs-%s" % arch)
- logging.debug("opening %s", file_name)
- f = open(file_name, "r")
- self.lines_dict[arch] = f.read().splitlines()
- f.close()
- return self.lines_dict[arch]
-
- def GetCommonPaths(self, arch):
- """Returns a list of paths for architecture, from gar/etc/commondirs*."""
- # TODO: If this was cached, it could save a significant amount of time.
- if arch not in ('i386', 'sparc', 'all'):
- logging.warn("Wrong arch: %s", repr(arch))
- return []
- if arch == 'all':
- archs = ('i386', 'sparc')
- else:
- archs = [arch]
- lines = []
- for arch in archs:
- lines.extend(self._GetPathsForArch(arch))
- return lines
-
-
-class IndividualCheckInterface(CheckInterfaceBase):
- """To be passed to the checking functions.
-
- Wraps the creation of tag.CheckpkgTag objects.
- """
-
- def __init__(self, pkgname, system_pkgmap=None):
- super(IndividualCheckInterface, self).__init__(system_pkgmap)
- self.pkgname = pkgname
- self.errors = []
-
- def ReportError(self, tag_name, tag_info=None, msg=None):
- logging.debug("self.error_mgr_mock.ReportError(%s, %s, %s)",
- repr(tag_name), repr(tag_info), repr(msg))
- checkpkg_tag = tag.CheckpkgTag(self.pkgname, tag_name, tag_info, msg=msg)
- self.errors.append(checkpkg_tag)
-
-
-class SetCheckInterface(CheckInterfaceBase):
- """To be passed to set checking functions."""
-
- def __init__(self, system_pkgmap=None):
- super(SetCheckInterface, self).__init__(system_pkgmap)
- self.errors = []
-
- def ReportError(self, pkgname, tag_name, tag_info=None, msg=None):
- logging.debug("self.error_mgr_mock.ReportError(%s, %s, %s, %s)",
- repr(pkgname),
- repr(tag_name), repr(tag_info), repr(msg))
- checkpkg_tag = tag.CheckpkgTag(pkgname, tag_name, tag_info, msg=msg)
- self.errors.append(checkpkg_tag)
-
-
-class CheckpkgMessenger(object):
- """Class responsible for passing messages from checks to the user."""
- def __init__(self):
- self.messages = []
- self.one_time_messages = {}
- self.gar_lines = []
-
- def Message(self, m):
- logging.debug("self.messenger.Message(%s)", repr(m))
- self.messages.append(m)
-
- def OneTimeMessage(self, key, m):
- logging.debug("self.messenger.OneTimeMessage(%s, %s)", repr(key), repr(m))
- if key not in self.one_time_messages:
- self.one_time_messages[key] = m
-
- def SuggestGarLine(self, m):
- logging.debug("self.messenger.SuggestGarLine(%s)", repr(m))
- self.gar_lines.append(m)
-
-
-class CheckpkgManager2(CheckpkgManagerBase):
- """The second incarnation of the checkpkg manager.
-
- Implements the API to be used by checking functions.
-
- Its purpose is to reduce the amount of boilerplate code and allow for easier
- unit test writing.
- """
- def _RegisterIndividualCheck(self, function):
- self.individual_checks.append(function)
-
- def _RegisterSetCheck(self, function):
- self.set_checks.append(function)
-
- def _AutoregisterChecks(self):
- """Autodetects all defined checks."""
- logging.debug("CheckpkgManager2._AutoregisterChecks()")
- checkpkg_module = package_checks
- members = dir(checkpkg_module)
- for member_name in members:
- logging.debug("Examining module member: %s", repr(member_name))
- member = getattr(checkpkg_module, member_name)
- if callable(member):
- if member_name.startswith("Check"):
- logging.debug("Registering individual check %s", repr(member_name))
- self._RegisterIndividualCheck(member)
- elif member_name.startswith("SetCheck"):
- logging.debug("Registering set check %s", repr(member_name))
- self._RegisterSetCheck(member)
-
- def GetAllTags(self, stats_obj_list):
- errors = {}
- pkgmap = SystemPkgmap()
- logging.debug("Loading all package statistics.")
- pkgs_data = self.GetOptimizedAllStats(stats_obj_list)
- logging.debug("All package statistics loaded.")
- messenger = CheckpkgMessenger()
- # Individual checks
- count = itertools.count()
- bar = progressbar.ProgressBar()
- bar.maxval = len(pkgs_data) * len(self.individual_checks)
- logging.info("Tasting candies one by one...")
- bar.start()
- for pkg_data in pkgs_data:
- pkgname = pkg_data["basic_stats"]["pkgname"]
- check_interface = IndividualCheckInterface(pkgname, pkgmap)
- for function in self.individual_checks:
- logger = logging.getLogger("%s-%s" % (pkgname, function.__name__))
- logger.debug("Calling %s", function.__name__)
- function(pkg_data, check_interface, logger=logger, messenger=messenger)
- if check_interface.errors:
- errors[pkgname] = check_interface.errors
- bar.update(count.next())
- bar.finish()
- # Set checks
- logging.info("Tasting them all at once...")
- for function in self.set_checks:
- logger = logging.getLogger(function.__name__)
- check_interface = SetCheckInterface(pkgmap)
- logger.debug("Calling %s", function.__name__)
- function(pkgs_data, check_interface, logger=logger, messenger=messenger)
- if check_interface.errors:
- errors = self.SetErrorsToDict(check_interface.errors, errors)
- messages = messenger.messages + messenger.one_time_messages.values()
- return errors, messages, messenger.gar_lines
-
- def Run(self):
- self._AutoregisterChecks()
- return super(CheckpkgManager2, self).Run()
-
-
-def GetIsalist():
- args = ["isalist"]
- isalist_proc = subprocess.Popen(args, stdout=subprocess.PIPE)
- stdout, stderr = isalist_proc.communicate()
- ret = isalist_proc.wait()
- if ret:
- logging.error("Calling isalist has failed.")
- isalist = re.split(r"\s+", stdout.strip())
- return tuple(isalist)
-
-
-def ErrorTagsFromFile(file_name):
- fd = open(file_name)
- error_tags = []
- for line in fd:
- if line.startswith("#"):
- continue
- pkgname, tag_name, tag_info = tag.ParseTagLine(line)
- error_tags.append(tag.CheckpkgTag(pkgname, tag_name, tag_info))
- return error_tags
-
-
-def SliceList(l, size):
- """Trasforms a list into a list of lists."""
- idxes = xrange(0, len(l), size)
- sliced = [l[i:i+size] for i in idxes]
- return sliced
-
def IsMd5(s):
# For optimization, move the compilation elsewhere.
- md5_re = re.compile(MD5_RE)
- return md5_re.match(s)
+ return MD5_RE.match(s)
def GetPackageStatsByFilenamesOrMd5s(args, debug=False):
filenames = []
@@ -1059,15 +128,15 @@
filenames.append(arg)
srv4_pkgs = [inspective_package.InspectiveCswSrv4File(x) for x in filenames]
pkgstat_objs = []
- bar = progressbar.ProgressBar()
- bar.maxval = len(md5s) + len(srv4_pkgs)
- bar.start()
+ pbar = progressbar.ProgressBar()
+ pbar.maxval = len(md5s) + len(srv4_pkgs)
+ pbar.start()
counter = itertools.count()
for pkg in srv4_pkgs:
pkgstat_objs.append(package_stats.PackageStats(pkg, debug=debug))
- bar.update(counter.next())
+ pbar.update(counter.next())
for md5 in md5s:
pkgstat_objs.append(package_stats.PackageStats(None, md5sum=md5, debug=debug))
- bar.update(counter.next())
- bar.finish()
+ pbar.update(counter.next())
+ pbar.finish()
return pkgstat_objs
Added: csw/mgar/gar/v2/lib/python/checkpkg2.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg2.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/checkpkg2.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -0,0 +1,179 @@
+#!/usr/bin/env python2.6
+#
+# checkpkg
+#
+
+import logging
+import operator
+import optparse
+import os
+import sys
+import textwrap
+import configuration
+import datetime
+import database
+
+import package_stats
+import checkpkg
+import checkpkg_lib
+import overrides
+import models
+import sqlobject
+
+USAGE = """%prog [ options ] pkg1 [ pkg2 [ ... ] ]"""
+CHECKPKG_MODULE_NAME = "The main checking module."
+BEFORE_OVERRIDES = """If any of the reported errors were false positives, you
+can override them pasting the lines below to the GAR recipe."""
+
+AFTER_OVERRIDES = """Please note that checkpkg isn't suggesting you should
+simply add these overrides do the Makefile. It only informs what the overrides
+could look like. You need to understand what are the reported issues about and
+use your best judgement to decide whether to fix the underlying problems or
+override them. For more information, scroll up and read the detailed
+messages."""
+
+UNAPPLIED_OVERRIDES = """WARNING: Some overrides did not match any errors.
+They can be removed, as they don't take any effect anyway. If you're getting
+errors at the same time, maybe you didn't specify the overrides correctly."""
+
+
+class Error(Exception):
+ """Generic error."""
+
+
+class UsageError(Error):
+ """Problem with usage, e.g. command line options."""
+
+
+def main():
+ configuration.SetUpSqlobjectConnection()
+ parser = optparse.OptionParser(USAGE)
+ parser.add_option("-d", "--debug",
+ dest="debug",
+ action="store_true",
+ default=False,
+ help="Switch on debugging messages")
+ parser.add_option("-q", "--quiet",
+ dest="quiet",
+ action="store_true",
+ default=False,
+ help="Display less messages")
+ parser.add_option("--catalog-release",
+ dest="catrel",
+ default="unstable",
+ help="A catalog release: experimental, unstable, testing, stable.")
+ parser.add_option("-r", "--os-releases",
+ dest="osrel_commas",
+ help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], "
+ "e.g. 'SunOS5.9,SunOS5.10'."))
+ parser.add_option("-a", "--architecture",
+ dest="arch",
+ help="Architecture: i386, sparc.")
+ parser.add_option("--profile", dest="profile",
+ default=False, action="store_true",
+ help="Enable profiling (a developer option).")
+ options, args = parser.parse_args()
+ assert len(args), "The list of files or md5 sums must be not empty."
+ logging_level = logging.INFO
+ if options.quiet:
+ logging_level = logging.WARNING
+ elif options.debug:
+ # If both flags are set, debug wins.
+ logging_level = logging.DEBUG
+ logging.basicConfig(level=logging_level)
+ logging.debug("Starting.")
+
+ dm = database.DatabaseManager()
+ dm.AutoManage()
+
+
+ err_msg_list = []
+ if not options.osrel_commas:
+ err_msg_list.append("Please specify --os-releases.")
+ if not options.arch:
+ err_msg_list.append("Please specify --architecture.")
+ if err_msg_list:
+ raise UsageError(" ".join(err_msg_list))
+
+ stats_list = []
+ collector = package_stats.StatsCollector(
+ logger=logging,
+ debug=options.debug)
+ # We need to separate files and md5 sums.
+ md5_sums, file_list = [], []
+ for arg in args:
+ if checkpkg.MD5_RE.match(arg):
+ md5_sums.append(arg)
+ else:
+ file_list.append(arg)
+ if file_list:
+ stats_list = collector.CollectStatsFromFiles(file_list, None)
+ # We need the md5 sums of these files
+ md5_sums.extend([x["basic_stats"]["md5_sum"] for x in stats_list])
+ assert md5_sums, "The list of md5 sums must not be empty."
+ logging.debug("md5_sums: %s", md5_sums)
+ osrel_list = options.osrel_commas.split(",")
+ logging.debug("Reading packages data from the database.")
+ # This part might need improvements in order to handle a whole
+ # catalog. On the other hand, if we already have the whole catalog in
+ # the database, we can do it altogether differently.
+ # Transforming the result to a list in order to force object
+ # retrieval.
+ sqo_pkgs = list(models.Srv4FileStats.select(
+ sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums)))
+ tags_for_all_osrels = []
+ sqo_arch = models.Architecture.selectBy(name=options.arch).getOne()
+ sqo_catrel = models.CatalogRelease.selectBy(name=options.catrel).getOne()
+ for osrel in osrel_list:
+ sqo_osrel = models.OsRelease.selectBy(short_name=osrel).getOne()
+ dm.VerifyContents(sqo_osrel, sqo_arch)
+ check_manager = checkpkg_lib.CheckpkgManager2(
+ CHECKPKG_MODULE_NAME,
+ sqo_pkgs,
+ osrel,
+ options.arch,
+ options.catrel,
+ debug=options.debug,
+ show_progress=(not options.quiet))
+ # Running the checks, reporting and exiting.
+ exit_code, screen_report, tags_report = check_manager.Run()
+ screen_report = unicode(screen_report)
+ if not options.quiet and screen_report:
+ # TODO: Write this to screen only after overrides are applied.
+ sys.stdout.write(screen_report)
+ else:
+ logging.debug("No screen report.")
+
+ overrides_list = [list(pkg.GetOverridesResult()) for pkg in sqo_pkgs]
+ override_list = reduce(operator.add, overrides_list)
+ args = (sqo_osrel, sqo_arch, sqo_catrel)
+ tag_lists = [list(pkg.GetErrorTagsResult(*args)) for pkg in sqo_pkgs]
+ error_tags = reduce(operator.add, tag_lists)
+ # TODO: Set the 'overriden' tag for specific error tags
+ (tags_after_overrides,
+ unapplied_overrides) = overrides.ApplyOverrides(error_tags, override_list)
+ tags_for_all_osrels.extend(tags_after_overrides)
+ if not options.quiet:
+ if tags_after_overrides:
+ print textwrap.fill(BEFORE_OVERRIDES, 80)
+ for checkpkg_tag in tags_after_overrides:
+ print checkpkg_tag.ToGarSyntax()
+ print textwrap.fill(AFTER_OVERRIDES, 80)
+ if unapplied_overrides:
+ print textwrap.fill(UNAPPLIED_OVERRIDES, 80)
+ for override in unapplied_overrides:
+ print "* Unused %s" % override
+ exit_code = bool(tags_for_all_osrels)
+ sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+ if "--profile" in sys.argv:
+ import cProfile
+ t_str = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")
+ home = os.environ["HOME"]
+ cprof_file_name = os.path.join(
+ home, ".checkpkg", "run-modules-%s.cprof" % t_str)
+ cProfile.run("main()", sort=1, filename=cprof_file_name)
+ else:
+ main()
Property changes on: csw/mgar/gar/v2/lib/python/checkpkg2.py
___________________________________________________________________
Added: svn:executable
+ *
Added: csw/mgar/gar/v2/lib/python/checkpkg_defaults.ini
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_defaults.ini (rev 0)
+++ csw/mgar/gar/v2/lib/python/checkpkg_defaults.ini 2010-12-11 18:01:58 UTC (rev 11882)
@@ -0,0 +1,6 @@
+; The default setting for checkpkg. Without this file, checkpkg creates a new
+; config file and starts to populate the database.
+
+[database]
+
+auto_manage = no
Added: csw/mgar/gar/v2/lib/python/checkpkg_lib.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -0,0 +1,637 @@
+# A collection of checkpkg-specific classes.
+#
+# This file is supposed to drain the checkpkg.py file until is becomes
+# empty and goes away.
+
+import copy
+from Cheetah import Template
+import logging
+import package_stats
+import package_checks
+import sqlobject
+import itertools
+import progressbar
+import database
+import models as m
+import textwrap
+import os.path
+import tag
+import pprint
+import operator
+import common_constants
+import sharedlib_utils
+import mute_progressbar
+import cPickle
+from sqlobject import sqlbuilder
+
+
+class Error(Exception):
+ pass
+
+
+class CatalogDatabaseError(Error):
+ pass
+
+
+SCREEN_ERROR_REPORT_TMPL = u"""#if $errors
+#if $debug
+ERROR: One or more errors have been found by $name.
+#end if
+#for $pkgname in $errors
+$pkgname:
+#for $error in $errors[$pkgname]
+#if $debug
+ $repr($error)
+#elif $error.msg
+$textwrap.fill($error.msg, 78, initial_indent="# ", subsequent_indent="# ")
+# -> $repr($error)
+
+#end if
+#end for
+#end for
+#else
+#if $debug
+OK: $repr($name) module found no problems.
+#end if
+#end if
+#if $messages
+#for $msg in $messages
+$textwrap.fill($msg, 78, initial_indent=" * ", subsequent_indent=" ")
+#end for
+#end if
+#if $gar_lines
+
+# Checkpkg suggests adding the following lines to the GAR recipe:
+# This is a summary; see above for details.
+#for $line in $gar_lines
+$line
+#end for
+#end if
+"""
+
+# http://www.cheetahtemplate.org/docs/users_guide_html_multipage/language.directives.closures.html
+TAG_REPORT_TMPL = u"""#if $errors
+# Tags reported by $name module
+#for $pkgname in $errors
+#for $tag in $errors[$pkgname]
+#if $tag.msg
+$textwrap.fill($tag.msg, 70, initial_indent="# ", subsequent_indent="# ")
+#end if
+$pkgname: ${tag.tag_name}#if $tag.tag_info# $tag.tag_info#end if#
+#end for
+#end for
+#end if
+"""
+
+
+class SqlobjectHelperMixin(object):
+
+ def __init__(self):
+ super(SqlobjectHelperMixin, self).__init__()
+ self.triad_cache = {}
+
+ def GetSqlobjectTriad(self, osrel, arch, catrel):
+ key = (osrel, arch, catrel)
+ if key not in self.triad_cache:
+ logging.debug("GetSqlobjectTriad(%s, %s, %s)", osrel, arch, catrel)
+ sqo_arch = m.Architecture.select(
+ m.Architecture.q.name==arch).getOne()
+ sqo_osrel = m.OsRelease.select(
+ m.OsRelease.q.short_name==osrel).getOne()
+ sqo_catrel = m.CatalogRelease.select(
+ m.CatalogRelease.q.name==catrel).getOne()
+ self.triad_cache[key] = sqo_osrel, sqo_arch, sqo_catrel
+ return self.triad_cache[key]
+
+
+class CheckpkgManagerBase(SqlobjectHelperMixin):
+ """Common functions between the older and newer calling functions."""
+
+ def __init__(self, name, sqo_pkgs_list, osrel, arch, catrel, debug=False,
+ show_progress=False):
+ super(CheckpkgManagerBase, self).__init__()
+ self.debug = debug
+ self.name = name
+ self.sqo_pkgs_list = sqo_pkgs_list
+ self.errors = []
+ self.individual_checks = []
+ self.set_checks = []
+ self.packages = []
+ self.osrel = osrel
+ self.arch = arch
+ self.catrel = catrel
+ self.show_progress = show_progress
+
+ def GetProgressBar(self):
+ if self.show_progress and not self.debug:
+ return progressbar.ProgressBar()
+ else:
+ return mute_progressbar.MuteProgressBar()
+
+ def GetSqlobjectTriad(self):
+ return super(CheckpkgManagerBase, self).GetSqlobjectTriad(
+ self.osrel, self.arch, self.catrel)
+
+ def GetPackageStatsList(self):
+ raise RuntimeError("Please don't use this function as it violates "
+ "the Law of Demeter.")
+
+ def FormatReports(self, errors, messages, gar_lines):
+ namespace = {
+ "name": self.name,
+ "errors": errors,
+ "debug": self.debug,
+ "textwrap": textwrap,
+ "messages": messages,
+ "gar_lines": gar_lines,
+ }
+ screen_t = Template.Template(SCREEN_ERROR_REPORT_TMPL, searchList=[namespace])
+ tags_report_t = Template.Template(TAG_REPORT_TMPL, searchList=[namespace])
+ return screen_t, tags_report_t
+
+ def SetErrorsToDict(self, set_errors, a_dict):
+ # These were generated by a set, but are likely to be bound to specific
+ # packages. We'll try to preserve the package assignments.
+ errors = copy.copy(a_dict)
+ for tag in set_errors:
+ if tag.pkgname:
+ if not tag.pkgname in errors:
+ errors[tag.pkgname] = []
+ errors[tag.pkgname].append(tag)
+ else:
+ if "package-set" not in errors:
+ errors["package-set"] = []
+ errors["package-set"].append(tag)
+ return errors
+
+ def GetOptimizedAllStats(self, stats_obj_list):
+ logging.info("Unwrapping candies...")
+ pkgs_data = []
+ counter = itertools.count()
+ length = len(stats_obj_list)
+ pbar = self.GetProgressBar()
+ pbar.maxval = length
+ pbar.start()
+ for stats_obj in stats_obj_list:
+ # This bit is tightly tied to the data structures returned by
+ # PackageStats.
+ #
+ # Python strings are already implementing the flyweight pattern. What's
+ # left is lists and dictionaries.
+ i = counter.next()
+ raw_pkg_data = cPickle.loads(stats_obj.data_obj.pickle)
+ pkg_data = raw_pkg_data
+ pkgs_data.append(pkg_data)
+ pbar.update(i)
+ pbar.finish()
+ return pkgs_data
+
+ def Run(self):
+ """Runs all the checks
+
+ Returns a tuple of an exit code and a report.
+ """
+ # packages_data = self.GetPackageStatsList()
+ assert self.sqo_pkgs_list, "The list of packages must not be empty."
+ db_stat_objs_by_pkgname = {}
+ for pkg in self.sqo_pkgs_list:
+ db_stat_objs_by_pkgname[pkg.pkginst.pkgname] = pkg
+ logging.debug("Deleting old errors from the database.")
+ for pkgname, db_obj in db_stat_objs_by_pkgname.iteritems():
+ sqo_os_rel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad()
+ db_obj.RemoveCheckpkgResults(
+ sqo_os_rel, sqo_arch, sqo_catrel)
+ errors, messages, gar_lines = self.GetAllTags(self.sqo_pkgs_list)
+ no_errors = len(errors) + 1
+ pbar = self.GetProgressBar()
+ pbar.maxval = no_errors
+ count = itertools.count(1)
+ logging.info("Stuffing the candies under the pillow...")
+ pbar.start()
+ for pkgname, es in errors.iteritems():
+ logging.debug("Saving errors of %s to the database.", pkgname)
+ for e in es:
+ if e.pkgname not in db_stat_objs_by_pkgname:
+ logging.warning("Not saving an error for %s.", e.pkgname)
+ continue
+ db_error = m.CheckpkgErrorTag(srv4_file=db_stat_objs_by_pkgname[e.pkgname],
+ pkgname=e.pkgname,
+ tag_name=e.tag_name,
+ tag_info=e.tag_info,
+ msg=e.msg,
+ os_rel=sqo_os_rel,
+ catrel=sqo_catrel,
+ arch=sqo_arch)
+ pbar.update(count.next())
+ pbar.finish()
+ flat_error_list = reduce(operator.add, errors.values(), [])
+ screen_report, tags_report = self.FormatReports(errors, messages, gar_lines)
+ exit_code = 0
+ return (exit_code, screen_report, tags_report)
+
+
+class CheckInterfaceBase(object):
+ """Provides an interface for checking functions.
+
+ It wraps access to the catalog database.
+ """
+
+ def __init__(self, osrel, arch, catrel, catalog=None, lines_dict=None):
+ self.osrel = osrel
+ self.arch = arch
+ self.catrel = catrel
+ self.catalog = catalog
+ if not self.catalog:
+ self.catalog = Catalog()
+ self.common_paths = {}
+ if lines_dict:
+ self.lines_dict = lines_dict
+ else:
+ self.lines_dict = {}
+
+ def GetPathsAndPkgnamesByBasename(self, basename):
+ """Proxies calls to class member."""
+ return self.catalog.GetPathsAndPkgnamesByBasename(
+ basename, self.osrel, self.arch, self.catrel)
+
+ def GetPkgByPath(self, file_path):
+ """Proxies calls to self.system_pkgmap."""
+ return self.catalog.GetPkgByPath(
+ file_path, self.osrel, self.arch, self.catrel)
+
+ def GetInstalledPackages(self):
+ return self.catalog.GetInstalledPackages(
+ self.osrel, self.arch, self.catrel)
+
+ def _GetPathsForArch(self, arch):
+ if not arch in self.lines_dict:
+ file_name = os.path.join(
+ os.path.dirname(__file__), "..", "..", "etc", "commondirs-%s" % arch)
+ logging.debug("opening %s", file_name)
+ f = open(file_name, "r")
+ self.lines_dict[arch] = f.read().splitlines()
+ f.close()
+ return self.lines_dict[arch]
+
+ def GetCommonPaths(self, arch):
+ """Returns a list of paths for architecture, from gar/etc/commondirs*."""
+ # TODO: If this was cached, it could save a significant amount of time.
+ if arch not in ('i386', 'sparc', 'all'):
+ logging.warn("Wrong arch: %s", repr(arch))
+ return []
+ if arch == 'all':
+ archs = ('i386', 'sparc')
+ else:
+ archs = [arch]
+ lines = []
+ for arch in archs:
+ lines.extend(self._GetPathsForArch(arch))
+ return lines
+
+
+class IndividualCheckInterface(CheckInterfaceBase):
+ """To be passed to the checking functions.
+
+ Wraps the creation of tag.CheckpkgTag objects.
+ """
+
+ def __init__(self, pkgname, osrel, arch, catrel, catalog=None):
+ super(IndividualCheckInterface, self).__init__(osrel, arch, catrel, catalog)
+ self.pkgname = pkgname
+ self.errors = []
+
+ def ReportError(self, tag_name, tag_info=None, msg=None):
+ logging.debug("self.error_mgr_mock.ReportError(%s, %s, %s)",
+ repr(tag_name), repr(tag_info), repr(msg))
+ checkpkg_tag = tag.CheckpkgTag(self.pkgname, tag_name, tag_info, msg=msg)
+ self.errors.append(checkpkg_tag)
+
+
+class SetCheckInterface(CheckInterfaceBase):
+ """To be passed to set checking functions."""
+
+ def __init__(self, osrel, arch, catrel, catalog=None):
+ super(SetCheckInterface, self).__init__(osrel, arch, catrel, catalog)
+ self.errors = []
+
+ def ReportError(self, pkgname, tag_name, tag_info=None, msg=None):
+ logging.debug("self.error_mgr_mock.ReportError(%s, %s, %s, %s)",
+ repr(pkgname),
+ repr(tag_name), repr(tag_info), repr(msg))
+ checkpkg_tag = tag.CheckpkgTag(pkgname, tag_name, tag_info, msg=msg)
+ self.errors.append(checkpkg_tag)
+
+
+class CheckpkgMessenger(object):
+ """Class responsible for passing messages from checks to the user."""
+ def __init__(self):
+ self.messages = []
+ self.one_time_messages = {}
+ self.gar_lines = []
+
+ def Message(self, m):
+ logging.debug("self.messenger.Message(%s)", repr(m))
+ self.messages.append(m)
+
+ def OneTimeMessage(self, key, m):
+ logging.debug("self.messenger.OneTimeMessage(%s, %s)", repr(key), repr(m))
+ if key not in self.one_time_messages:
+ self.one_time_messages[key] = m
+
+ def SuggestGarLine(self, m):
+ logging.debug("self.messenger.SuggestGarLine(%s)", repr(m))
+ self.gar_lines.append(m)
+
+
+class CheckpkgManager2(CheckpkgManagerBase):
+ """The second incarnation of the checkpkg manager.
+
+ Implements the API to be used by checking functions.
+
+ Its purpose is to reduce the amount of boilerplate code and allow for easier
+ unit test writing.
+ """
+ def _RegisterIndividualCheck(self, function):
+ self.individual_checks.append(function)
+
+ def _RegisterSetCheck(self, function):
+ self.set_checks.append(function)
+
+ def _AutoregisterChecks(self):
+ """Autodetects all defined checks."""
+ logging.debug("CheckpkgManager2._AutoregisterChecks()")
+ checkpkg_module = package_checks
+ members = dir(checkpkg_module)
+ for member_name in members:
+ logging.debug("Examining module member: %s", repr(member_name))
+ member = getattr(checkpkg_module, member_name)
+ if callable(member):
+ if member_name.startswith("Check"):
+ logging.debug("Registering individual check %s", repr(member_name))
+ self._RegisterIndividualCheck(member)
+ elif member_name.startswith("SetCheck"):
+ logging.debug("Registering set check %s", repr(member_name))
+ self._RegisterSetCheck(member)
+
+ def GetAllTags(self, stats_obj_list):
+ errors = {}
+ catalog = Catalog()
+ logging.debug("Loading all package statistics.")
+ pkgs_data = self.GetOptimizedAllStats(stats_obj_list)
+ logging.debug("All package statistics loaded.")
+ messenger = CheckpkgMessenger()
+ # Individual checks
+ count = itertools.count()
+ pbar = self.GetProgressBar()
+ pbar.maxval = len(pkgs_data) * len(self.individual_checks)
+ logging.info("Tasting candies one by one...")
+ pbar.start()
+ for pkg_data in pkgs_data:
+ pkgname = pkg_data["basic_stats"]["pkgname"]
+ check_interface = IndividualCheckInterface(
+ pkgname, self.osrel, self.arch, self.catrel, catalog)
+ for function in self.individual_checks:
+ logger = logging.getLogger("%s-%s" % (pkgname, function.__name__))
+ logger.debug("Calling %s", function.__name__)
+ function(pkg_data, check_interface, logger=logger, messenger=messenger)
+ if check_interface.errors:
+ errors[pkgname] = check_interface.errors
+ pbar.update(count.next())
+ pbar.finish()
+ # Set checks
+ logging.info("Tasting them all at once...")
+ for function in self.set_checks:
+ logger = logging.getLogger(function.__name__)
+ check_interface = SetCheckInterface(
+ self.osrel, self.arch, self.catrel, catalog)
+ logger.debug("Calling %s", function.__name__)
+ function(pkgs_data, check_interface, logger=logger, messenger=messenger)
+ if check_interface.errors:
+ errors = self.SetErrorsToDict(check_interface.errors, errors)
+ messages = messenger.messages + messenger.one_time_messages.values()
+ return errors, messages, messenger.gar_lines
+
+ def Run(self):
+ self._AutoregisterChecks()
+ return super(CheckpkgManager2, self).Run()
+
+
+def SliceList(l, size):
+ """Trasforms a list into a list of lists."""
+ idxes = xrange(0, len(l), size)
+ sliced = [l[i:i+size] for i in idxes]
+ return sliced
+
+
+class CatalogMixin(SqlobjectHelperMixin):
+ """Responsible for functionality related to catalog operations.
+
+ These include:
+ - getting a list of all packages
+ - getting a list of packages that contain certain files
+ - getting a list of packages that contain files of certain names
+ """
+
+ def __init__(self):
+ super(CatalogMixin, self).__init__()
+ self.pkgs_by_path_cache = {}
+
+ def GetInstalledPackages(self, osrel, arch, catrel):
+ sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
+ osrel, arch, catrel)
+ logging.debug("GetInstalledPackages(%s, %s, %s)"
+ % (osrel, arch, catrel))
+
+ # Object defining a filter for our OS release, architecture and catalog
+ # release.
+ oac = sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel)
+ # Join object, responsible for connecting the tables the right way.
+ join = [
+ sqlbuilder.INNERJOINOn(None,
+ m.Srv4FileStats,
+ m.Pkginst.q.id==m.Srv4FileStats.q.pkginst),
+ sqlbuilder.INNERJOINOn(None,
+ m.Srv4FileInCatalog,
+ m.Srv4FileStats.q.id==m.Srv4FileInCatalog.q.srv4file),
+ ]
+
+ res = m.Pkginst.select(oac, join=join)
+ pkgs = [x.pkgname for x in res]
+ return pkgs
+
+ def GetPathsAndPkgnamesByBasename(self, basename, osrel, arch, catrel):
+ """Retrieves pkginst names of packages that have certain files.
+
+ Since it needs to match a specific catalog, a table join is required:
+ - CswFile (basename)
+ - related Srv4FileStats
+ - related Srv4FileInCatalog
+
+ Args:
+ basename: u'libfoo.so.1'
+ osrel: u'5.9'
+ arch: 'sparc', 'x86'
+ catrel: 'stable'
+
+ Returns:
+ {"/opt/csw/lib": ["CSWfoo", "CSWbar"],
+ "/opt/csw/1/lib": ["CSWfoomore"]}
+ """
+ pkgs = {}
+ sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
+ osrel, arch, catrel)
+
+ # Looks like this join is hard to do that way.
+ # res = m.Srv4FileInCatalog.select(
+ # sqlobject.AND(
+ # m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ # m.Srv4FileInCatalog.q.arch==sqo_arch,
+ # m.Srv4FileInCatalog.q.catrel==sqo_catrel)).
+ # throughTo.srv4file.thoughTo.files
+
+ # We'll implement it on the application level. First, we'll get all
+ # the files that match the basename, and then filter them based on
+ # catalog properties.
+ res = m.CswFile.select(m.CswFile.q.basename==basename)
+ file_list = []
+ for f in res:
+ # Check whether osrel, arch and catrel are matching.
+ for cat in f.srv4_file.in_catalogs:
+ if (f.srv4_file.registered
+ and cat.osrel == sqo_osrel
+ and cat.arch == sqo_arch
+ and cat.catrel == sqo_catrel):
+ file_list.append(f)
+ for obj in file_list:
+ pkgs.setdefault(obj.path, [])
+ pkgs[obj.path].append(obj.pkginst.pkgname)
+ logging.debug("self.error_mgr_mock.GetPathsAndPkgnamesByBasename(%s)"
+ ".AndReturn(%s)", repr(basename), pprint.pformat(pkgs))
+ return pkgs
+
+ def GetPkgByPath(self, full_file_path, osrel, arch, catrel):
+ """Returns a list of packages."""
+ # Memoization for performance
+
+ # Memoization won't buy us much. Perhaps we can fetch all the files
+ # belonging to the same package, so that we quickly prepopulate the cache.
+
+ logging.debug("GetPkgByPath(%s, %s, %s, %s)"
+ % (full_file_path, osrel, arch, catrel))
+ key = (full_file_path, osrel, arch, catrel)
+ if key not in self.pkgs_by_path_cache:
+ file_path, basename = os.path.split(full_file_path)
+ sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
+ osrel, arch, catrel)
+ oac = sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel)
+ path_filter = sqlobject.AND(
+ oac,
+ m.CswFile.q.path==file_path,
+ m.CswFile.q.basename==basename,
+ m.Srv4FileStats.q.registered==True)
+ join = [
+ sqlbuilder.INNERJOINOn(None,
+ m.Srv4FileStats,
+ m.Pkginst.q.id==m.Srv4FileStats.q.pkginst),
+ sqlbuilder.INNERJOINOn(None,
+ m.Srv4FileInCatalog,
+ m.Srv4FileStats.q.id==m.Srv4FileInCatalog.q.srv4file),
+ sqlbuilder.INNERJOINOn(None,
+ m.CswFile,
+ m.Srv4FileStats.q.id==m.CswFile.q.srv4_file),
+ ]
+ res = m.Pkginst.select(path_filter, join=join)
+ pkgs = [x.pkgname for x in res]
+ self.pkgs_by_path_cache[key] = frozenset(pkgs)
+ return self.pkgs_by_path_cache[key]
+
+ def CommonArchByString(self, s):
+ return sharedlib_utils.ArchByString(s)
+
+ def Srv4MatchesCatalog(self, sqo_srv4, sqo_arch):
+ cat_arch = self.CommonArchByString(sqo_arch.name)
+ pkg_arch = self.CommonArchByString(sqo_srv4.arch.name)
+ ans = (cat_arch == pkg_arch) or (pkg_arch == common_constants.ARCH_ALL)
+ if not ans:
+ logging.debug("Srv4MatchesCatalog(): mismatch: %s / %s and %s / %s",
+ cat_arch, pkg_arch, pkg_arch, common_constants.ARCH_ALL)
+ # Some packages have the architecture in the file saying 'all', but pkginfo
+ # says 'i386'.
+ filename_arch = self.CommonArchByString(sqo_srv4.filename_arch.name)
+ if filename_arch == common_constants.ARCH_ALL:
+ ans = True
+ if filename_arch != pkg_arch:
+ logging.warning(
+ "Package %s declares %s in pkginfo and %s in the filename.",
+ sqo_srv4, repr(pkg_arch), repr(filename_arch))
+ return ans
+
+ def AddSrv4ToCatalog(self, sqo_srv4, osrel, arch, catrel):
+ """Registers a srv4 file in a catalog."""
+ logging.debug("AddSrv4ToCatalog(%s, %s, %s, %s)",
+ sqo_srv4, osrel, arch, catrel)
+ sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
+ osrel, arch, catrel)
+ if not self.Srv4MatchesCatalog(sqo_srv4, sqo_arch):
+ raise CatalogDatabaseError(
+ "Specified package does not match the catalog. "
+ "Package: %s, catalog: %s %s %s"
+ % (sqo_srv4, osrel, arch, catrel))
+ if not sqo_srv4.registered:
+ raise CatalogDatabaseError(
+ "Package %s (%s) is not registered for releases."
+ % (sqo_srv4.basename, sqo_srv4.md5_sum))
+ # Checking for presence of a different srv4 with the same pkginst in the
+ # same catalog
+ pkginst = sqo_srv4.pkginst
+ res = m.Srv4FileStats.select(
+ m.Srv4FileStats.q.pkginst==pkginst).throughTo.in_catalogs.filter(
+ sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ m.Srv4FileInCatalog.q.srv4file!=sqo_srv4))
+ if len(list(res)):
+ raise CatalogDatabaseError(
+ "There already is a package with that pkgname: %s" % pkginst)
+ # Checking for presence of the same srv4 already in the catalog.
+ res = m.Srv4FileInCatalog.select(
+ sqlobject.AND(
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ m.Srv4FileInCatalog.q.srv4file==sqo_srv4))
+ if len(list(res)):
+ logging.debug("%s is already part of %s %s %s",
+ sqo_srv4, osrel, arch, catrel)
+ # Our srv4 is already part of that catalog.
+ return
+ obj = m.Srv4FileInCatalog(
+ arch=sqo_arch,
+ osrel=sqo_osrel,
+ catrel=sqo_catrel,
+ srv4file=sqo_srv4)
+
+ def RemoveSrv4(self, sqo_srv4, osrel, arch, catrel):
+ sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad(
+ osrel, arch, catrel)
+ sqo_srv4_in_cat = m.Srv4FileInCatalog.select(
+ sqlobject.AND(
+ m.Srv4FileInCatalog.q.arch==sqo_arch,
+ m.Srv4FileInCatalog.q.osrel==sqo_osrel,
+ m.Srv4FileInCatalog.q.catrel==sqo_catrel,
+ m.Srv4FileInCatalog.q.srv4file==sqo_srv4)).getOne()
+ sqo_srv4_in_cat.registered = False
+ # TODO(maciej): Remove all files belonging to that one
+ for cat_file in sqo_srv4_in_cat.srv4file.files:
+ cat_file.destroySelf()
+
+
+class Catalog(CatalogMixin):
+ pass
Added: csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py (rev 0)
+++ csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -0,0 +1,74 @@
+#!/usr/bin/env python2.6
+
+import copy
+import unittest
+import checkpkg_lib
+import tag
+import package_stats
+import database
+import sqlobject
+import models
+import package_stats
+import inspective_package
+import mox
+import test_base
+
+from testdata.neon_stats import pkgstats as neon_stats
+
+
+class CheckpkgManager2UnitTest(unittest.TestCase):
+
+ def setUp(self):
+ super(CheckpkgManager2UnitTest, self).setUp()
+ self.mox = mox.Mox()
+
+ def testSingleTag(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ tags = {
+ "CSWfoo": [
+ tag.CheckpkgTag("CSWfoo", "foo-tag", "foo-info"),
+ ],
+ }
+ screen_report, tags_report = m.FormatReports(tags, [], [])
+ expected = u'# Tags reported by testname module\nCSWfoo: foo-tag foo-info\n'
+ self.assertEqual(expected, unicode(tags_report))
+
+ def testThreeTags(self):
+ m = checkpkg_lib.CheckpkgManager2(
+ "testname", [], "5.9", "sparc", "unstable")
+ tags = {
+ "CSWfoo": [
+ tag.CheckpkgTag("CSWfoo", "foo-tag", "foo-info"),
+ tag.CheckpkgTag("CSWfoo", "bar-tag", "bar-info"),
+ tag.CheckpkgTag("CSWfoo", "baz-tag"),
+ ],
+ }
+ screen_report, tags_report = m.FormatReports(tags, [], [])
+ expected = (u'# Tags reported by testname module\n'
+ u'CSWfoo: foo-tag foo-info\n'
+ u'CSWfoo: bar-tag bar-info\n'
+ u'CSWfoo: baz-tag\n')
+ self.assertEqual(expected, unicode(tags_report))
+
+
+class CheckpkgManager2DatabaseIntegrationTest(
+ test_base.SqlObjectTestMixin, unittest.TestCase):
+
+ def setUp(self):
+ super(CheckpkgManager2DatabaseIntegrationTest, self).setUp()
+ self.mox = mox.Mox()
+
+ def testInsertNeon(self):
+ self.dbc.InitialDataImport()
+ sqo_pkg = package_stats.PackageStats.SaveStats(neon_stats[0], True)
+ cm = checkpkg_lib.CheckpkgManager2(
+ "testname", [sqo_pkg], "SunOS5.9", "sparc", "unstable",
+ show_progress=False)
+ cm.Run()
+ # Verifying that there are some reported error tags.
+ self.assertTrue(list(models.CheckpkgErrorTag.select()))
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: csw/mgar/gar/v2/lib/python/checkpkg_lib_test.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: csw/mgar/gar/v2/lib/python/checkpkg_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-12-11 00:26:22 UTC (rev 11881)
+++ csw/mgar/gar/v2/lib/python/checkpkg_test.py 2010-12-11 18:01:58 UTC (rev 11882)
@@ -1,16 +1,24 @@
-#!/opt/csw/bin/python2.6
+#!/usr/bin/env python2.6
# $Id$
+import copy
import re
import unittest
import mox
import difflib
import checkpkg
+import checkpkg_lib
+import database
+import models as m
import tag
-import testdata.dump_output_1 as dump_1
-import testdata.dump_output_2 as dump_2
-import testdata.dump_output_3 as dump_3
+import package_stats
+import sqlite3
+import sqlobject
+import test_base
+from testdata.tree_stats import pkgstats as tree_stats
+from testdata.neon_stats import pkgstats as neon_stats
+
"""A set of unit tests for the library checking code.
A bunch of lines to test in the interactive Python shell.
@@ -23,250 +31,6 @@
SELECT * FROM systempkgmap WHERE basename = 'libncursesw.so.5';
"""
-class GetLinesBySonameUnitTest(unittest.TestCase):
-
- def setUp(self):
- self.pkgmap_mocker = mox.Mox()
- self.e = checkpkg.LddEmulator()
-
- def testExpandRunpath_1(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/lib/$ISALIST"
- expected = ["/opt/csw/lib", "/opt/csw/lib/foo", "/opt/csw/lib/bar"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_2(self):
- isalist = ["foo", "bar"]
- runpath = "/opt/csw/mysql5/lib/$ISALIST/mysql"
- expected = ["/opt/csw/mysql5/lib/mysql",
- "/opt/csw/mysql5/lib/foo/mysql",
- "/opt/csw/mysql5/lib/bar/mysql"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_OriginSimple(self):
- isalist = ()
- runpath = "$ORIGIN"
- expected = ["/opt/csw/lib"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_OriginDots(self):
- isalist = ()
- runpath = "$ORIGIN/.."
- expected = ["/opt/csw/lib"]
- bin_path = "opt/csw/lib/subdir"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_Caching(self):
- """Make sure that the cache doesn't mess it up.
-
- Two invocations, where the only difference is the binary path.
- """
- isalist = ()
- runpath = "/opt/csw/lib/foo"
- expected = ["/opt/csw/lib/foo"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
- expected = ["/opt/csw/lib/foo"]
- bin_path = "/opt/csw/lib/foo"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_OriginCaching(self):
- """Make sure that the cache doesn't mess it up.
-
- Two invocations, where the only difference is the binary path.
- """
- isalist = ()
- runpath = "$ORIGIN"
- expected = ["/opt/csw/lib"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected,
- self.e.ExpandRunpath(runpath, isalist, bin_path))
- expected = ["/opt/csw/foo/lib"]
- bin_path = "/opt/csw/foo/lib"
- self.assertEquals(expected,
- self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testExpandRunpath_OnlyIsalist(self):
- """Make sure that the cache doesn't mess it up.
-
- Two invocations, where the only difference is the binary path.
- """
- isalist = ("bar",)
- runpath = "/opt/csw/lib/$ISALIST"
- expected = ["/opt/csw/lib", "/opt/csw/lib/bar"]
- bin_path = "opt/csw/lib"
- self.assertEquals(expected, self.e.ExpandRunpath(runpath, isalist, bin_path))
-
- def testEmulate64BitSymlinks_1(self):
- runpath_list = ["/opt/csw/mysql5/lib/foo/mysql/64"]
- expected = "/opt/csw/mysql5/lib/foo/mysql/amd64"
- self.assertTrue(expected in self.e.Emulate64BitSymlinks(runpath_list))
-
- def testEmulate64BitSymlinks_2(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/amd64/mysql/foo"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_3(self):
- runpath_list = ["/opt/csw/mysql5/lib/64/mysql/foo"]
- expected = "/opt/csw/mysql5/lib/sparcv9/mysql/foo"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulate64BitSymlinks_4(self):
- """No repeated paths because of symlink expansion"""
- runpath_list = ["/opt/csw/lib"]
- expected = "/opt/csw/lib"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertEquals(1, len(result), "len(%s) != %s" % (result, 1))
-
- def testEmulateSymlinks_3(self):
- runpath_list = ["/opt/csw/bdb4"]
- expected = "/opt/csw/bdb42"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_4(self):
- runpath_list = ["/opt/csw/bdb42"]
- expected = "/opt/csw/bdb42"
- not_expected = "/opt/csw/bdb422"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result,
- "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result,
- "%s is in %s" % (not_expected, result))
-
- def testEmulateSymlinks_5(self):
- """Install time symlink expansion."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- result = self.e.Emulate64BitSymlinks(runpath_list)
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
-
- def testEmulateSymlinks_6(self):
- """ExpandSymlink for /opt/csw/lib/i386."""
- runpath_list = ["/opt/csw/lib/i386"]
- expected = "/opt/csw/lib"
- not_expected = "/opt/csw/lib/i386"
- result = self.e.ExpandSymlink("/opt/csw/lib/i386",
- "/opt/csw/lib",
- "/opt/csw/lib/i386")
- self.assertTrue(expected in result, "%s not in %s" % (expected, result))
- self.assertFalse(not_expected in result,
- "%s is in %s" % (not_expected, result))
-
- def testSanitizeRunpath_1(self):
- self.assertEqual("/opt/csw/lib",
- self.e.SanitizeRunpath("/opt/csw/lib/"))
-
- def testSanitizeRunpath_2(self):
- self.assertEqual("/opt/csw/lib",
- self.e.SanitizeRunpath("/opt//csw////lib/"))
-
-
-
-class ParseDumpOutputUnitTest(unittest.TestCase):
-
- def test_1(self):
- expected = {
- 'RPATH set': True,
- 'RUNPATH RPATH the same': True,
- 'RUNPATH set': True,
- 'needed sonames': ('librt.so.1',
- 'libresolv.so.2',
- 'libc.so.1',
- 'libgen.so.1',
- 'libsocket.so.1',
- 'libnsl.so.1',
- 'libm.so.1',
- 'libz.so.1'),
- 'runpath': ('/opt/csw/lib/$ISALIST',
- '/opt/csw/lib',
- '/opt/csw/mysql5/lib/$ISALIST',
- '/opt/csw/mysql5/lib',
- '/opt/csw/mysql5/lib/$ISALIST/mysql'),
- 'soname': 'libmysqlclient.so.15',
- }
- self.assertEqual(expected,
- checkpkg.ParseDumpOutput(dump_1.DATA_DUMP_OUTPUT))
-
- def testEmpty(self):
- expected_runpath = ()
- self.assertEqual(
- expected_runpath,
- checkpkg.ParseDumpOutput(dump_2.DATA_DUMP_OUTPUT)["runpath"])
-
- def testRpathOnly(self):
- expected = {
- 'RPATH set': True,
- 'RUNPATH RPATH the same': False,
- 'RUNPATH set': False,
- 'needed sonames': ('librt.so.1',
- 'libresolv.so.2',
- 'libc.so.1',
- 'libgen.so.1',
- 'libsocket.so.1',
- 'libnsl.so.1',
- 'libm.so.1',
- 'libz.so.1'),
- 'runpath': ('/opt/csw/lib/$ISALIST',
- '/opt/csw/lib',
- '/opt/csw/mysql5/lib/$ISALIST',
- '/opt/csw/mysql5/lib',
- '/opt/csw/mysql5/lib/$ISALIST/mysql'),
- 'soname': 'libmysqlclient.so.15',
- }
- self.assertEqual(
- expected,
- checkpkg.ParseDumpOutput(dump_3.DATA_DUMP_OUTPUT))
-
-
-class SystemPkgmapUnitTest(unittest.TestCase):
-
- def testParsePkginfoLine(self):
- line = ('application CSWcswclassutils '
- 'cswclassutils - CSW class action utilities')
- expected = ('CSWcswclassutils',
- 'cswclassutils - CSW class action utilities')
- spkgmap = checkpkg.SystemPkgmap()
- self.assertEqual(expected, spkgmap._ParsePkginfoLine(line))
-
- def test_InferPackagesFromPkgmapLine(self):
- line = ("/opt/csw/sbin d none 0755 root bin CSWfping CSWbonobo2 "
- "CSWkrb5libdev CSWsasl CSWschilybase CSWschilyutils CSWstar "
- "CSWcommon CSWcacertificates CSWfacter")
- expected = ["CSWfping", "CSWbonobo2", "CSWkrb5libdev", "CSWsasl",
- "CSWschilybase", "CSWschilyutils", "CSWstar", "CSWcommon",
- "CSWcacertificates", "CSWfacter"]
- spkgmap = checkpkg.SystemPkgmap()
- self.assertEqual(expected, spkgmap._InferPackagesFromPkgmapLine(line))
-
- def test_InferPackagesFromPkgmapLine_2(self):
- line = ("/usr/lib/sparcv9/libpthread.so.1 f none 0755 root bin 41296 28258 "
- "1018129099 SUNWcslx")
- expected = ["SUNWcslx"]
- spkgmap = checkpkg.SystemPkgmap()
- self.assertEqual(expected, spkgmap._InferPackagesFromPkgmapLine(line))
-
- def test_InferPackagesFromPkgmapLine_3(self):
- line = ("/usr/lib/libCrun.so.1 f none 0755 root bin 63588 "
- "6287 1256043984 SUNWlibC")
- expected = ["SUNWlibC"]
- spkgmap = checkpkg.SystemPkgmap()
- self.assertEqual(expected, spkgmap._InferPackagesFromPkgmapLine(line))
-
- def test_InferPackagesFromPkgmapLine_4(self):
- line = ("/opt/csw/apache2/lib/libapr-1.so.0=libapr-1.so.0.3.8 s none "
- "CSWapache2rt")
- expected = ["CSWapache2rt"]
- spkgmap = checkpkg.SystemPkgmap()
- self.assertEqual(expected, spkgmap._InferPackagesFromPkgmapLine(line))
-
-
class ExtractorsUnitTest(unittest.TestCase):
def testExtractDescriptionFromGoodData(self):
@@ -296,68 +60,67 @@
self.assertEqual(expected, re.match(checkpkg.PSTAMP_RE, pstamp).groupdict())
-class CheckpkgManager2UnitTest(unittest.TestCase):
-
- def test_1(self):
- m = checkpkg.CheckpkgManager2("testname", "/tmp", ["CSWfoo"])
- tags = {
- "CSWfoo": [
- tag.CheckpkgTag("CSWfoo", "foo-tag", "foo-info"),
- ],
- }
- screen_report, tags_report = m.FormatReports(tags, [], [])
- expected = u'# Tags reported by testname module\nCSWfoo: foo-tag foo-info\n'
- self.assertEqual(expected, unicode(tags_report))
-
- def test_2(self):
- m = checkpkg.CheckpkgManager2("testname", "/tmp", ["CSWfoo"])
- tags = {
- "CSWfoo": [
- tag.CheckpkgTag("CSWfoo", "foo-tag", "foo-info"),
- tag.CheckpkgTag("CSWfoo", "bar-tag", "bar-info"),
- tag.CheckpkgTag("CSWfoo", "baz-tag"),
- ],
- }
- screen_report, tags_report = m.FormatReports(tags, [], [])
- expected = (u'# Tags reported by testname module\n'
- u'CSWfoo: foo-tag foo-info\n'
- u'CSWfoo: bar-tag bar-info\n'
- u'CSWfoo: baz-tag\n')
- self.assertEqual(expected, unicode(tags_report))
-
-
class SliceListUnitTest(unittest.TestCase):
def testOne(self):
l = [1, 2, 3, 4, 5]
s = 1
expected = [[1], [2], [3], [4], [5]]
- self.assertTrue(expected, checkpkg.SliceList(l, s))
+ self.assertTrue(expected, checkpkg_lib.SliceList(l, s))
def testTwo(self):
l = [1, 2, 3, 4, 5]
s = 2
expected = [[1, 2], [3, 4], [5]]
- self.assertTrue(expected, checkpkg.SliceList(l, s))
@@ Diff output truncated at 100000 characters. @@
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list