[csw-devel] SF.net SVN: gar:[10422] csw/mgar/gar/v2
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Jul 4 15:52:34 CEST 2010
Revision: 10422
http://gar.svn.sourceforge.net/gar/?rev=10422&view=rev
Author: wahwah
Date: 2010-07-04 13:52:34 +0000 (Sun, 04 Jul 2010)
Log Message:
-----------
mGAR v2: Architecture vs placement check improvements, less false positives.
Modified Paths:
--------------
csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/package_checks.py
csw/mgar/gar/v2/lib/python/package_checks_test.py
Modified: csw/mgar/gar/v2/bin/checkpkg_collect_stats.py
===================================================================
--- csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-07-04 13:51:05 UTC (rev 10421)
+++ csw/mgar/gar/v2/bin/checkpkg_collect_stats.py 2010-07-04 13:52:34 UTC (rev 10422)
@@ -12,6 +12,7 @@
import os.path
import subprocess
import sys
+import progressbar
# The following bit of code sets the correct path to Python libraries
# distributed with GAR.
@@ -58,13 +59,18 @@
stats_list.reverse()
total_packages = len(stats_list)
counter = itertools.count(1)
+ bar = progressbar.ProgressBar()
+ bar.maxval = total_packages
+ bar.start()
+ logging.debug("Making sure package statistics are collected.")
while stats_list:
# This way objects will get garbage collected as soon as they are removed
# from the list by pop(). The destructor (__del__()) of the srv4 class
# removes the temporary directory from the disk. This allows to process
# the whole catalog.
stats_list.pop().CollectStats()
- logging.debug("Collected stats %s of %s.", counter.next(), total_packages)
+ bar.update(counter.next())
+ bar.finish()
if __name__ == '__main__':
main()
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-04 13:51:05 UTC (rev 10421)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-04 13:52:34 UTC (rev 10422)
@@ -731,6 +731,8 @@
def GetOptimizedAllStats(self, stats_obj_list):
pkgs_data = []
+ counter = itertools.count()
+ length = len(stats_obj_list)
for stats_obj in stats_obj_list:
# pkg_data = {}
# This bit is tightly tied to the data structures returned by
@@ -738,7 +740,9 @@
#
# Python strings are already implementing the flyweight pattern. What's
# left is lists and dictionaries.
- logging.debug("Loading stats for %s", stats_obj.md5sum)
+ i = counter.next()
+ logging.debug("Loading stats for %s (%s/%s)",
+ stats_obj.md5sum, i, length)
raw_pkg_data = stats_obj.GetAllStats()
pkg_data = raw_pkg_data
pkgs_data.append(pkg_data)
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-04 13:51:05 UTC (rev 10421)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-04 13:52:34 UTC (rev 10422)
@@ -30,7 +30,10 @@
import checkpkg
from Cheetah import Template
-ARCHITECTURES = ["i386", "sparc", "all"]
+ARCH_SPARC = "sparc"
+ARCH_i386 = "i386"
+ARCH_ALL = "all"
+ARCHITECTURES = [ARCH_SPARC, ARCH_i386, ARCH_ALL]
MAJOR_VERSION = "major version"
MINOR_VERSION = "minor version"
PATCHLEVEL = "patchlevel"
Modified: csw/mgar/gar/v2/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-04 13:51:05 UTC (rev 10421)
+++ csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-04 13:52:34 UTC (rev 10422)
@@ -50,7 +50,7 @@
ARCH_RE = re.compile(r"(sparcv(8|9)|i386|amd64)")
MAX_CATALOGNAME_LENGTH = 20
MAX_PKGNAME_LENGTH = 20
-ARCH_LIST = ["sparc", "i386", "all"]
+ARCH_LIST = opencsw.ARCHITECTURES
VERSION_RE = r".*,REV=(20[01][0-9]\.[0-9][0-9]\.[0-9][0-9]).*"
# Pkgnames matching these regexes must not be ARCHALL = 1
ARCH_SPECIFIC_PKGNAMES_RE_LIST = [
@@ -90,24 +90,50 @@
# Valid URLs in the VENDOR field in pkginfo
VENDORURL_RE = r"^(http|ftp)s?\://.+\..+$"
-BASE_BINARY_PATHS = ('bin', 'lib', 'libexec')
-SPARCV8_PATHS = BASE_BINARY_PATHS + ('sparcv8', 'sparcv8-fsmuld',
- 'sparcv7', 'sparc')
+BASE_BINARY_PATHS = ('bin', 'sbin', 'lib', 'libexec', 'cgi-bin')
+SPARCV8_PATHS = ('sparcv8', 'sparcv8-fsmuld',
+ 'sparcv7', 'sparc')
SPARCV8PLUS_PATHS = ('sparcv8plus+vis2', 'sparcv8plus+vis', 'sparcv8plus')
SPARCV9_PATHS = ('sparcv9+vis2', 'sparcv9+vis', 'sparcv9')
-INTEL_386_PATHS = BASE_BINARY_PATHS + ('pentium_pro+mmx', 'pentium_pro',
- 'pentium+mmx', 'pentium',
- 'i486', 'i386', 'i86')
+INTEL_386_PATHS = ('pentium_pro+mmx', 'pentium_pro',
+ 'pentium+mmx', 'pentium',
+ 'i486', 'i386', 'i86')
AMD64_PATHS = ('amd64',)
HACHOIR_MACHINES = {
- # id: (name, allowed_paths)
- -1: ("Unknown", ()),
- 2: ("sparcv8", SPARCV8_PATHS),
- 3: ("i386", INTEL_386_PATHS),
- 6: ("i486", INTEL_386_PATHS),
- 18: ("sparcv8+", SPARCV8PLUS_PATHS),
- 43: ("sparcv9", SPARCV9_PATHS),
- 62: ("amd64", AMD64_PATHS),
+ # id: (name, allowed_paths, disallowed_paths)
+ -1: {"name": "Unknown",
+ "allowed": (), "disallowed": (),
+ "type": "unknown"},
+ 2: {"name": "sparcv8",
+ "type": opencsw.ARCH_SPARC,
+ "allowed": BASE_BINARY_PATHS + SPARCV8_PATHS,
+ "disallowed": SPARCV9_PATHS + INTEL_386_PATHS + AMD64_PATHS,
+ },
+ 3: {"name": "i386",
+ "type": opencsw.ARCH_i386,
+ "allowed": INTEL_386_PATHS,
+ "disallowed": (),
+ },
+ 6: {"name": "i486",
+ "type": opencsw.ARCH_i386,
+ "allowed": INTEL_386_PATHS,
+ "disallowed": (),
+ },
+ 18: {"name": "sparcv8+",
+ "type": opencsw.ARCH_SPARC,
+ "allowed": SPARCV8PLUS_PATHS,
+ "disallowed": (),
+ },
+ 43: {"name": "sparcv9",
+ "type": opencsw.ARCH_SPARC,
+ "allowed": SPARCV9_PATHS,
+ "disallowed": (),
+ },
+ 62: {"name": "amd64",
+ "type": opencsw.ARCH_i386,
+ "allowed": AMD64_PATHS,
+ "disallowed": (),
+ },
}
def RemovePackagesUnderInstallation(paths_and_pkgs_by_soname,
@@ -434,6 +460,12 @@
% (pkgname, arch))
messenger.SuggestGarLine("ARCHALL_%s = 1" % pkgname)
+
+# TODO: Verify that architecture type of binaries matches the actual binaries.
+# Correlate architecture type from files_metadata and HACHOIR_MACHINES with
+# pkginfo.
+
+
def CheckFileNameSanity(pkg_data, error_mgr, logger, messenger):
basic_stats = pkg_data["basic_stats"]
revision_info = basic_stats["parsed_basename"]["revision_info"]
@@ -820,20 +852,41 @@
if "machine_id" not in metadata:
continue
logger.debug("CheckArchitecture(): %s", metadata)
- cpu_type, allowed_paths = HACHOIR_MACHINES[metadata["machine_id"]]
- binary_path, unused_binary_name = os.path.split(metadata["path"])
- unused_dir, binary_subdir = os.path.split(binary_path)
- if binary_subdir not in allowed_paths:
+ machine_data = HACHOIR_MACHINES[metadata["machine_id"]]
+ cpu_type = machine_data["name"]
+ allowed_paths = set(machine_data["allowed"])
+ disallowed_paths = set(machine_data["disallowed"])
+ path_parts = set(metadata["path"].split(os.path.sep))
+ if not path_parts.intersection(allowed_paths):
error_mgr.ReportError(
- "binary-wrong-architecture",
- "id=%s name=%s subdir=%s" % (
+ "binary-architecture-does-not-match-placement",
+ "file=%s arch_id=%s arch_name=%s" % (
+ metadata["path"],
metadata["machine_id"],
- cpu_type,
- binary_subdir))
+ cpu_type))
messenger.Message(
"Files compiled for specific architectures must be placed in "
- "subdirectories that match the architecture. For more "
- "information, visit "
+ "subdirectories that match the architecture. "
+ "For example, a sparcv8+ binary must not be placed under "
+ "/opt/csw/lib, but under /opt/csw/lib/sparcv8plus. "
+ "For more information, visit "
"http://www.opencsw.org/extend-it/contribute-packages/"
"build-standards/"
"architecture-optimization-using-isaexec-and-isalist/")
+ else:
+ for bad_path in path_parts.intersection(disallowed_paths):
+ error_mgr.ReportError(
+ "binary-disallowed-placement",
+ "file=%s arch_id=%s arch_name=%s bad_path=%s" % (
+ metadata["path"],
+ metadata["machine_id"],
+ cpu_type,
+ bad_path))
+ messenger.Message(
+ "The %s binary is placed in a disallowed path. "
+ "For example, a sparcv8+ binary must not be placed "
+ "under a directory named sparcv9. "
+ "For more information, visit "
+ "http://www.opencsw.org/extend-it/contribute-packages/"
+ "build-standards/"
+ "architecture-optimization-using-isaexec-and-isalist/")
Modified: csw/mgar/gar/v2/lib/python/package_checks_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-07-04 13:51:05 UTC (rev 10421)
+++ csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-07-04 13:52:34 UTC (rev 10422)
@@ -698,6 +698,32 @@
'path': 'opt/csw/bin/tree'}]
+class TestCheckArchitecture_LibSubdir(CheckpkgUnitTestHelper,
+ unittest.TestCase):
+ FUNCTION_NAME = "CheckArchitecture"
+ def CheckpkgTest(self):
+ self.pkg_data["files_metadata"] = [
+ {'endian': 'Big endian',
+ 'machine_id': 2,
+ 'mime_type': 'application/x-sharedlib; charset=binary',
+ 'path': 'opt/csw/lib/foo/subdir/libfoo.so.1'}]
+
+
+class TestCheckArchitecture_LibSubdirWrong(CheckpkgUnitTestHelper,
+ unittest.TestCase):
+ FUNCTION_NAME = "CheckArchitecture"
+ def CheckpkgTest(self):
+ self.pkg_data["files_metadata"] = [
+ {'endian': 'Big endian',
+ 'machine_id': 2,
+ 'mime_type': 'application/x-sharedlib; charset=binary',
+ 'path': 'opt/csw/lib/sparcv9/foo/subdir/libfoo.so.1'}]
+ self.error_mgr_mock.ReportError(
+ 'binary-wrong-wrong-placement',
+ 'file=opt/csw/lib/sparcv9/foo/subdir/libfoo.so.1 '
+ 'arch_id=2 arch_name=sparcv8 bad_path=sparcv9')
+
+
class TestConflictingFiles(CheckpkgUnitTestHelper,
unittest.TestCase):
"""Throw an error if there's a conflicting file in the package set."""
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list