[csw-devel] SF.net SVN: gar:[10410] csw/mgar/gar/v2/lib/python
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sat Jul 3 22:12:12 CEST 2010
Revision: 10410
http://gar.svn.sourceforge.net/gar/?rev=10410&view=rev
Author: wahwah
Date: 2010-07-03 20:12:11 +0000 (Sat, 03 Jul 2010)
Log Message:
-----------
mGAR v2: checkpkg, checking binary architectures, sparcv8 vs sparcv8+ vs sparcv9.
Modified Paths:
--------------
csw/mgar/gar/v2/lib/python/checkpkg.py
csw/mgar/gar/v2/lib/python/opencsw.py
csw/mgar/gar/v2/lib/python/package_checks.py
csw/mgar/gar/v2/lib/python/package_checks_test.py
Added Paths:
-----------
csw/mgar/gar/v2/lib/python/testdata/stats/46/461a24f02dd5020b4aa014b76f3ec2cc/files_metadata.yml
Modified: csw/mgar/gar/v2/lib/python/checkpkg.py
===================================================================
--- csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-03 15:57:24 UTC (rev 10409)
+++ csw/mgar/gar/v2/lib/python/checkpkg.py 2010-07-03 20:12:11 UTC (rev 10410)
@@ -29,7 +29,7 @@
DEBUG_BREAK_PKGMAP_AFTER = False
DB_SCHEMA_VERSION = 3L
-PACKAGE_STATS_VERSION = 5L
+PACKAGE_STATS_VERSION = 6L
SYSTEM_PKGMAP = "/var/sadm/install/contents"
WS_RE = re.compile(r"\s+")
NEEDED_SONAMES = "needed sonames"
@@ -204,11 +204,8 @@
class SystemPkgmap(object):
- """A class to hold and manipulate the /var/sadm/install/contents file.
+ """A class to hold and manipulate the /var/sadm/install/contents file."""
- TODO: Implement timestamp checking and refreshing the cache.
- """
-
STOP_PKGS = ["SUNWbcp", "SUNWowbcp", "SUNWucb"]
CHECKPKG_DIR = ".checkpkg"
SQLITE3_DBNAME_TMPL = "var-sadm-install-contents-cache-%s"
@@ -1033,6 +1030,7 @@
# This entry needs to be last because of the assumption in the
# CollectStats() function.
"basic_stats",
+ "files_metadata",
]
def __init__(self, srv4_pkg, stats_basedir=None, md5sum=None):
@@ -1282,6 +1280,7 @@
# in one of the previous runs, the basic_stats.pickle file is not there
# or not updated, and the collection is started again.
self.DumpObject(self.GetBasicStats(), "basic_stats")
+ self.DumpObject(dir_pkg.GetFilesMetadata(), "files_metadata")
logging.debug("Statistics of %s have been collected.", repr(dir_pkg.pkgname))
def GetAllStats(self):
Modified: csw/mgar/gar/v2/lib/python/opencsw.py
===================================================================
--- csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-03 15:57:24 UTC (rev 10409)
+++ csw/mgar/gar/v2/lib/python/opencsw.py 2010-07-03 20:12:11 UTC (rev 10410)
@@ -12,14 +12,11 @@
# the terms of the GNU General Public License version 2 as published by the
# Free Software Foundation.
-ENABLE_HACHOIR = False
-
import copy
import datetime
import difflib
-if ENABLE_HACHOIR:
- import hachoir_parser as hp
- import hachoir_core as hc
+import hachoir_parser as hp
+import hachoir_core as hc
import hashlib
import magic
import logging
@@ -45,7 +42,10 @@
PKG_URL_TMPL = "http://www.opencsw.org/packages/%s"
CATALOG_URL = "http://mirror.opencsw.org/opencsw/current/i386/5.10/catalog"
WS_RE = re.compile(r"\s+")
-
+BIN_MIMETYPES = (
+ 'application/x-executable',
+ 'application/x-sharedlib',
+)
ADMIN_FILE_CONTENT = """
basedir=default
runlevel=nocheck
@@ -90,6 +90,7 @@
Generated by submitpkg
"""
+
class Error(Exception):
pass
@@ -653,7 +654,6 @@
Allows some read-write operations.
"""
-
def __init__(self, directory):
self.directory = directory
self.pkgname = os.path.basename(directory)
@@ -813,30 +813,23 @@
magic_cookie.load()
magic_cookie.setflags(magic.MAGIC_MIME)
for file_path in all_files:
+ full_path = unicode(self.MakeAbsolutePath(file_path))
file_info = {
"path": StripRe(file_path, root_re),
- "mime_type": None,
+ "mime_type": magic_cookie.file(full_path),
}
- full_path = unicode(self.MakeAbsolutePath(file_path))
- if ENABLE_HACHOIR:
+ if IsBinary(file_info):
parser = hp.createParser(full_path)
if not parser:
- print "Can't parse file %s" % (file_path)
+ logging.warning("Can't parse file %s", file_path)
else:
- print "found file: %s, it's a %s" % (file_path, parser.mime_type)
- file_info["mime_type"] = parser.mime_type
- f = parser["/header/machine"]
- print "/header/machine: ", (f, f.display, f.value)
- i = 0
- while True:
- try:
- f = parser["/header"].getField(i)
- print "Field", i, ": ", (f, f.display, f.value)
- except hc.field.field.MissingField:
- print "No field number", i
- break
- i += 1
- file_info["mime_type"] = magic_cookie.file(full_path)
+ file_info["mime_type_by_hachoir"] = parser.mime_type
+ machine_id = parser["/header/machine"].value
+ file_info["machine_id"] = machine_id
+ file_info["endian"] = parser["/header/endian"].display
+ else:
+ logging.debug("%s is not a binary, or hachoir is disabled.",
+ full_path)
self.files_metadata.append(file_info)
return self.files_metadata
@@ -861,19 +854,14 @@
Now that there are files_metadata, this function can safely go away, once
all its callers are modified to use files_metadata instead.
"""
- bin_mimetypes = (
- 'application/x-executable',
- 'application/x-sharedlib',
- )
if self.binaries is None:
self.CheckPkgpathExists()
files_metadata = self.GetFilesMetadata()
self.binaries = []
# The nested for-loop looks inefficient.
for file_info in files_metadata:
- for mimetype in bin_mimetypes:
- if mimetype in file_info["mime_type"]:
- self.binaries.append(file_info["path"])
+ if IsBinary(file_info):
+ self.binaries.append(file_info["path"])
self.binaries.sort()
return self.binaries
@@ -1193,3 +1181,13 @@
logging.error("%s is missing the file_basename field", d)
self.by_basename[d["file_basename"]] = d
return self.by_basename
+
+def IsBinary(file_info):
+ """Returns True or False depending on file metadata."""
+ is_a_binary = False
+ logging.debug("IsBinary(%s)", repr(file_info))
+ for mimetype in BIN_MIMETYPES:
+ if mimetype in file_info["mime_type"]:
+ is_a_binary = True
+ break
+ return is_a_binary
Modified: csw/mgar/gar/v2/lib/python/package_checks.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-03 15:57:24 UTC (rev 10409)
+++ csw/mgar/gar/v2/lib/python/package_checks.py 2010-07-03 20:12:11 UTC (rev 10410)
@@ -86,7 +86,27 @@
# Valid URLs in the VENDOR field in pkginfo
VENDORURL_RE = r"^(http|ftp)s?\://.+\..+$"
+BASE_BINARY_PATHS = ('bin', 'lib', 'libexec')
+SPARCV8_PATHS = BASE_BINARY_PATHS + ('sparcv8', 'sparcv8-fsmuld',
+ 'sparcv7', 'sparc')
+SPARCV8PLUS_PATHS = ('sparcv8plus+vis2', 'sparcv8plus+vis', 'sparcv8plus')
+SPARCV9_PATHS = ('sparcv9+vis2', 'sparcv9+vis', 'sparcv9')
+INTEL_386_PATHS = BASE_BINARY_PATHS + ('pentium_pro+mmx', 'pentium_pro',
+ 'pentium+mmx', 'pentium',
+ 'i486', 'i386', 'i86')
+AMD64_PATHS = ('amd64',)
+HACHOIR_MACHINES = {
+ # id: (name, allowed_paths)
+ -1: ("Unknown", ()),
+ 2: ("sparcv8", SPARCV8_PATHS),
+ 3: ("i386", INTEL_386_PATHS),
+ 6: ("i486", INTEL_386_PATHS),
+ 18: ("sparcv8+", SPARCV8PLUS_PATHS),
+ 43: ("sparcv9", SPARCV9_PATHS),
+ 62: ("amd64", AMD64_PATHS),
+}
+
def CatalognameLowercase(pkg_data, error_mgr, logger, messenger):
catalogname = pkg_data["basic_stats"]["catalogname"]
if catalogname != catalogname.lower():
@@ -771,3 +791,28 @@
"For example, %s. "
"However, the catalogname doesn't start with 'py_'."
% repr(example_py_file))
+
+
+def CheckArchitecture(pkg_data, error_mgr, logger, messenger):
+ pkgname = pkg_data["basic_stats"]["pkgname"]
+ for metadata in pkg_data["files_metadata"]:
+ if "machine_id" not in metadata:
+ continue
+ logger.debug("CheckArchitecture(): %s", metadata)
+ cpu_type, allowed_paths = HACHOIR_MACHINES[metadata["machine_id"]]
+ binary_path, unused_binary_name = os.path.split(metadata["path"])
+ unused_dir, binary_subdir = os.path.split(binary_path)
+ if binary_subdir not in allowed_paths:
+ error_mgr.ReportError(
+ "binary-wrong-architecture",
+ "id=%s name=%s subdir=%s" % (
+ metadata["machine_id"],
+ cpu_type,
+ binary_subdir))
+ messenger.Message(
+ "Files compiled for specific architectures must be placed in "
+ "subdirectories that match the architecture. For more "
+ "information, visit "
+ "http://www.opencsw.org/extend-it/contribute-packages/"
+ "build-standards/"
+ "architecture-optimization-using-isaexec-and-isalist/")
Modified: csw/mgar/gar/v2/lib/python/package_checks_test.py
===================================================================
--- csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-07-03 15:57:24 UTC (rev 10409)
+++ csw/mgar/gar/v2/lib/python/package_checks_test.py 2010-07-03 20:12:11 UTC (rev 10410)
@@ -655,7 +655,49 @@
'This path is already provided by CSWcommon '
'or is not allowed for other reasons.')
+class TestCheckArchitecture_sparcv8plus(CheckpkgUnitTestHelper,
+ unittest.TestCase):
+ FUNCTION_NAME = "CheckArchitecture"
+ def CheckpkgTest(self):
+ self.pkg_data["files_metadata"] = [
+ {'endian': 'Big endian',
+ 'machine_id': 18,
+ 'mime_type': 'application/x-executable; charset=binary',
+ 'mime_type_by_hachoir': u'application/x-executable',
+ 'path': 'opt/csw/bin/tree'},
+ {'mime_type': 'text/troff; charset=us-ascii',
+ 'path': 'opt/csw/share/man/man1/tree.1'},
+ {'mime_type': 'text/plain; charset=us-ascii',
+ 'path': 'opt/csw/share/doc/tree/license'}]
+ self.error_mgr_mock.ReportError('binary-wrong-architecture',
+ 'id=18 name=sparcv8+ subdir=bin')
+
+class TestCheckArchitecture_sparcv8plus(CheckpkgUnitTestHelper,
+ unittest.TestCase):
+ FUNCTION_NAME = "CheckArchitecture"
+ def CheckpkgTest(self):
+ self.pkg_data["files_metadata"] = [
+ {'endian': 'Big endian',
+ 'machine_id': 18,
+ 'mime_type': 'application/x-executable; charset=binary',
+ 'mime_type_by_hachoir': u'application/x-executable',
+ 'path': 'opt/csw/bin/sparcv8plus/tree'},
+ ]
+
+
+class TestCheckArchitecture_sparcv8(CheckpkgUnitTestHelper,
+ unittest.TestCase):
+ FUNCTION_NAME = "CheckArchitecture"
+ def CheckpkgTest(self):
+ self.pkg_data["files_metadata"] = [
+ {'endian': 'Big endian',
+ 'machine_id': 2,
+ 'mime_type': 'application/x-executable; charset=binary',
+ 'mime_type_by_hachoir': u'application/x-executable',
+ 'path': 'opt/csw/bin/tree'}]
+
+
class TestConflictingFiles(CheckpkgUnitTestHelper,
unittest.TestCase):
"""Throw an error if there's a conflicting file in the package set."""
Added: csw/mgar/gar/v2/lib/python/testdata/stats/46/461a24f02dd5020b4aa014b76f3ec2cc/files_metadata.yml
===================================================================
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list